1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_event_declarations,
25 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
26 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
27 },
28 scope::{
29 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
30 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
31 SCOPE_SYNC_OP,
32 },
33 session::{
34 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
35 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
36 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
37 ROLE_VERSIONS_FIELD,
38 },
39 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
40 Priority, RouteTarget, PROTOCOL_VERSION,
41};
42use tokio::time::{timeout_at, Instant};
43use tracing::{debug, info, warn};
44
45use crate::{
46 caller_identity::CallerIdentity,
47 capability_requirements::{
48 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
49 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
50 RuntimeModule,
51 },
52 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
53 forwarding::{
54 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
55 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
56 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
57 },
58 observability::{
59 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
60 },
61 privileged_prompt::{prompt_text, PrivilegedOperation},
62 provenance::{
63 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
64 },
65 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
66 router::{RouteCtx, RouterError},
67 scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
68 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
69 stderr_tail::{CaptureState, TailEntry},
70 supervise::{
71 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
72 SupervisorHandle, SwapHelloAdmission,
73 },
74 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
75};
76
77pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
83
84const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
85const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
86const CAP_PING_PONG: &str = "ping_pong_v1";
87const CAP_SESSION_ATTACH: &str = "session_attach_v1";
88const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
89
90const SUBC_CONTROL_OPS: &[&str] = &[
91 ops::ACTIVITY_RECORD,
92 ops::ACTIVITY_QUERY,
93 ops::ACTIVITY_SUBSCRIBE,
94 ops::SERVER_DESCRIBE,
95 ops::CATALOG_LIST,
96 ops::ROUTE_OPEN,
97 ops::ROUTE_POLL,
98 ops::ROUTE_CLOSING,
99 ops::ROUTE_CLOSED,
100 ops::SUPERVISOR_LIST,
101 ops::SUPERVISOR_RESTART,
102 ops::SUPERVISOR_SWAP,
103 ops::SUPERVISOR_RELOAD,
104 ops::SUPERVISOR_RESCAN,
105 ops::SUPERVISOR_RELEASE_RESERVED,
106 ops::SUPERVISOR_SET_ENABLED,
107 ops::SUPERVISOR_HEALTH_PROBE,
108 ops::SUPERVISOR_HEALTH,
109 ops::SUPERVISOR_STDERR_TAIL,
110 ops::SUPERVISOR_TERMINALS,
111 ops::SUPERVISOR_ROUTES,
112 ops::SUPERVISOR_PROVENANCE,
113 ops::SUPERVISOR_SPAWN_SNAPSHOT,
114 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
115];
116
117const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
118 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
119 "supervisor.live_roots",
120 SCOPE_SYNC_OP,
121 SCOPE_APPLY_OP,
122 SCOPE_DESCRIBE_OP,
123 "operator.confirm",
124];
125
126const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
130
131const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
132
133pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
141
142pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
157
158pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
176
177const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
178const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
179
180fn reload_verdict(
181 configured: &Path,
182 spawned_from: Option<&Path>,
183 image: subc_control::RunningImageAgreement,
184) -> PendingReloadVerdict {
185 let path = match spawned_from {
186 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
187 ReloadPathAgreement::Match
188 }
189 Some(spawned_from) => ReloadPathAgreement::Mismatch {
190 configured: configured.to_path_buf(),
191 spawned_from: spawned_from.to_path_buf(),
192 },
193 None => ReloadPathAgreement::Unavailable {
194 reason: if matches!(
195 image,
196 subc_control::RunningImageAgreement::Unavailable {
197 reason: subc_control::RunningImageUnavailableReason::NotRunning
198 }
199 ) {
200 ReloadPathUnavailableReason::NotRunning
201 } else {
202 ReloadPathUnavailableReason::SpawnedPathUnavailable
203 },
204 },
205 };
206 PendingReloadVerdict { path, image }
207}
208
209fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
210 if configured == spawned_from {
211 return true;
212 }
213 #[cfg(windows)]
214 {
215 match (
221 std::fs::canonicalize(configured),
222 std::fs::canonicalize(spawned_from),
223 ) {
224 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
225 _ => false,
226 }
227 }
228 #[cfg(not(windows))]
229 {
230 false
231 }
232}
233
234#[derive(Clone)]
235struct DaemonProvenanceFacts {
236 build: DaemonBuildProvenance,
237 pid: Option<u32>,
238 started_at_ms: Option<u64>,
239 start_clock: Option<crate::clock::StartClock>,
240 executable_path: Option<PathBuf>,
241 executable_identity: Option<SpawnedFileIdentity>,
242 process_start_time: Option<u64>,
243 probe: ExecutableIdentityProbe,
244}
245
246impl Default for DaemonProvenanceFacts {
247 fn default() -> Self {
248 Self {
249 build: DaemonBuildProvenance {
250 build_git_sha: None,
251 build_lock_digest: None,
252 },
253 pid: None,
254 started_at_ms: None,
255 start_clock: None,
256 executable_path: None,
257 executable_identity: None,
258 process_start_time: None,
259 probe: ExecutableIdentityProbe::default(),
260 }
261 }
262}
263
264#[derive(Debug, Clone)]
265struct SupervisorRescanContext {
266 supervisor: Supervisor,
267 config_path: PathBuf,
268 configured_port: Option<u16>,
269 storage_config: Option<crate::daemon_config::StorageConfig>,
270 admission_facts_carrier_module_id: Option<String>,
271 admission_facts_targets: Option<Vec<String>>,
272 scope_authority_owners: Vec<String>,
273 privileged_operator_prompt: PrivilegedOperatorPrompt,
274}
275
276const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
297 "reloading",
298 "supervisor_not_live",
299 "registration_not_active",
300 "no_forwarding_connection",
301 "relay_send_failed",
302];
303
304#[derive(Clone)]
306pub struct ControlHandler {
307 registry: Arc<Registry>,
308 activity: Option<crate::activity::ActivityLedger>,
309 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
310 forwarding: Arc<ForwardingTable>,
311 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
312 supervisor: SupervisorHandle,
313 subc_capabilities: Arc<[String]>,
314 route_bind_relay_timeout: Duration,
318 route_bind_relay_timeouts: BTreeMap<String, Duration>,
322 route_bind_breakers: RouteBindBreakers,
325 route_bind_concurrency: RouteBindConcurrency,
328 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
333 route_bind_breaker_threshold: u32,
335 route_bind_breaker_cooldown: Duration,
337 health_probe_timeout: Duration,
338 storage_config: Option<crate::daemon_config::StorageConfig>,
341 machine_id: Option<crate::machine_id::MachineId>,
345 admission_facts_carrier_module_id: Option<String>,
346 admission_facts_targets: Option<Vec<String>>,
347 scopes: Arc<RwLock<ScopeTable>>,
351 scope_deadline_added: Arc<tokio::sync::Notify>,
355 scope_authority_owners: Vec<String>,
358 privileged_operator_prompt: PrivilegedOperatorPrompt,
360 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
363 #[cfg(test)]
364 caller_lookup: Option<TestCallerLookup>,
365 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
368 rescan: Option<SupervisorRescanContext>,
369 connected_clients: ConnectedClients,
370 counters: DaemonCounters,
371 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
372 daemon_provenance: DaemonProvenanceFacts,
373 #[cfg(test)]
374 control_dispatch_delay: Option<Duration>,
375 #[cfg(test)]
376 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
377 #[cfg(test)]
381 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
382 #[cfg(test)]
388 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
389}
390
391impl fmt::Debug for ControlHandler {
392 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
393 f.debug_struct("ControlHandler")
394 .field("registry", &self.registry)
395 .field("forwarding", &self.forwarding)
396 .field("process_liveness", &self.process_liveness.is_some())
397 .field("supervisor", &self.supervisor)
398 .field("subc_capabilities", &self.subc_capabilities)
399 .finish()
400 }
401}
402
403struct RouteOpenRequest {
404 target: RouteTarget,
405 identity: BindIdentity,
406 consumer_identity: Option<ConsumerIdentity>,
407 consumer_capabilities: Option<Vec<String>>,
408 role_versions: Option<BTreeMap<String, String>>,
409 admission_facts: Option<serde_json::Value>,
410 scope: Option<ScopeSelector>,
411}
412
413struct RouteBindReservationGuard {
414 forwarding: Arc<ForwardingTable>,
415 endpoint: ModuleEndpointId,
416 relay_corr: u64,
417 armed: bool,
418}
419
420struct ModuleControlRpcGuard {
421 forwarding: Arc<ForwardingTable>,
422 endpoint: ModuleEndpointId,
423 corr: u64,
424 armed: bool,
425}
426
427impl ModuleControlRpcGuard {
428 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
429 Self {
430 forwarding,
431 endpoint,
432 corr,
433 armed: true,
434 }
435 }
436
437 fn disarm(&mut self) {
438 self.armed = false;
439 }
440}
441
442impl Drop for ModuleControlRpcGuard {
443 fn drop(&mut self) {
444 if self.armed {
445 let _ = self
446 .forwarding
447 .cancel_module_control_rpc(self.endpoint, self.corr);
448 }
449 }
450}
451
452impl RouteBindReservationGuard {
453 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
454 Self {
455 forwarding,
456 endpoint,
457 relay_corr,
458 armed: true,
459 }
460 }
461
462 fn release_and_disarm(&mut self) {
463 if !self.armed {
464 return;
465 }
466 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
467 self.endpoint,
468 self.relay_corr,
469 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
470 ) {
471 send_goodbye_target_best_effort(
472 &self.forwarding.counters(),
473 &target,
474 "canceled route.bind",
475 );
476 }
477 self.armed = false;
478 }
479
480 fn disarm(&mut self) {
481 self.armed = false;
482 }
483}
484
485impl Drop for RouteBindReservationGuard {
486 fn drop(&mut self) {
487 self.release_and_disarm();
488 }
489}
490
491#[derive(Debug, Clone, Default)]
521pub(crate) struct RouteBindBreakers {
522 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
523}
524
525#[derive(Debug, Clone, Default)]
526pub(crate) struct RouteBindConcurrency {
527 modules: Arc<Mutex<HashMap<String, usize>>>,
528}
529
530struct RouteBindConcurrencyGuard {
531 concurrency: RouteBindConcurrency,
532 module_id: String,
533}
534
535impl RouteBindConcurrency {
536 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
539 let mut modules = self
540 .modules
541 .lock()
542 .expect("route.bind concurrency mutex poisoned");
543 let in_flight = modules.entry(module_id.to_string()).or_default();
544 if *in_flight >= limit {
545 return Err(*in_flight);
546 }
547 *in_flight += 1;
548 Ok(RouteBindConcurrencyGuard {
549 concurrency: self.clone(),
550 module_id: module_id.to_string(),
551 })
552 }
553}
554
555impl Drop for RouteBindConcurrencyGuard {
556 fn drop(&mut self) {
557 let mut modules = self
558 .concurrency
559 .modules
560 .lock()
561 .expect("route.bind concurrency mutex poisoned");
562 let remove = {
563 let in_flight = modules
564 .get_mut(&self.module_id)
565 .expect("admitted route.bind has a concurrency entry");
566 *in_flight -= 1;
567 *in_flight == 0
568 };
569 if remove {
570 modules.remove(&self.module_id);
571 }
572 }
573}
574
575#[derive(Debug, Default)]
576struct ModuleBreakerState {
577 consecutive_timeouts: u32,
579 cooldown_until: Option<Instant>,
582 probe_in_flight: Option<Arc<()>>,
587}
588
589enum RouteBindAdmission<'a> {
591 Admitted {
592 guard: RouteBindBreakerGuard<'a>,
593 probe: bool,
596 },
597 Refused {
598 consecutive_timeouts: u32,
599 retry_in: Duration,
603 probe_in_flight: bool,
604 },
605}
606
607struct RouteBindBreakerGuard<'a> {
614 breakers: RouteBindBreakers,
615 module_id: &'a str,
616 probe_token: Option<Arc<()>>,
617 settled: bool,
618}
619
620impl RouteBindBreakerGuard<'_> {
621 fn record_accepted(&mut self) -> bool {
625 self.settled = true;
626 self.breakers.record_accepted(self.module_id)
627 }
628
629 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
632 self.settled = true;
633 self.breakers.record_timeout(
634 self.module_id,
635 self.probe_token.as_ref(),
636 threshold,
637 cooldown,
638 )
639 }
640
641 fn record_inconclusive(&mut self) {
650 self.settled = true;
651 self.breakers
652 .record_inconclusive(self.module_id, self.probe_token.as_ref());
653 }
654}
655
656impl Drop for RouteBindBreakerGuard<'_> {
657 fn drop(&mut self) {
658 if !self.settled {
659 self.breakers
660 .record_inconclusive(self.module_id, self.probe_token.as_ref());
661 }
662 }
663}
664
665struct BreakerOpened {
669 consecutive_timeouts: u32,
670 reopened_after_probe: bool,
673}
674
675impl RouteBindBreakers {
676 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
677 self.modules
678 .lock()
679 .expect("route.bind breaker mutex poisoned")
680 }
681
682 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
685 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
686 probe: probe_token.is_some(),
687 guard: RouteBindBreakerGuard {
688 breakers: self.clone(),
689 module_id,
690 probe_token,
691 settled: false,
692 },
693 };
694
695 let mut modules = self.lock();
696 let Some(state) = modules.get_mut(module_id) else {
697 return admitted(None);
698 };
699 let Some(cooldown_until) = state.cooldown_until else {
700 return admitted(None);
701 };
702 if state.probe_in_flight.is_some() {
703 return RouteBindAdmission::Refused {
704 consecutive_timeouts: state.consecutive_timeouts,
705 retry_in: Duration::ZERO,
706 probe_in_flight: true,
707 };
708 }
709 let now = Instant::now();
710 if now < cooldown_until {
711 return RouteBindAdmission::Refused {
712 consecutive_timeouts: state.consecutive_timeouts,
713 retry_in: cooldown_until - now,
714 probe_in_flight: false,
715 };
716 }
717 let token = Arc::new(());
718 state.probe_in_flight = Some(Arc::clone(&token));
719 admitted(Some(token))
720 }
721
722 fn record_accepted(&self, module_id: &str) -> bool {
723 self.lock()
724 .remove(module_id)
725 .is_some_and(|state| state.cooldown_until.is_some())
726 }
727
728 fn record_timeout(
729 &self,
730 module_id: &str,
731 probe_token: Option<&Arc<()>>,
732 threshold: u32,
733 cooldown: Duration,
734 ) -> Option<BreakerOpened> {
735 let mut modules = self.lock();
736 let state = modules.entry(module_id.to_string()).or_default();
737 let was_open = state.cooldown_until.is_some();
738 let was_probe = Self::owns_probe(state, probe_token);
739 if was_probe {
740 state.probe_in_flight = None;
741 }
742 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
743 if state.consecutive_timeouts < threshold {
744 return None;
745 }
746 state.cooldown_until = Some(Instant::now() + cooldown);
747 Some(BreakerOpened {
748 consecutive_timeouts: state.consecutive_timeouts,
749 reopened_after_probe: was_open && was_probe,
750 })
751 }
752
753 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
754 state
758 .probe_in_flight
759 .as_ref()
760 .zip(token)
761 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
762 }
763
764 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
765 if let Some(state) = self.lock().get_mut(module_id) {
766 if Self::owns_probe(state, probe_token) {
767 state.probe_in_flight = None;
768 }
769 }
770 }
771
772 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
787 self.lock()
788 .remove(module_id)
789 .map(|state| state.consecutive_timeouts)
790 .filter(|discarded| *discarded > 0)
791 }
792
793 fn open_snapshot(&self) -> Option<serde_json::Value> {
800 let now = Instant::now();
801 let modules = self.lock();
802 let open = modules
803 .iter()
804 .filter_map(|(module_id, state)| {
805 let cooldown_until = state.cooldown_until?;
806 Some((
807 module_id.clone(),
808 serde_json::json!({
809 "consecutive_timeouts": state.consecutive_timeouts,
810 "cooldown_remaining_ms":
811 cooldown_until.saturating_duration_since(now).as_millis() as u64,
812 "probe_in_flight": state.probe_in_flight.is_some(),
813 }),
814 ))
815 })
816 .collect::<serde_json::Map<String, serde_json::Value>>();
817 (!open.is_empty()).then_some(serde_json::Value::Object(open))
818 }
819}
820
821impl ControlHandler {
822 pub fn new(registry: Arc<Registry>) -> Self {
823 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
824 }
825
826 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
827 let counters = forwarding.counters();
828 let route_bind_breakers = forwarding.route_bind_breakers();
832 let route_bind_concurrency = forwarding.route_bind_concurrency();
833 let route_outages = forwarding.route_outages();
834 Self {
835 registry,
836 activity: None,
837 activity_streams: Arc::new(Mutex::new(HashMap::new())),
838 forwarding,
839 process_liveness: None,
840 supervisor: SupervisorHandle::new(),
841 subc_capabilities: Arc::from([
842 CAP_MANIFEST_REGISTRATION.to_string(),
843 CAP_CHANNEL_LIFECYCLE.to_string(),
844 CAP_PING_PONG.to_string(),
845 CAP_SESSION_ATTACH.to_string(),
846 CAP_ADMISSION_FACTS_RELAY.to_string(),
847 CAP_SCOPES_V1.to_string(),
848 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
849 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
850 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
851 ]),
852 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
853 route_bind_relay_timeouts: BTreeMap::new(),
854 route_bind_breakers,
855 route_bind_concurrency,
856 route_outages,
857 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
858 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
859 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
860 storage_config: None,
861 machine_id: None,
862 admission_facts_carrier_module_id: None,
863 admission_facts_targets: None,
864 scopes: Arc::new(RwLock::new(ScopeTable::new(
865 crate::daemon_config::default_scope_authority_owners(),
866 ))),
867 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
868 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
869 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
870 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
871 #[cfg(test)]
872 caller_lookup: None,
873 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
874 rescan: None,
875 connected_clients: ConnectedClients::new(),
876 counters,
877 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
878 daemon_provenance: DaemonProvenanceFacts::default(),
879 #[cfg(test)]
880 control_dispatch_delay: None,
881 #[cfg(test)]
882 provenance_probe_override: None,
883 #[cfg(test)]
884 route_bind_relay_sent_hold: None,
885 #[cfg(test)]
886 hello_registry_to_forwarding_hold: None,
887 }
888 }
889
890 pub fn with_storage_config(
893 mut self,
894 storage_config: Option<crate::daemon_config::StorageConfig>,
895 ) -> Self {
896 self.storage_config = storage_config;
897 self
898 }
899
900 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
903 self.activity = Some(crate::activity::ActivityLedger::open(
904 run_dir.join("activity.jsonl"),
905 incarnation,
906 ));
907 self.supervisor.set_activity(self.activity.clone());
908 self
909 }
910
911 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
914 self.machine_id = machine_id;
915 self
916 }
917
918 pub fn with_admission_facts_config(
922 mut self,
923 carrier_module_id: Option<String>,
924 targets: Option<Vec<String>>,
925 ) -> Self {
926 self.admission_facts_carrier_module_id = carrier_module_id;
927 self.admission_facts_targets = targets;
928 self
929 }
930
931 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
935 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
936 self.scope_authority_owners = owners;
937 self
938 }
939
940 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
942 self.privileged_operator_prompt = mode;
943 self
944 }
945
946 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
949 self.route_bind_relay_timeout = timeout;
950 self
951 }
952
953 pub fn with_route_bind_relay_timeouts(
959 mut self,
960 timeouts: impl IntoIterator<Item = (String, Duration)>,
961 ) -> Self {
962 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
963 self
964 }
965
966 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
972 self.route_bind_relay_timeouts
973 .get(module_id)
974 .copied()
975 .unwrap_or(self.route_bind_relay_timeout)
976 }
977
978 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
986 self.route_bind_breaker_threshold = threshold.max(1);
987 self.route_bind_breaker_cooldown = cooldown;
988 self
989 }
990
991 #[cfg(test)]
992 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
993 self.health_probe_timeout = timeout;
994 self
995 }
996
997 #[cfg(test)]
998 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
999 self.control_dispatch_delay = Some(delay);
1000 self
1001 }
1002
1003 #[cfg(test)]
1007 pub(crate) fn with_route_bind_relay_sent_hold(
1008 mut self,
1009 hold: Arc<tokio::sync::Barrier>,
1010 ) -> Self {
1011 self.route_bind_relay_sent_hold = Some(hold);
1012 self
1013 }
1014
1015 #[cfg(test)]
1020 pub(crate) fn with_hello_registry_to_forwarding_hold(
1021 mut self,
1022 hold: Arc<std::sync::Barrier>,
1023 ) -> Self {
1024 self.hello_registry_to_forwarding_hold = Some(hold);
1025 self
1026 }
1027
1028 #[cfg(test)]
1029 fn hold_between_registry_and_forwarding(&self) {
1030 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1031 hold.wait();
1032 hold.wait();
1033 }
1034 }
1035
1036 pub fn with_process_liveness(
1037 mut self,
1038 process_liveness: Arc<dyn ModuleProcessLiveness>,
1039 ) -> Self {
1040 self.process_liveness = Some(process_liveness);
1041 self
1042 }
1043
1044 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1045 self.supervisor = supervisor;
1046 self.supervisor.set_activity(self.activity.clone());
1047 self
1048 }
1049
1050 pub fn with_daemon_provenance(
1051 mut self,
1052 pid: u32,
1053 started_at_ms: u64,
1054 executable_path: Option<PathBuf>,
1055 build_git_sha: Option<String>,
1056 build_lock_digest: Option<String>,
1057 ) -> Self {
1058 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1059 let process_start_time = process_start_time(pid);
1060 self.daemon_provenance = DaemonProvenanceFacts {
1061 build: DaemonBuildProvenance {
1062 build_git_sha,
1063 build_lock_digest,
1064 },
1065 pid: Some(pid),
1066 started_at_ms: Some(started_at_ms),
1067 start_clock: None,
1068 executable_path,
1069 executable_identity,
1070 process_start_time,
1071 probe: ExecutableIdentityProbe::default(),
1072 };
1073 self
1074 }
1075
1076 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1077 self.daemon_provenance.start_clock = Some(clock);
1078 self
1079 }
1080
1081 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1083 self.scopes
1084 .write()
1085 .unwrap_or_else(|poisoned| poisoned.into_inner())
1086 .set_wall_clock(Arc::new(clock));
1087 self
1088 }
1089
1090 #[cfg(test)]
1091 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1092 self.provenance_probe_override = Some(result);
1093 self
1094 }
1095
1096 pub fn with_capability_config(
1100 self,
1101 modules: impl IntoIterator<Item = (String, bool)>,
1102 reserved_capabilities: BTreeMap<String, String>,
1103 ) -> Self {
1104 self.capability_evaluator
1105 .configure(modules, reserved_capabilities);
1106 self
1107 }
1108
1109 pub fn with_supervisor_rescan(
1110 mut self,
1111 supervisor: Supervisor,
1112 config_path: impl Into<PathBuf>,
1113 configured_port: Option<u16>,
1114 ) -> Self {
1115 self.rescan = Some(SupervisorRescanContext {
1116 supervisor,
1117 config_path: config_path.into(),
1118 configured_port,
1119 storage_config: self.storage_config.clone(),
1120 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1121 admission_facts_targets: self.admission_facts_targets.clone(),
1122 scope_authority_owners: self.scope_authority_owners.clone(),
1123 privileged_operator_prompt: self.privileged_operator_prompt,
1124 });
1125 self
1126 }
1127
1128 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1129 self.connected_clients = connected_clients;
1130 self
1131 }
1132
1133 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1134 Arc::clone(&self.forwarding)
1135 }
1136
1137 pub(crate) fn counters(&self) -> DaemonCounters {
1138 self.counters.clone()
1139 }
1140
1141 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1144 tokio::spawn(async move {
1145 loop {
1146 self.capability_evaluator
1147 .wait_for_change_or_deadline()
1148 .await;
1149 self.refresh_capability_requirements();
1150 }
1151 });
1152 }
1153
1154 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1160 tokio::spawn(async move {
1161 loop {
1162 let has_deadlines = self
1163 .scopes
1164 .read()
1165 .unwrap_or_else(|poisoned| poisoned.into_inner())
1166 .has_deadlines();
1167 if !has_deadlines {
1168 self.scope_deadline_added.notified().await;
1169 continue;
1170 }
1171 tokio::time::sleep(Duration::from_secs(1)).await;
1172 if let Err(error) = self.sweep_expired_scopes() {
1173 warn!(%error, "scope expiry sweep failed");
1174 }
1175 }
1176 });
1177 }
1178
1179 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1180 let mut table = self
1181 .scopes
1182 .write()
1183 .unwrap_or_else(|poisoned| poisoned.into_inner());
1184 let swept = table.sweep_expired();
1185 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1186 drop(table);
1187 self.close_scope_drained_routes(drained);
1188 Ok(())
1189 }
1190
1191 fn publish_scope_changes(
1192 &self,
1193 changes: &[crate::scopes::ScopeTagChange],
1194 expired: &[crate::scopes::ScopeExpired],
1195 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1196 if changes.is_empty() {
1197 return Ok(Vec::new());
1198 }
1199 let drained = self
1200 .forwarding
1201 .publish_scope_changes(changes)
1202 .map_err(RouterError::Forwarding)?;
1203 let mut counts = HashMap::new();
1204 for route in &drained {
1205 *counts
1206 .entry((
1207 route.scope.owner.as_str(),
1208 route.scope.scope_ref.as_str(),
1209 route.scope.tag.scope_epoch,
1210 ))
1211 .or_insert(0usize) += 1;
1212 }
1213 for scope in expired {
1214 let routes_closed = counts
1215 .get(&(
1216 scope.owner.as_str(),
1217 scope.scope_ref.as_str(),
1218 scope.scope_epoch,
1219 ))
1220 .copied()
1221 .unwrap_or(0);
1222 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1223 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1224 routes_closed, "scope expired");
1225 }
1226 Ok(drained)
1227 }
1228
1229 fn runtime_capability_snapshot(
1230 &self,
1231 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1232 let runtime = self
1233 .supervisor
1234 .list()
1235 .into_iter()
1236 .map(|module| {
1237 let status = module.status().map_err(|err| {
1238 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1239 })?;
1240 Ok(RuntimeModule {
1241 module_id: status.module_id,
1242 state: status.state,
1243 enabled: status.enabled,
1244 })
1245 })
1246 .collect::<Result<Vec<_>, RouterError>>()?;
1247 let registrations = self.registry.list_registered_modules().map_err(|err| {
1253 RouterError::backend(
1254 0,
1255 0,
1256 format!("failed to list capability registrations: {err}"),
1257 )
1258 })?;
1259 let registrations = registrations
1260 .into_iter()
1261 .map(|registration| RegisteredModule {
1262 module_id: registration.manifest.module_id,
1263 module_version: registration.manifest.module_version,
1264 capabilities: registration.manifest.capabilities,
1265 })
1266 .collect();
1267 Ok((runtime, registrations))
1268 }
1269
1270 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1277 let cached_registration = RegisteredModule {
1278 module_id: registration.manifest.module_id.clone(),
1279 module_version: registration.manifest.module_version.clone(),
1280 capabilities: registration.manifest.capabilities.clone(),
1281 };
1282 if self.capability_evaluator.record_hello(&cached_registration) {
1283 warn!(
1284 module_id = %cached_registration.module_id,
1285 "capability claims drifted from the cached manifest"
1286 );
1287 }
1288 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1289 self.enforce_capability_denies();
1290 }
1291 self.refresh_capability_requirements();
1292 }
1293
1294 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1298 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1299 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1300 self.supervisor.set_swap_promotion_observer(observer);
1301 }
1302
1303 pub fn refresh_capability_requirements(&self) {
1304 match self.runtime_capability_snapshot() {
1305 Ok((runtime, registrations)) => {
1306 log_requirement_events(
1307 self.capability_evaluator
1308 .evaluate_now(&runtime, ®istrations),
1309 );
1310 }
1311 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1312 }
1313 }
1314
1315 fn enforce_capability_denies(&self) {
1319 let registrations = match self.registry.list_registered_modules() {
1320 Ok(snapshot) => snapshot,
1321 Err(err) => {
1322 warn!(error = %err, "failed to read registrations for capability deny census");
1323 return;
1324 }
1325 };
1326 let manifests = registrations
1327 .into_iter()
1328 .map(|registration| {
1329 (
1330 registration.manifest.module_id.clone(),
1331 registration.manifest,
1332 )
1333 })
1334 .collect::<BTreeMap<_, _>>();
1335 let census = match self.forwarding.route_census(None) {
1336 Ok(census) => census,
1337 Err(err) => {
1338 warn!(error = %err, "failed to read route census for capability deny enforcement");
1339 return;
1340 }
1341 };
1342
1343 for (target_module_id, routes) in census {
1344 let Some(target_manifest) = manifests.get(&target_module_id) else {
1345 continue;
1346 };
1347 let mut closed_routes = Vec::new();
1348 let mut module_goodbyes = Vec::new();
1349 for route in routes {
1350 let Principal::Reserved {
1351 module_id: opening_module_id,
1352 } = &route.principal
1353 else {
1354 continue;
1355 };
1356 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1357 continue;
1358 };
1359 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1360 continue;
1361 };
1362
1363 match self.forwarding.release_client_route(
1364 route.goodbye_target.connection_id,
1365 route.goodbye_target.channel,
1366 route.goodbye_target.epoch,
1367 ) {
1368 Ok(RouteRelease::Removed(module_goodbye)) => {
1369 warn!(
1370 opening_module_id,
1371 target_module_id,
1372 capability,
1373 "force-closing route because an attested capability deny edge now matches"
1374 );
1375 closed_routes.push(route);
1376 module_goodbyes.push(module_goodbye);
1377 }
1378 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1379 Err(err) => warn!(
1380 opening_module_id,
1381 target_module_id,
1382 capability,
1383 error = %err,
1384 "failed to force-close capability-denied route"
1385 ),
1386 }
1387 }
1388
1389 if closed_routes.is_empty() {
1390 continue;
1391 }
1392 send_route_control_pushes(
1393 &self.forwarding,
1394 closed_routes,
1395 ClientControlPush::RouteClosed {
1396 module_id: target_module_id,
1397 channels: Vec::new(),
1398 reason: RouteCloseReason::CapabilityDenied,
1399 drained: false,
1400 abandoned: 0,
1401 excluded_subscriptions: 0,
1402 terminal: Some(false),
1403 },
1404 );
1405 self.emit_route_goodbyes(module_goodbyes);
1406 }
1407 }
1408
1409 fn not_ready_reason(
1415 &self,
1416 registration: &crate::registry::ModuleRegistration,
1417 ) -> Option<NotReadyReason> {
1418 if !registration.ready {
1419 return Some(NotReadyReason {
1420 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1421 capability: None,
1422 });
1423 }
1424 self.first_unprovided_required_capability(registration)
1425 .map(|capability| NotReadyReason {
1426 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1427 capability: Some(capability),
1428 })
1429 }
1430
1431 fn first_unprovided_required_capability(
1449 &self,
1450 registration: &crate::registry::ModuleRegistration,
1451 ) -> Option<String> {
1452 let required = registration
1453 .manifest
1454 .capabilities
1455 .iter()
1456 .flat_map(|declarations| declarations.requires.iter())
1457 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1458 .map(|requirement| requirement.capability.as_str())
1459 .collect::<BTreeSet<_>>();
1460 if required.is_empty() {
1461 return None;
1462 }
1463 let module_id = registration.manifest.module_id.as_str();
1464 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1465 if required
1466 .iter()
1467 .any(|capability| verdict(capability).is_none())
1468 {
1469 self.refresh_capability_requirements();
1470 }
1471 required
1472 .into_iter()
1473 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1474 .map(str::to_string)
1475 }
1476
1477 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1478 self.capability_evaluator
1479 .statuses()
1480 .into_iter()
1481 .map(capability_requirement_status)
1482 .collect()
1483 }
1484
1485 fn deregister_connection(
1492 &self,
1493 connection_id: ConnectionId,
1494 reason: RegistrationEndReason,
1495 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1496 self.registry
1497 .deregister_connection_with_reason(connection_id, reason)
1498 }
1499
1500 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1501 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1502 return None;
1503 }
1504 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1505 parse_client_control_request(&frame.body)
1506 else {
1507 return None;
1508 };
1509 Some(target_module_id(&target).to_string())
1510 }
1511
1512 pub(crate) fn route_open_capacity_refusal(
1513 &self,
1514 ctx: &RouteCtx,
1515 frame: &Frame,
1516 target_module_id: &str,
1517 in_flight: usize,
1518 limit: usize,
1519 ) -> Result<Frame, RouterError> {
1520 self.route_open_admission_refusal_frame(
1521 ctx,
1522 frame,
1523 target_module_id,
1524 "open_admission_full",
1525 (in_flight, limit),
1526 format!(
1527 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1528 ),
1529 )
1530 }
1531
1532 fn route_open_target_capacity_refusal(
1533 &self,
1534 ctx: &RouteCtx,
1535 frame: &Frame,
1536 target_module_id: &str,
1537 in_flight: usize,
1538 ) -> Result<Frame, RouterError> {
1539 self.route_open_admission_refusal_frame(
1540 ctx,
1541 frame,
1542 target_module_id,
1543 "target_binds_full",
1544 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1545 format!(
1546 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1547 ),
1548 )
1549 }
1550
1551 fn route_open_admission_refusal_frame(
1559 &self,
1560 ctx: &RouteCtx,
1561 frame: &Frame,
1562 target_module_id: &str,
1563 reason: &'static str,
1564 (in_flight, limit): (usize, usize),
1565 message: impl Into<String>,
1566 ) -> Result<Frame, RouterError> {
1567 let code = error_codes::TARGET_UNAVAILABLE;
1568 self.counters.increment_route_open_refused(code);
1569 info!(
1570 target: "control",
1571 code,
1572 reason,
1573 module_id = ?target_module_id,
1574 connection_id = ctx.connection_id.get(),
1575 in_flight,
1576 limit,
1577 "route.open refused"
1578 );
1579 control_error_frame(frame, code, message.into())
1580 }
1581
1582 #[cfg(test)]
1588 pub fn handle_control(
1589 &self,
1590 connection_id: ConnectionId,
1591 frame: Frame,
1592 ) -> Result<Vec<Frame>, RouterError> {
1593 match frame.header.ty {
1594 FrameType::Ping => Ok(vec![pong(&frame)?]),
1595 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1596 FrameType::Goodbye => self.handle_goodbye(connection_id),
1597 ty => Ok(vec![control_error_frame(
1598 &frame,
1599 "unsupported_control_frame",
1600 format!("unsupported channel-0 frame {ty:?}"),
1601 )?]),
1602 }
1603 }
1604
1605 pub async fn handle_control_frame(
1606 &self,
1607 ctx: &RouteCtx,
1608 frame: Frame,
1609 ) -> Result<Vec<Frame>, RouterError> {
1610 self.handle_control_frame_timed(ctx, frame, None).await
1611 }
1612
1613 pub(crate) fn record_caller_endpoints(
1614 &self,
1615 connection: ConnectionId,
1616 endpoints: Option<(SocketAddr, SocketAddr)>,
1617 ) {
1618 let mut callers = self
1619 .caller_endpoints
1620 .lock()
1621 .unwrap_or_else(|p| p.into_inner());
1622 callers.remove(&connection);
1623 if let Some(endpoints) = endpoints {
1624 callers.insert(connection, endpoints);
1625 }
1626 }
1627
1628 #[cfg(test)]
1629 pub(crate) fn with_caller_lookup_for_test(
1630 mut self,
1631 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1632 ) -> Self {
1633 self.caller_lookup = Some(Arc::new(lookup));
1634 self
1635 }
1636
1637 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1638 let endpoints = self
1639 .caller_endpoints
1640 .lock()
1641 .unwrap_or_else(|p| p.into_inner())
1642 .get(&connection)
1643 .copied();
1644 #[cfg(test)]
1645 let lookup = self.caller_lookup.clone();
1646 tokio::task::spawn_blocking(move || {
1647 #[cfg(test)]
1648 if let Some(lookup) = lookup {
1649 return lookup(endpoints);
1650 }
1651 endpoints
1652 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1653 .unwrap_or_default()
1654 })
1655 .await
1656 .unwrap_or_default()
1657 }
1658
1659 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1660 frame.header.channel == 0
1661 && frame.header.ty == FrameType::Request
1662 && self
1663 .forwarding
1664 .module_endpoint_for_connection(ctx.connection_id)
1665 .is_ok_and(|endpoint| endpoint.is_none())
1666 && parse_client_control_request(&frame.body)
1667 .ok()
1668 .is_some_and(|request| privileged_operation(&request).is_some())
1669 }
1670
1671 pub(crate) async fn handle_control_frame_timed(
1672 &self,
1673 ctx: &RouteCtx,
1674 frame: Frame,
1675 dispatch_started_at: Option<StdInstant>,
1676 ) -> Result<Vec<Frame>, RouterError> {
1677 match frame.header.ty {
1678 FrameType::Ping => Ok(vec![pong(&frame)?]),
1679 FrameType::Hello => {
1680 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1681 }
1682 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1683 FrameType::Cancel => {
1684 if self
1687 .forwarding
1688 .operator_confirms()
1689 .cancel(ctx.connection_id, frame.header.corr)
1690 || self
1691 .supervisor
1692 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1693 || self.cancel_activity_subscription(ctx, &frame)?
1694 {
1695 Ok(Vec::new())
1696 } else {
1697 Ok(vec![control_error_frame(
1698 &frame,
1699 "unknown_subscription",
1700 "no subscription has this correlation id",
1701 )?])
1702 }
1703 }
1704 FrameType::Request => {
1705 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1708 if op
1709 .as_ref()
1710 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1711 {
1712 return self.handle_activity_record(ctx, frame).await;
1713 }
1714 if op.as_ref().is_some_and(|probe| {
1715 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1716 }) {
1717 return self.handle_activity_read(ctx, frame).await;
1718 }
1719 if op
1720 .as_ref()
1721 .is_some_and(|probe| probe.op == "operator.confirm")
1722 {
1723 return self.handle_operator_confirm(ctx, frame);
1724 }
1725 if self
1726 .forwarding
1727 .module_endpoint_for_connection(ctx.connection_id)
1728 .map_err(RouterError::Forwarding)?
1729 .is_some()
1730 {
1731 if !is_known_module_request_op(&frame.body) {
1732 return Ok(vec![control_error_frame(
1733 &frame,
1734 "unsupported_control_frame",
1735 "module-originated channel-0 REQUEST is not supported",
1736 )?]);
1737 }
1738 let request = match parse_module_control_request_from_module(&frame.body) {
1739 Ok(request) => request,
1740 Err((err, ControlRequestBodyError::UnknownOp)) => {
1741 return Ok(vec![control_error_frame(
1742 &frame,
1743 "unsupported_control_frame",
1744 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1745 )?])
1746 }
1747 Err((err, ControlRequestBodyError::InvalidBody)) => {
1748 return Ok(vec![control_error_frame(
1749 &frame,
1750 "invalid_control_body",
1751 format!("malformed module control body: {err}"),
1752 )?])
1753 }
1754 };
1755 let op = module_control_request_op(&request);
1756 let corr = frame.header.corr;
1757 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1758 let result =
1759 self.handle_module_control_request(ctx.connection_id, frame, request);
1760 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1761 return result;
1762 }
1763
1764 if is_known_module_request_op(&frame.body) {
1765 return Ok(vec![control_error_frame(
1766 &frame,
1767 "not_registered",
1768 "catalog.update requires an active module registration owned by this connection",
1769 )?]);
1770 }
1771
1772 let request = match parse_client_control_request(&frame.body) {
1773 Ok(request) => request,
1774 Err((err, ControlRequestBodyError::UnknownOp)) => {
1775 return Ok(vec![control_error_frame(
1776 &frame,
1777 "unknown_control_op",
1778 format!("unknown client control op: {err}"),
1779 )?])
1780 }
1781 Err((err, ControlRequestBodyError::InvalidBody)) => {
1782 return Ok(vec![control_error_frame(
1783 &frame,
1784 "invalid_control_body",
1785 format!("malformed client control body: {err}"),
1786 )?])
1787 }
1788 };
1789 let op = client_control_request_op(&request);
1790 let corr = frame.header.corr;
1791 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1792 #[cfg(test)]
1793 if let Some(delay) = self.control_dispatch_delay {
1794 tokio::time::sleep(delay).await;
1795 }
1796 let result = self
1797 .handle_client_control_request(ctx, frame, request)
1798 .await;
1799 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1800 result
1801 }
1802 FrameType::Push => {
1803 let Some(endpoint) = self
1804 .forwarding
1805 .module_endpoint_for_connection(ctx.connection_id)
1806 .map_err(RouterError::Forwarding)?
1807 else {
1808 return Ok(vec![control_error_frame(
1809 &frame,
1810 "unsupported_control_frame",
1811 "client-originated channel-0 PUSH is not supported",
1812 )?]);
1813 };
1814 self.handle_status_update(endpoint, frame)
1815 }
1816 FrameType::Response | FrameType::Error
1817 if self
1818 .forwarding
1819 .module_endpoint_for_connection(ctx.connection_id)
1820 .map_err(RouterError::Forwarding)?
1821 .is_some() =>
1822 {
1823 self.handle_module_relay_response(ctx.connection_id, frame)
1824 }
1825 ty => Ok(vec![control_error_frame(
1826 &frame,
1827 "unsupported_control_frame",
1828 format!("unsupported channel-0 frame {ty:?}"),
1829 )?]),
1830 }
1831 }
1832
1833 pub fn cleanup_connection(
1834 &self,
1835 connection_id: ConnectionId,
1836 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1837 self.caller_endpoints
1838 .lock()
1839 .unwrap_or_else(|p| p.into_inner())
1840 .remove(&connection_id);
1841 self.forwarding
1842 .operator_confirms()
1843 .connection_closed(connection_id);
1844 self.cleanup_connection_with_end_reason(
1845 connection_id,
1846 RegistrationEndReason::ConnectionClosed,
1847 )
1848 }
1849
1850 fn cleanup_connection_with_end_reason(
1851 &self,
1852 connection_id: ConnectionId,
1853 requested_reason: RegistrationEndReason,
1854 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1855 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1856 self.registry
1857 .get_module_by_connection(connection_id)?
1858 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
1859 .and_then(|module| module.registration_end_reason().ok().flatten())
1860 .unwrap_or(requested_reason)
1861 } else {
1862 requested_reason
1863 };
1864 let crash_closed = self
1865 .registry
1866 .get_module_by_connection(connection_id)?
1867 .and_then(|registration| {
1868 self.forwarding
1869 .module_endpoint_for_connection(connection_id)
1870 .ok()
1871 .flatten()
1872 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1873 .map(|routes| (registration.manifest.module_id, routes))
1874 });
1875 let crash_closed = crash_closed.map(|(module_id, routes)| {
1876 let terminal = match self.supervisor.get(&module_id) {
1877 None => false,
1878 Some(module) => match module.will_recover_after_connection_loss() {
1879 Ok(will_recover) => !will_recover,
1880 Err(err) => {
1881 warn!(
1882 %module_id,
1883 error = %err,
1884 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1885 );
1886 false
1887 }
1888 },
1889 };
1890 let (reason, drained) = match self
1894 .forwarding
1895 .daemon_drain_result_for_connection(connection_id)
1896 {
1897 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
1898 Ok(None) => (RouteCloseReason::Crash, false),
1899 Err(err) => {
1900 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1901 (RouteCloseReason::Crash, false)
1902 }
1903 };
1904 (module_id, routes, reason, drained, terminal)
1905 });
1906 let registrations = self.deregister_connection(connection_id, end_reason);
1907 let cleanup = if crash_closed.is_some() {
1908 self.forwarding.cleanup_connection_counted(connection_id)
1909 } else {
1910 self.forwarding
1916 .cleanup_connection(connection_id)
1917 .map(|released| crate::forwarding::ConnectionCleanup {
1918 released,
1919 abandoned_relays: 0,
1920 })
1921 };
1922 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
1927 let abandoned = cleanup
1928 .as_ref()
1929 .map_or(0, |cleanup| cleanup.abandoned_relays);
1930 send_route_control_pushes(
1931 &self.forwarding,
1932 routes,
1933 ClientControlPush::RouteClosed {
1934 module_id,
1935 channels: Vec::new(),
1936 reason,
1937 drained,
1938 abandoned,
1939 excluded_subscriptions: 0,
1940 terminal: Some(terminal),
1941 },
1942 );
1943 }
1944 if let Ok(cleanup) = cleanup {
1945 self.emit_route_goodbyes(cleanup.released);
1946 }
1947 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1951 crate::supervise::notify_registration_release();
1952 self.capability_evaluator.wake_deadline_loop();
1953 self.refresh_capability_requirements();
1954 }
1955 self.supervisor.remove_spawn_subscribers(connection_id);
1956 self.activity_streams
1957 .lock()
1958 .unwrap_or_else(|p| p.into_inner())
1959 .retain(|(id, _), task| {
1960 if *id == connection_id {
1961 task.abort();
1962 false
1963 } else {
1964 true
1965 }
1966 });
1967 self.hello_launch_nonces
1970 .lock()
1971 .unwrap_or_else(|poisoned| poisoned.into_inner())
1972 .forget(connection_id);
1973 self.scopes
1974 .write()
1975 .unwrap_or_else(|poisoned| poisoned.into_inner())
1976 .release_connection(connection_id);
1977 registrations
1978 }
1979
1980 pub(crate) fn handle_route_goodbye(
1981 &self,
1982 connection_id: ConnectionId,
1983 route_channel: u16,
1984 route_epoch: u32,
1985 ) -> Result<bool, RouterError> {
1986 debug!(
1987 connection_id = connection_id.get(),
1988 route_channel, route_epoch, "handling route GOODBYE"
1989 );
1990 let RouteRelease::Removed(released_route) = self
1991 .forwarding
1992 .release_client_route(connection_id, route_channel, route_epoch)
1993 .map_err(RouterError::Forwarding)?
1994 else {
1995 return Ok(false);
1996 };
1997 self.emit_route_goodbyes(vec![released_route]);
1998 Ok(true)
1999 }
2000
2001 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2002 for released in released_routes {
2003 let frame = match Frame::build_with_version(
2004 released.negotiated_ver,
2005 FrameType::Goodbye,
2006 control_flags(),
2007 released.channel,
2008 released.epoch,
2009 0,
2010 Vec::new(),
2011 ) {
2012 Ok(frame) => frame,
2013 Err(err) => {
2014 warn!(
2015 route_channel = released.channel,
2016 error = %err,
2017 "failed to build route GOODBYE frame"
2018 );
2019 continue;
2020 }
2021 };
2022 if !released.close_on_delivery_failure() {
2023 crate::forwarding::send_module_route_goodbye(
2024 &self.counters,
2025 &released.sink,
2026 frame,
2027 released.module_id.as_deref(),
2028 "client route released",
2029 );
2030 continue;
2031 }
2032 if let Err(err) = released.sink.try_send(frame) {
2033 warn!(
2034 target_connection_id = released.connection_id.get(),
2035 route_channel = released.channel,
2036 error = %err,
2037 "route GOODBYE was not delivered to client; closing target connection"
2038 );
2039 if self
2040 .forwarding
2041 .escalate_client_delivery_failure(
2042 released.connection_id,
2043 released.channel,
2044 released.epoch,
2045 CloseReason::new(
2046 "route_goodbye_delivery_failed",
2047 format!(
2048 "failed to enqueue route GOODBYE for channel {}: {err}",
2049 released.channel
2050 ),
2051 ),
2052 crate::forwarding::UndeliveredFrame {
2053 module_id: released.module_id.as_deref(),
2054 sink: &released.sink,
2055 },
2056 )
2057 .unwrap_or(false)
2058 {
2059 self.counters.increment_goodbye_relay_client_failed();
2060 }
2061 }
2062 }
2063 }
2064
2065 fn send_abandoned_route_bind_goodbye(
2077 &self,
2078 module_sink: &crate::FrameSink,
2079 negotiated_ver: u8,
2080 module_channel: u16,
2081 module_epoch: u32,
2082 ) {
2083 let frame = match Frame::build_with_version(
2084 negotiated_ver,
2085 FrameType::Goodbye,
2086 control_flags(),
2087 module_channel,
2088 module_epoch,
2089 0,
2090 Vec::new(),
2091 ) {
2092 Ok(frame) => frame,
2093 Err(err) => {
2094 warn!(
2095 route_channel = module_channel,
2096 error = %err,
2097 "failed to build GOODBYE for abandoned route.bind"
2098 );
2099 return;
2100 }
2101 };
2102 crate::forwarding::send_module_route_goodbye(
2103 &self.counters,
2104 module_sink,
2105 frame,
2106 None,
2107 "abandoned route.bind",
2108 );
2109 }
2110
2111 fn handle_hello(
2112 &self,
2113 connection_id: ConnectionId,
2114 sink: Option<crate::FrameSink>,
2115 frame: Frame,
2116 ) -> Result<Vec<Frame>, RouterError> {
2117 debug!(
2118 connection_id = connection_id.get(),
2119 corr = frame.header.corr,
2120 "handling HELLO"
2121 );
2122 if self
2126 .registry
2127 .get_module_by_connection(connection_id)
2128 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2129 .is_some()
2130 {
2131 return Ok(vec![control_error_frame(
2132 &frame,
2133 "invalid_hello",
2134 "connection is already registered as a module",
2135 )?]);
2136 }
2137 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2138 Ok(value) => value,
2139 Err(err) => {
2140 return Ok(vec![control_error_frame(
2141 &frame,
2142 "invalid_hello",
2143 format!("malformed HELLO body: {err}"),
2144 )?])
2145 }
2146 };
2147 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2148 return Ok(vec![control_error_frame(
2149 &frame,
2150 "invalid_capability_grammar",
2151 err.to_string(),
2152 )?]);
2153 }
2154 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2155 return Ok(vec![control_error_frame(
2156 &frame,
2157 "invalid_manifest",
2158 err.to_string(),
2159 )?]);
2160 }
2161 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2162 return Ok(vec![control_error_frame(
2163 &frame,
2164 "invalid_event_declaration",
2165 err.to_string(),
2166 )?]);
2167 }
2168 if let Some(provenance) = hello_value
2169 .get("manifest")
2170 .and_then(|manifest| manifest.get("provenance"))
2171 {
2172 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2173 return Ok(vec![control_error_frame(
2174 &frame,
2175 "invalid_manifest",
2176 format!("malformed manifest provenance: {err}"),
2177 )?]);
2178 }
2179 }
2180 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2181 Ok(hello) => hello,
2182 Err(err) => {
2183 return Ok(vec![control_error_frame(
2184 &frame,
2185 "invalid_hello",
2186 format!("malformed HELLO body: {err}"),
2187 )?])
2188 }
2189 };
2190
2191 if hello.protocol_ver != hello.manifest.protocol_ver {
2192 return Ok(vec![control_error_frame(
2193 &frame,
2194 "invalid_manifest",
2195 format!(
2196 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2197 hello.protocol_ver, hello.manifest.protocol_ver
2198 ),
2199 )?]);
2200 }
2201
2202 if hello.manifest.module_id.trim().is_empty() {
2203 return Ok(vec![control_error_frame(
2204 &frame,
2205 "invalid_manifest",
2206 "manifest module_id must not be empty",
2207 )?]);
2208 }
2209
2210 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2211 Ok(negotiated_ver) => negotiated_ver,
2212 Err(message) => {
2213 return Ok(vec![control_error_frame(
2214 &frame,
2215 "version_unsupported",
2216 message,
2217 )?])
2218 }
2219 };
2220
2221 let swap_admission = self
2229 .supervisor
2230 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2231 if swap_admission == SwapHelloAdmission::Refused {
2232 warn!(
2233 module_id = %hello.manifest.module_id,
2234 connection_id = connection_id.get(),
2235 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2236 );
2237 return Ok(vec![control_error_frame(
2238 &frame,
2239 "swap_token_invalid",
2240 format!(
2241 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2242 hello.manifest.module_id
2243 ),
2244 )?]);
2245 }
2246 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2247
2248 if let Some(rejection) = (!swap_candidate)
2256 .then(|| {
2257 self.supervisor.reserved_hello_rejection(
2258 &hello.manifest.module_id,
2259 hello.launch_nonce.as_deref(),
2260 )
2261 })
2262 .flatten()
2263 {
2264 let message = match rejection {
2265 ReservedHelloRejection::Exact { module_id } => format!(
2266 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2267 ),
2268 ReservedHelloRejection::Prefix {
2269 prefix,
2270 owner_module_id,
2271 } => format!(
2272 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2273 hello.manifest.module_id
2274 ),
2275 };
2276 return Ok(vec![control_error_frame(
2277 &frame,
2278 "reserved_module",
2279 message,
2280 )?]);
2281 }
2282
2283 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2284 &hello.manifest.module_id,
2285 hello.manifest.capabilities.as_ref(),
2286 );
2287 if let Some(refusal) = reserved_capability_refusals.first() {
2288 let capability = refusal.capability.clone();
2289 let bound_module = refusal.claimants[0].clone();
2290 log_duplicate_claim_events(reserved_capability_refusals);
2291 return Ok(vec![control_error_frame(
2292 &frame,
2293 "reserved_capability",
2294 format!(
2295 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2296 capability, bound_module, hello.manifest.module_id
2297 ),
2298 )?]);
2299 }
2300
2301 if self
2304 .forwarding
2305 .connection_has_client_routes(connection_id)
2306 .map_err(RouterError::Forwarding)?
2307 {
2308 return Ok(vec![control_error_frame(
2309 &frame,
2310 "invalid_hello",
2311 "connection has open client routes and cannot also register as a module",
2312 )?]);
2313 }
2314
2315 self.hello_launch_nonces
2320 .lock()
2321 .unwrap_or_else(|poisoned| poisoned.into_inner())
2322 .record(connection_id, hello.launch_nonce.as_deref());
2323 let control_ops = effective_module_control_ops(hello.control_ops);
2324 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2327 if swap_candidate {
2328 return self.register_swap_candidate(
2329 connection_id,
2330 sink,
2331 &frame,
2332 hello.manifest,
2333 negotiated_ver,
2334 control_ops,
2335 hello_ack,
2336 );
2337 }
2338 let registration = match self.registry.register_unpublished_with_control_ops(
2345 hello.manifest,
2346 negotiated_ver,
2347 connection_id,
2348 control_ops,
2349 ) {
2350 Ok(registration) => registration,
2351 Err(RegistryError::DuplicateModuleId { module_id }) => {
2352 return Ok(vec![control_error_frame(
2353 &frame,
2354 "duplicate_module_id",
2355 format!(
2356 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2357 ),
2358 )?])
2359 }
2360 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2361 return Ok(vec![control_error_frame(
2362 &frame,
2363 "invalid_module_id",
2364 err.to_string(),
2365 )?])
2366 }
2367 Err(err) => {
2368 return Ok(vec![control_error_frame(
2369 &frame,
2370 "registry_error",
2371 err.to_string(),
2372 )?])
2373 }
2374 };
2375 #[cfg(test)]
2376 self.hold_between_registry_and_forwarding();
2377
2378 let reply = if let Some(sink) = sink {
2379 let concurrency = manifest_concurrency(®istration.manifest);
2395 if let Err(err) = self.forwarding.register_module_connection_acked(
2396 connection_id,
2397 registration.manifest.module_id.clone(),
2398 negotiated_ver,
2399 concurrency,
2400 sink,
2401 hello_ack,
2402 ) {
2403 if matches!(
2408 self.deregister_connection(
2409 connection_id,
2410 RegistrationEndReason::RegistrationFailed,
2411 ),
2412 Ok(r) if !r.is_empty()
2413 ) {
2414 crate::supervise::notify_registration_release();
2415 }
2416 return Ok(vec![control_error_frame(
2417 &frame,
2418 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2419 "invalid_hello"
2420 } else {
2421 forwarding_error_code(&err)
2422 },
2423 err.to_string(),
2424 )?]);
2425 }
2426 Vec::new()
2427 } else {
2428 vec![hello_ack]
2431 };
2432
2433 if !self
2438 .registry
2439 .publish_registration(connection_id)
2440 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2441 {
2442 debug!(
2443 module_id = %registration.manifest.module_id,
2444 connection_id = connection_id.get(),
2445 "registration was removed before it could be published"
2446 );
2447 }
2448
2449 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2458 info!(
2459 module_id = %registration.manifest.module_id,
2460 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2461 );
2462 }
2463
2464 self.apply_registration_capabilities(®istration);
2465
2466 self.supervisor
2467 .record_activity_registration(®istration.manifest.module_id);
2468
2469 info!(
2470 module_id = %registration.manifest.module_id,
2471 module_version = %registration.manifest.module_version,
2472 negotiated_ver,
2473 routable_provider = manifest_provides_routable_role(®istration.manifest),
2474 connection_id = connection_id.get(),
2475 "module registered"
2476 );
2477
2478 Ok(reply)
2479 }
2480
2481 #[allow(clippy::too_many_arguments)]
2498 fn register_swap_candidate(
2499 &self,
2500 connection_id: ConnectionId,
2501 sink: Option<crate::FrameSink>,
2502 frame: &Frame,
2503 manifest: ModuleManifest,
2504 negotiated_ver: u8,
2505 control_ops: Vec<String>,
2506 hello_ack: Frame,
2507 ) -> Result<Vec<Frame>, RouterError> {
2508 let module_id = manifest.module_id.clone();
2509 let registration = match self.registry.register_unpublished_candidate_with_control_ops(
2510 manifest,
2511 negotiated_ver,
2512 connection_id,
2513 control_ops,
2514 ) {
2515 Ok(registration) => registration,
2516 Err(RegistryError::DuplicateModuleId { module_id }) => {
2517 return Ok(vec![control_error_frame(
2518 frame,
2519 "duplicate_module_id",
2520 format!(
2521 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2522 ),
2523 )?])
2524 }
2525 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2526 return Ok(vec![control_error_frame(
2527 frame,
2528 "invalid_module_id",
2529 err.to_string(),
2530 )?])
2531 }
2532 Err(err) => {
2533 return Ok(vec![control_error_frame(
2534 frame,
2535 "registry_error",
2536 err.to_string(),
2537 )?])
2538 }
2539 };
2540 #[cfg(test)]
2541 self.hold_between_registry_and_forwarding();
2542 let reply = if let Some(sink) = sink {
2543 let concurrency = manifest_concurrency(®istration.manifest);
2547 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2548 connection_id,
2549 module_id.clone(),
2550 negotiated_ver,
2551 concurrency,
2552 sink,
2553 hello_ack,
2554 ) {
2555 if matches!(
2556 self.deregister_connection(
2557 connection_id,
2558 RegistrationEndReason::RegistrationFailed,
2559 ),
2560 Ok(r) if !r.is_empty()
2561 ) {
2562 crate::supervise::notify_registration_release();
2563 }
2564 return Ok(vec![control_error_frame(
2565 frame,
2566 forwarding_error_code(&err),
2567 err.to_string(),
2568 )?]);
2569 }
2570 Vec::new()
2571 } else {
2572 vec![hello_ack]
2573 };
2574 if !self
2575 .registry
2576 .publish_registration(connection_id)
2577 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2578 {
2579 debug!(
2580 module_id = %module_id,
2581 connection_id = connection_id.get(),
2582 "swap candidate registration was removed before it could be published"
2583 );
2584 }
2585 self.supervisor.mark_swap_candidate_admitted(&module_id);
2586 info!(
2587 module_id = %module_id,
2588 module_version = %registration.manifest.module_version,
2589 negotiated_ver,
2590 ready = registration.ready,
2591 connection_id = connection_id.get(),
2592 "swap candidate registered; not routable until cutover"
2593 );
2594 Ok(reply)
2595 }
2596
2597 fn build_hello_ack(
2598 &self,
2599 frame: &Frame,
2600 negotiated_ver: u8,
2601 module_id: &str,
2602 ) -> Result<Frame, RouterError> {
2603 let ack = ModuleHelloAckBody {
2604 negotiated_ver,
2605 subc_ops: module_subc_ops(),
2606 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2607 storage: self
2608 .storage_config
2609 .as_ref()
2610 .map(|cfg| cfg.descriptor_for(module_id)),
2611 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2612 };
2613 let body = serde_json::to_vec(&ack).map_err(|err| {
2614 RouterError::backend(
2615 0,
2616 frame.header.corr,
2617 format!("failed to encode HELLO_ACK: {err}"),
2618 )
2619 })?;
2620
2621 Frame::build_with_version(
2622 negotiated_ver,
2623 FrameType::HelloAck,
2624 control_flags(),
2625 0,
2626 0,
2627 frame.header.corr,
2628 body,
2629 )
2630 .map_err(RouterError::FrameBuild)
2631 }
2632
2633 async fn handle_client_control_request(
2634 &self,
2635 ctx: &RouteCtx,
2636 frame: Frame,
2637 request: ClientControlRequest,
2638 ) -> Result<Vec<Frame>, RouterError> {
2639 if let Some(operation) = privileged_operation(&request) {
2640 let caller = self.resolve_caller(ctx.connection_id).await;
2641 let op = client_control_request_op(&request);
2642 let text = prompt_text(operation, &caller);
2643 match self.privileged_operator_prompt {
2644 PrivilegedOperatorPrompt::Log => {
2645 info!(target: "subc_daemon::privileged_prompt", operation = op,
2646 target = ?operation, caller = %caller.description(), summary = %text,
2647 "privileged request would prompt the operator");
2648 }
2649 PrivilegedOperatorPrompt::Enforce => {
2650 let key =
2651 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2652 let outcome = self
2653 .forwarding
2654 .operator_confirms()
2655 .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2656 .await;
2657 if outcome.code != "confirmed" {
2658 return Ok(vec![outcome.frame(&frame)]);
2659 }
2660 }
2661 }
2662 }
2663 match request {
2664 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2665 ClientControlRequest::CatalogList { module_id } => {
2666 self.handle_catalog_list(frame, module_id)
2667 }
2668 ClientControlRequest::RouteOpen {
2669 target,
2670 identity,
2671 consumer_identity,
2672 consumer_capabilities,
2673 role_versions,
2674 admission_facts,
2675 scope,
2676 } => {
2677 self.handle_route_open(
2678 ctx,
2679 frame,
2680 RouteOpenRequest {
2681 target,
2682 identity,
2683 consumer_identity,
2684 consumer_capabilities,
2685 role_versions,
2686 admission_facts,
2687 scope,
2688 },
2689 )
2690 .await
2691 }
2692 ClientControlRequest::RoutePoll {
2693 route_channel,
2694 route_epoch,
2695 kind,
2696 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2697 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2698 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2699 self.handle_supervisor_spawn_snapshot(frame)
2700 }
2701 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2702 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2703 }
2704 ClientControlRequest::SupervisorRestart {
2705 module_id,
2706 drain_timeout_ms,
2707 } => {
2708 self.log_supervisor_request_received(
2709 ctx,
2710 frame.header.corr,
2711 ops::SUPERVISOR_RESTART,
2712 Some(&module_id),
2713 None,
2714 )?;
2715 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2716 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2717 .await
2718 }
2719 ClientControlRequest::SupervisorSwap {
2720 module_id,
2721 ready_timeout_ms,
2722 } => {
2723 self.log_supervisor_request_received(
2724 ctx,
2725 frame.header.corr,
2726 ops::SUPERVISOR_SWAP,
2727 Some(&module_id),
2728 None,
2729 )?;
2730 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2731 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2732 .await
2733 }
2734 ClientControlRequest::SupervisorReload { module_id } => {
2735 self.log_supervisor_request_received(
2736 ctx,
2737 frame.header.corr,
2738 ops::SUPERVISOR_RELOAD,
2739 Some(&module_id),
2740 None,
2741 )?;
2742 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2743 self.handle_supervisor_reload(frame, module_id, requester)
2744 .await
2745 }
2746 ClientControlRequest::SupervisorRescan { preview } => {
2747 if !preview {
2748 self.log_supervisor_request_received(
2749 ctx,
2750 frame.header.corr,
2751 ops::SUPERVISOR_RESCAN,
2752 None,
2753 None,
2754 )?;
2755 }
2756 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2757 self.handle_supervisor_rescan(frame, preview, requester)
2758 .await
2759 }
2760 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2761 self.log_supervisor_request_received(
2762 ctx,
2763 frame.header.corr,
2764 ops::SUPERVISOR_RELEASE_RESERVED,
2765 Some(&module_id),
2766 None,
2767 )?;
2768 self.handle_supervisor_release_reserved(frame, module_id)
2769 .await
2770 }
2771 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2772 self.log_supervisor_request_received(
2773 ctx,
2774 frame.header.corr,
2775 ops::SUPERVISOR_SET_ENABLED,
2776 Some(&module_id),
2777 Some(enabled),
2778 )?;
2779 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2780 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2781 .await
2782 }
2783 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2784 self.handle_supervisor_health_probe(frame, module_id).await
2785 }
2786 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2787 ClientControlRequest::SupervisorRoutes { module_id } => {
2788 self.handle_supervisor_routes(frame, module_id)
2789 }
2790 ClientControlRequest::SupervisorProvenance { module_id } => {
2791 self.handle_supervisor_provenance(frame, module_id).await
2792 }
2793 ClientControlRequest::SupervisorStderrTail {
2794 module_id,
2795 max_lines,
2796 max_bytes,
2797 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2798 ClientControlRequest::SupervisorTerminals { module_id } => {
2799 self.handle_supervisor_terminals(frame, module_id).await
2800 }
2801 }
2802 }
2803
2804 fn log_supervisor_request_received(
2805 &self,
2806 ctx: &RouteCtx,
2807 corr: u64,
2808 op: &'static str,
2809 module_id: Option<&str>,
2810 enabled: Option<bool>,
2811 ) -> Result<(), RouterError> {
2812 let caller = self.supervisor_requester(ctx, corr)?;
2813 let caller = principal_label(&caller);
2814 info!(
2817 target: "control",
2818 op,
2819 module_id,
2820 enabled,
2821 connection_id = ctx.connection_id.get(),
2822 caller = %caller,
2823 "supervisor request received"
2824 );
2825 Ok(())
2826 }
2827
2828 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
2829 Ok(self
2830 .registry
2831 .get_module_by_connection(ctx.connection_id)
2832 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2833 .map(|registration| Principal::Reserved {
2834 module_id: registration.manifest.module_id,
2835 })
2836 .unwrap_or(Principal::Direct))
2837 }
2838
2839 fn handle_module_control_request(
2840 &self,
2841 connection_id: ConnectionId,
2842 frame: Frame,
2843 request: ModuleControlRequestFromModule,
2844 ) -> Result<Vec<Frame>, RouterError> {
2845 match request {
2846 ModuleControlRequestFromModule::CatalogUpdate {
2847 provides,
2848 capabilities,
2849 ready,
2850 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2851 ModuleControlRequestFromModule::LiveRoots {} => {
2852 let registered = self
2853 .registry
2854 .get_module_by_connection(connection_id)
2855 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2856 let Some(registration) = registered else {
2857 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2858 };
2859 let response = self
2860 .forwarding
2861 .live_roots(®istration.manifest.module_id)
2862 .map_err(RouterError::Forwarding)?;
2863 Ok(vec![control_response_body_frame(
2864 &frame,
2865 &response,
2866 "ModuleControlResponseToModule::LiveRoots",
2867 )?])
2868 }
2869 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2870 self.handle_scope_sync(connection_id, frame, generation, scopes)
2871 }
2872 ModuleControlRequestFromModule::ScopeApply {
2873 generation,
2874 upsert,
2875 end,
2876 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
2877 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2878 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2879 }
2880 }
2881 }
2882
2883 fn handle_operator_confirm(
2884 &self,
2885 ctx: &RouteCtx,
2886 frame: Frame,
2887 ) -> Result<Vec<Frame>, RouterError> {
2888 use crate::operator_confirm::{audit, Outcome};
2889 let registration = self
2890 .registry
2891 .get_module_by_connection(ctx.connection_id)
2892 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2893 let Some(registration) = registration else {
2894 let outcome = Outcome::refusal("not_registered");
2895 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2896 return Ok(vec![outcome.frame(&frame)]);
2897 };
2898 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
2899 Ok(request) => request,
2900 Err(_) => {
2901 let outcome = Outcome::refusal("invalid_control_body");
2902 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2903 return Ok(vec![outcome.frame(&frame)]);
2904 }
2905 };
2906 let module_id = registration.manifest.module_id;
2907 let nonce = self
2911 .hello_launch_nonces
2912 .lock()
2913 .unwrap_or_else(|p| p.into_inner())
2914 .nonce(ctx.connection_id)
2915 .map(str::to_owned);
2916 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
2917 self.supervisor
2918 .spawned_consumer_authorized(&module_id, nonce)
2919 });
2920 let confirms = self.forwarding.operator_confirms();
2921 self.forwarding
2922 .with_operator_route(
2923 ctx.connection_id,
2924 request.route_channel,
2925 request.route_epoch,
2926 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
2927 )
2928 .map_err(RouterError::Forwarding)
2929 }
2930
2931 fn handle_scope_sync(
2935 &self,
2936 connection_id: ConnectionId,
2937 frame: Frame,
2938 generation: u64,
2939 scopes: Vec<ScopeRecord>,
2940 ) -> Result<Vec<Frame>, RouterError> {
2941 self.handle_scope_change(connection_id, frame, generation, scopes, None)
2942 }
2943
2944 fn handle_scope_change(
2945 &self,
2946 connection_id: ConnectionId,
2947 frame: Frame,
2948 generation: u64,
2949 scopes: Vec<ScopeRecord>,
2950 end: Option<Vec<ScopeEnd>>,
2951 ) -> Result<Vec<Frame>, RouterError> {
2952 let is_apply = end.is_some();
2953 let Some(registration) = self
2954 .registry
2955 .get_module_by_connection(connection_id)
2956 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2957 else {
2958 return Ok(vec![control_error_frame(
2959 &frame,
2960 "not_registered",
2961 "scope.sync and scope.apply require an active module registration owned by this connection",
2962 )?]);
2963 };
2964 let owner = registration.manifest.module_id;
2965 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2966 let is_current_launch = |connection: ConnectionId| {
2967 self.hello_launch_nonces
2968 .lock()
2969 .unwrap_or_else(|poisoned| poisoned.into_inner())
2970 .presented(connection, current_nonce.as_deref())
2971 };
2972 let mut table = self
2977 .scopes
2978 .write()
2979 .unwrap_or_else(|poisoned| poisoned.into_inner());
2980 let outcome = match end {
2981 Some(end) => table.apply(
2982 &owner,
2983 connection_id,
2984 is_current_launch,
2985 generation,
2986 scopes,
2987 end,
2988 ),
2989 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
2990 };
2991 let drained = match &outcome {
2992 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
2993 Err(_) => Vec::new(),
2994 };
2995 let has_deadlines = outcome.is_ok() && table.has_deadlines();
2996 drop(table);
2997 if has_deadlines {
2998 self.scope_deadline_added.notify_one();
2999 }
3000 match outcome {
3001 Ok(applied) => {
3002 let counts = ScopeOutcomeCounts::of(&applied.results);
3003 info!(
3004 owner = %owner,
3005 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3006 generation,
3007 records = applied.results.len(),
3008 created = counts.created,
3009 replaced = counts.replaced,
3010 updated = counts.updated,
3011 unchanged = counts.unchanged,
3012 refused = counts.refused,
3013 ended = applied.ended.len(),
3014 tag_changes = applied.tag_changes.len(),
3015 routes_closed = drained.len(),
3016 "scope change accepted"
3017 );
3018 for refused in applied
3024 .results
3025 .iter()
3026 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3027 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3028 {
3029 warn!(
3030 owner = %owner,
3031 generation,
3032 scope_ref = %refused.scope_ref,
3033 scope_epoch = refused.scope_epoch,
3034 code = refused.code.as_deref().unwrap_or(""),
3035 "scope record refused"
3036 );
3037 }
3038 self.close_scope_drained_routes(drained);
3039 let response = if is_apply {
3040 ModuleControlResponseToModule::ScopeApply {
3041 generation,
3042 results: applied.results,
3043 end_results: applied.end_results,
3044 ended: applied.ended,
3045 }
3046 } else {
3047 ModuleControlResponseToModule::ScopeSync {
3048 generation,
3049 results: applied.results,
3050 ended: applied.ended,
3051 }
3052 };
3053 Ok(vec![control_response_body_frame(
3054 &frame,
3055 &response,
3056 "scope change response",
3057 )?])
3058 }
3059 Err(refusal) => {
3060 info!(
3061 owner = %owner,
3062 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3063 generation,
3064 code = refusal.code,
3065 "scope change refused"
3066 );
3067 Ok(vec![control_error_frame(
3068 &frame,
3069 refusal.code,
3070 refusal.message,
3071 )?])
3072 }
3073 }
3074 }
3075
3076 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3082 if drained.is_empty() {
3083 return;
3084 }
3085 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3086 BTreeMap::new();
3087 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3088 for route in drained {
3089 warn!(
3090 module_id = %route.module_id,
3091 reason = ?route.reason,
3092 client_connection_id = route.client.connection_id.get(),
3093 route_channel = route.client.channel,
3094 "closing route because its scope changed"
3095 );
3096 pushes
3097 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3098 .or_insert_with(|| (route.reason, Vec::new()))
3099 .1
3100 .push(EndpointRoute {
3101 goodbye_target: route.client.clone(),
3102 principal: Principal::Unverified,
3103 bound_at: Instant::now(),
3104 draining: false,
3105 drain_reason: None,
3106 });
3107 goodbyes.push(route.module);
3108 goodbyes.push(route.client);
3109 }
3110 for ((module_id, _), (reason, routes)) in pushes {
3111 send_route_control_pushes(
3112 &self.forwarding,
3113 routes,
3114 ClientControlPush::RouteClosed {
3115 module_id,
3116 channels: Vec::new(),
3117 reason,
3118 drained: false,
3119 abandoned: 0,
3120 excluded_subscriptions: 0,
3121 terminal: Some(false),
3122 },
3123 );
3124 }
3125 self.emit_route_goodbyes(goodbyes);
3126 }
3127
3128 fn handle_scope_describe(
3131 &self,
3132 connection_id: ConnectionId,
3133 frame: Frame,
3134 owner: Principal,
3135 scope_ref: String,
3136 ) -> Result<Vec<Frame>, RouterError> {
3137 let registered = self
3138 .registry
3139 .get_module_by_connection(connection_id)
3140 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3141 if registered.is_none() {
3142 return Ok(vec![control_error_frame(
3143 &frame,
3144 "not_registered",
3145 "scope.describe requires an active module registration owned by this connection",
3146 )?]);
3147 }
3148 let description = self
3149 .scopes
3150 .read()
3151 .unwrap_or_else(|poisoned| poisoned.into_inner())
3152 .describe(&owner, &scope_ref);
3153 let owner_configured = match &owner {
3154 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3159 _ => false,
3160 };
3161 let response = ModuleControlResponseToModule::ScopeDescribe {
3162 status: description.status,
3163 scope_epoch: description.scope_epoch,
3164 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3165 owner_synced: description.owner_synced,
3166 owner_configured,
3167 scope: description.stamp,
3168 };
3169 Ok(vec![control_response_body_frame(
3170 &frame,
3171 &response,
3172 "ModuleControlResponseToModule::ScopeDescribe",
3173 )?])
3174 }
3175
3176 fn handle_catalog_update(
3177 &self,
3178 connection_id: ConnectionId,
3179 frame: Frame,
3180 provides: Vec<ProviderRole>,
3181 capabilities: Option<CapabilityDeclarations>,
3182 ready: Option<bool>,
3183 ) -> Result<Vec<Frame>, RouterError> {
3184 self.refresh_capability_requirements();
3185 let Some(registration) = self
3186 .registry
3187 .get_module_by_connection(connection_id)
3188 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3189 else {
3190 return Ok(vec![control_error_frame(
3191 &frame,
3192 "not_registered",
3193 "catalog.update requires an active module registration owned by this connection",
3194 )?]);
3195 };
3196
3197 if let Some(message) =
3198 catalog_update_frozen_field_message(®istration.manifest, &provides)
3199 {
3200 return Ok(vec![control_error_frame(
3201 &frame,
3202 "catalog_update_frozen_field",
3203 message,
3204 )?]);
3205 }
3206
3207 let mut candidate = registration.manifest.clone();
3208 candidate.provides = provides.clone();
3209 candidate.capabilities = capabilities
3210 .clone()
3211 .or_else(|| registration.manifest.capabilities.clone());
3212 if let Err(err) = candidate.validate_capability_grammar() {
3213 return Ok(vec![control_error_frame(
3214 &frame,
3215 "invalid_capability_grammar",
3216 err.to_string(),
3217 )?]);
3218 }
3219
3220 let mut conflicts = self
3223 .capability_evaluator
3224 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3225 if let Some(conflict) = conflicts.first() {
3226 let message = format!(
3227 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3228 conflict.capability, conflict.claimants[0], candidate.module_id
3229 );
3230 for conflict in &mut conflicts {
3231 conflict.source = DuplicateClaimSource::CatalogUpdate;
3232 }
3233 log_duplicate_claim_events(conflicts);
3234 return Ok(vec![control_error_frame(
3235 &frame,
3236 "reserved_capability",
3237 message,
3238 )?]);
3239 }
3240
3241 let updated = self
3242 .registry
3243 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3244 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3245 if updated.is_none() {
3246 return Ok(vec![control_error_frame(
3247 &frame,
3248 "not_registered",
3249 "catalog.update requires an active module registration owned by this connection",
3250 )?]);
3251 }
3252 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3253 log_duplicate_claim_events(
3254 self.capability_evaluator
3255 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3256 );
3257 }
3258 if capability_census_trigger(
3259 registration.manifest.capabilities.as_ref(),
3260 updated
3261 .as_ref()
3262 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3263 ) {
3264 self.enforce_capability_denies();
3265 }
3266 self.refresh_capability_requirements();
3267
3268 let response = ModuleControlResponseToModule::CatalogUpdate {};
3269 control_response_body_frame(
3270 &frame,
3271 &response,
3272 "ModuleControlResponseToModule::CatalogUpdate",
3273 )
3274 .map(|frame| vec![frame])
3275 }
3276
3277 pub(crate) fn with_activity_ledger_config(
3278 mut self,
3279 run_dir: &Path,
3280 incarnation: String,
3281 config: crate::activity::ActivityConfig,
3282 ) -> Self {
3283 self.activity = Some(crate::activity::ActivityLedger::open_config(
3284 run_dir.join("activity.jsonl"),
3285 incarnation,
3286 config,
3287 ));
3288 self.supervisor.set_activity(self.activity.clone());
3289 self
3290 }
3291
3292 pub(crate) async fn configure_activity(
3293 &self,
3294 config: crate::activity::ActivityConfig,
3295 ) -> Result<(), &'static str> {
3296 match &self.activity {
3297 Some(ledger) => ledger.configure(config).await,
3298 None => Ok(()),
3299 }
3300 }
3301
3302 fn cancel_activity_subscription(
3303 &self,
3304 ctx: &RouteCtx,
3305 frame: &Frame,
3306 ) -> Result<bool, RouterError> {
3307 let task = self
3308 .activity_streams
3309 .lock()
3310 .unwrap_or_else(|p| p.into_inner())
3311 .remove(&(ctx.connection_id, frame.header.corr));
3312 let Some(task) = task else {
3313 return Ok(false);
3314 };
3315 task.abort();
3316 let end = Frame::build_with_version(
3317 response_version(frame),
3318 FrameType::StreamEnd,
3319 control_flags(),
3320 0,
3321 0,
3322 frame.header.corr,
3323 Vec::new(),
3324 )
3325 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3326 let sink = ctx.egress.clone();
3327 tokio::spawn(async move {
3328 let _ = sink.send(end).await;
3329 });
3330 Ok(true)
3331 }
3332
3333 async fn handle_activity_read(
3334 &self,
3335 ctx: &RouteCtx,
3336 frame: Frame,
3337 ) -> Result<Vec<Frame>, RouterError> {
3338 use subc_control::{
3339 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3340 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3341 };
3342 #[derive(Deserialize)]
3343 struct Identity {
3344 #[serde(default)]
3345 consumer_identity: Option<ConsumerIdentity>,
3346 }
3347 let principal = match self
3351 .forwarding
3352 .module_id_for_connection(ctx.connection_id)
3353 .map_err(RouterError::Forwarding)?
3354 {
3355 Some(module_id) => Ok(Principal::Reserved { module_id }),
3356 None => {
3357 let identity: Identity = match serde_json::from_slice(&frame.body) {
3358 Ok(identity) => identity,
3359 Err(_) => {
3360 return Ok(vec![control_error_frame(
3361 &frame,
3362 ACTIVITY_MALFORMED,
3363 "malformed activity read",
3364 )?])
3365 }
3366 };
3367 self.route_open_principal(&frame, identity.consumer_identity)?
3368 }
3369 };
3370 let admitted = matches!(principal, Ok(Principal::Direct))
3371 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3372 if !admitted {
3373 return Ok(vec![control_error_frame(
3374 &frame,
3375 ACTIVITY_READ_FORBIDDEN,
3376 "activity reads require an operator or trusted mirror principal",
3377 )?]);
3378 }
3379 let Some(ledger) = &self.activity else {
3380 return Ok(vec![control_error_frame(
3381 &frame,
3382 ACTIVITY_UNAVAILABLE,
3383 "activity ledger unavailable",
3384 )?]);
3385 };
3386 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3387 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3388 {
3389 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3390 Ok(request) => ledger.query(request).await,
3391 Err(_) => Err(ACTIVITY_MALFORMED),
3392 };
3393 return match result {
3394 Ok(reply) => Ok(vec![control_response_body_frame(
3395 &frame,
3396 &reply,
3397 ops::ACTIVITY_QUERY,
3398 )?]),
3399 Err(code) => Ok(vec![control_error_frame(
3400 &frame,
3401 code,
3402 "activity query refused",
3403 )?]),
3404 };
3405 }
3406 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3407 Ok(request) => request,
3408 Err(_) => {
3409 return Ok(vec![control_error_frame(
3410 &frame,
3411 ACTIVITY_MALFORMED,
3412 "malformed activity subscription",
3413 )?])
3414 }
3415 };
3416 let mut subscription = match ledger.subscribe(request.since).await {
3417 Ok(subscription) => subscription,
3418 Err(code) => {
3419 return Ok(vec![control_error_frame(
3420 &frame,
3421 code,
3422 "activity subscription refused",
3423 )?])
3424 }
3425 };
3426 let key = (ctx.connection_id, frame.header.corr);
3427 let sink = ctx.egress.clone();
3428 let mut streams = self
3429 .activity_streams
3430 .lock()
3431 .unwrap_or_else(|p| p.into_inner());
3432 streams.retain(|_, task| !task.is_finished());
3433 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3434 return Ok(vec![control_error_frame(
3435 &frame,
3436 "duplicate_subscription",
3437 "activity subscription correlation id already in use",
3438 )?]);
3439 }
3440 let task = tokio::spawn(async move {
3441 loop {
3442 let entry = if let Some(entry) = subscription.replay.pop_front() {
3443 Some(entry)
3444 } else {
3445 subscription.receiver.recv().await
3446 };
3447 let Some(entry) = entry else {
3448 break;
3449 };
3450 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3451 return;
3452 };
3453 let Ok(next) = Frame::build_with_version(
3454 response_version(&frame),
3455 FrameType::StreamData,
3456 control_flags(),
3457 0,
3458 0,
3459 frame.header.corr,
3460 body,
3461 ) else {
3462 return;
3463 };
3464 if sink.send(next).await.is_err() {
3465 return;
3466 }
3467 }
3468 if let Some(code) = subscription.terminal_code() {
3471 if let Ok(end) =
3472 control_error_frame(&frame, code, "activity subscriber fell behind")
3473 {
3474 let _ = sink.send(end).await;
3475 }
3476 }
3477 });
3478 streams.insert(key, task.abort_handle());
3479 Ok(Vec::new())
3480 }
3481
3482 async fn handle_activity_record(
3483 &self,
3484 ctx: &RouteCtx,
3485 frame: Frame,
3486 ) -> Result<Vec<Frame>, RouterError> {
3487 use subc_control::{
3488 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3489 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3490 };
3491 let result = async {
3492 let module = self
3493 .forwarding
3494 .module_id_for_connection(ctx.connection_id)
3495 .ok()
3496 .flatten()
3497 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3498 let request: ActivityRecordRequest =
3499 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3500 crate::activity::validate(&module, &request)?;
3501 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3502 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3503 let at_ms = std::time::SystemTime::now()
3504 .duration_since(std::time::UNIX_EPOCH)
3505 .unwrap_or_default()
3506 .as_millis() as u64;
3507 ledger.record(module, request, actor, at_ms).await
3508 }
3509 .await;
3510 match result {
3511 Ok(entry) => Ok(vec![control_response_body_frame(
3512 &frame,
3513 &ActivityRecordResponse { seq: entry.seq },
3514 "activity.record",
3515 )?]),
3516 Err(code) => Ok(vec![control_error_frame(
3517 &frame,
3518 code,
3519 "activity recording refused",
3520 )?]),
3521 }
3522 }
3523
3524 fn activity_actor(
3525 &self,
3526 connection: ConnectionId,
3527 module: &str,
3528 on: Option<&subc_control::ActivityOn>,
3529 ) -> Result<subc_control::ActivityActor, &'static str> {
3530 use crate::forwarding::{DataRoute, DataRouteState};
3531 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3532 let principal = |id: &str| Principal::Reserved {
3533 module_id: id.to_owned(),
3534 };
3535 let mut actor = ActivityActor {
3536 principal: principal(module),
3537 scope_owner: None,
3538 agent_id: None,
3539 run_id: None,
3540 flow_id: None,
3541 project_root: None,
3542 };
3543 let scope = match on {
3544 None => return Ok(actor),
3545 Some(ActivityOn::Scope(scope)) => {
3546 let selector = subc_protocol::scope::ScopeSelector::new(
3547 principal(module),
3548 scope.scope_ref.clone(),
3549 Some(scope.epoch),
3550 );
3551 self.scopes
3552 .read()
3553 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3554 .attribute(&principal(module), module, &selector, true)
3555 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3556 }
3557 Some(ActivityOn::Route(route)) => {
3558 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3559 .forwarding
3560 .lookup_data_route(connection, route.channel, route.epoch)
3561 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3562 else {
3563 return Err(ACTIVITY_ACTOR_UNKNOWN);
3564 };
3565 if binding.module_id != module {
3566 return Err(ACTIVITY_ACTOR_UNKNOWN);
3567 }
3568 actor.principal = binding.principal.clone();
3569 actor.project_root = binding
3570 .project_root
3571 .as_ref()
3572 .map(|root| root.as_path().to_path_buf());
3573 let Some(bound) = &binding.scope else {
3574 return Ok(actor);
3575 };
3576 let owner = principal(&bound.owner);
3577 let selector = subc_protocol::scope::ScopeSelector::new(
3578 owner.clone(),
3579 bound.scope_ref.clone(),
3580 Some(bound.tag.scope_epoch),
3581 );
3582 let admitted = self
3583 .scopes
3584 .read()
3585 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3586 .attribute(&owner, module, &selector, true)
3587 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3588 if admitted.tag != bound.tag {
3589 return Err(ACTIVITY_ACTOR_UNKNOWN);
3590 }
3591 admitted
3592 }
3593 };
3594 actor.scope_owner = Some(scope.stamp.owner);
3595 actor.agent_id = scope.stamp.attributes.agent_id;
3596 actor.run_id = scope.stamp.attributes.run_id;
3597 actor.flow_id = scope.stamp.attributes.flow_id;
3598 Ok(actor)
3599 }
3600
3601 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3602 self.refresh_capability_requirements();
3603 let mut counters = self.counters.snapshot();
3609 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3610 obj.insert(
3611 "activity_unparseable_lines".into(),
3612 ledger.read_errors().into(),
3613 );
3614 obj.insert(
3615 "activity_lifecycle_append_failures".into(),
3616 ledger.lifecycle_failures().into(),
3617 );
3618 obj.insert(
3619 "activity_started_digest_skips".into(),
3620 ledger.lifecycle_digest_skips().into(),
3621 );
3622 }
3623 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3624 self.forwarding.client_route_concentration(),
3625 counters.as_object_mut(),
3626 ) {
3627 obj.insert(
3628 "client_connections_with_routes".into(),
3629 connections_with_routes.into(),
3630 );
3631 obj.insert("max_routes_on_one_connection".into(), max.into());
3632 }
3633 if let (Some(open_breakers), Some(obj)) = (
3639 self.route_bind_breakers.open_snapshot(),
3640 counters.as_object_mut(),
3641 ) {
3642 obj.insert("route_bind_breakers_open".into(), open_breakers);
3643 }
3644 let response = ClientControlResponse::ServerDescribe {
3645 protocol_ver: PROTOCOL_VERSION,
3646 subc_ops: subc_ops(),
3647 capabilities: self.subc_capabilities.as_ref().to_vec(),
3648 connected_clients: self.connected_clients.count(),
3649 counters: Some(counters),
3650 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3651 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3652 capability_requirements: self.capability_requirement_statuses(),
3653 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3654 };
3655 Ok(vec![control_response_body_frame(
3656 &frame,
3657 &response,
3658 "ClientControlResponse::ServerDescribe",
3659 )?])
3660 }
3661
3662 fn handle_catalog_list(
3663 &self,
3664 frame: Frame,
3665 module_id: Option<String>,
3666 ) -> Result<Vec<Frame>, RouterError> {
3667 let (generation, modules) = self.registry.list_modules().map_err(|err| {
3668 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3669 })?;
3670 let entries = modules
3671 .into_iter()
3672 .filter(|registration| {
3673 module_id
3674 .as_deref()
3675 .map(|wanted| registration.manifest.module_id == wanted)
3676 .unwrap_or(true)
3677 })
3678 .map(|registration| {
3679 let not_ready = self.not_ready_reason(®istration);
3680 let roles = registration.manifest.provides;
3681 CatalogEntry::new(
3682 registration.manifest.module_id,
3683 roles,
3684 registration.control_ops,
3685 )
3686 .with_ready(not_ready.is_none())
3687 .with_not_ready(not_ready)
3688 .with_module_version(Some(registration.manifest.module_version))
3689 .with_capabilities(registration.manifest.capabilities)
3690 .with_self_signals(registration.manifest.self_signals)
3691 .with_events(registration.manifest.events)
3692 })
3693 .collect();
3694 let response = ClientControlResponse::CatalogList {
3695 generation,
3696 modules: entries,
3697 subc_ops: subc_ops(),
3698 };
3699 Ok(vec![control_response_body_frame(
3700 &frame,
3701 &response,
3702 "ClientControlResponse::CatalogList",
3703 )?])
3704 }
3705
3706 fn route_open_principal(
3707 &self,
3708 frame: &Frame,
3709 consumer_identity: Option<ConsumerIdentity>,
3710 ) -> Result<Result<Principal, Frame>, RouterError> {
3711 let Some(consumer_identity) = consumer_identity else {
3712 return Ok(Ok(Principal::Direct));
3713 };
3714
3715 if self.supervisor.spawned_consumer_authorized(
3716 &consumer_identity.module_id,
3717 &consumer_identity.launch_nonce,
3718 ) {
3719 return Ok(Ok(Principal::Reserved {
3720 module_id: consumer_identity.module_id,
3721 }));
3722 }
3723
3724 Ok(Err(control_error_frame(
3725 frame,
3726 "bad_consumer_identity",
3727 format!(
3728 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3729 consumer_identity.module_id
3730 ),
3731 )?))
3732 }
3733
3734 fn route_open_refusal_frame(
3741 &self,
3742 ctx: &RouteCtx,
3743 frame: &Frame,
3744 module_id: &str,
3745 reason: &'static str,
3746 code: &'static str,
3747 message: impl Into<String>,
3748 ) -> Result<Frame, RouterError> {
3749 self.observe_route_open_refusal(ctx, module_id, reason, code);
3750 control_error_frame(frame, code, message.into())
3751 }
3752
3753 fn route_open_breaker_refusal_frame(
3771 &self,
3772 ctx: &RouteCtx,
3773 frame: &Frame,
3774 module_id: &str,
3775 consecutive_timeouts: u32,
3776 retry_in: Duration,
3777 probe_in_flight: bool,
3778 ) -> Result<Frame, RouterError> {
3779 self.counters
3780 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
3781 debug!(
3782 target: "control",
3783 code = "module_timeout",
3784 module_id = ?module_id,
3785 connection_id = ctx.connection_id.get(),
3786 consecutive_timeouts,
3787 retry_in_ms = retry_in.as_millis() as u64,
3788 probe_in_flight,
3789 "route.open refused by open bind-relay breaker"
3790 );
3791 let detail = if probe_in_flight {
3798 "one test route is already being tried; retry once it settles".to_string()
3799 } else {
3800 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
3801 };
3802 control_error_frame(
3803 frame,
3804 "module_timeout",
3805 format!(
3806 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
3807 timed out in a row); {detail}; its established routes are unaffected"
3808 ),
3809 )
3810 }
3811
3812 fn observe_route_open_refusal(
3825 &self,
3826 ctx: &RouteCtx,
3827 module_id: &str,
3828 reason: &'static str,
3829 code: &'static str,
3830 ) {
3831 self.counters.increment_route_open_refused(code);
3832 info!(
3833 target: "control",
3834 code,
3835 reason,
3836 module_id = ?module_id,
3837 connection_id = ctx.connection_id.get(),
3838 "route.open refused"
3839 );
3840 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
3841 self.route_outages.record_not_serving(module_id, reason);
3842 }
3843 }
3844
3845 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
3890 self.route_outages.record_accepted(module_id);
3891 self.counters.increment_route_open_accepted(principal);
3892 info!(
3893 target: "control",
3894 principal,
3895 module_id,
3896 connection_id = ctx.connection_id.get(),
3897 "route.open accepted"
3898 );
3899 }
3900
3901 fn supervised_absent_route_open_refusal_frame(
3902 &self,
3903 ctx: &RouteCtx,
3904 frame: &Frame,
3905 module_id: &str,
3906 code: &'static str,
3907 status: &crate::supervise::ModuleStatus,
3908 ) -> Result<Frame, RouterError> {
3909 self.counters.increment_route_open_refused(code);
3910 info!(
3911 target: "control",
3912 code,
3913 reason = "supervised_not_registered",
3914 module_id = ?module_id,
3915 connection_id = ctx.connection_id.get(),
3916 state = %status.state,
3917 enabled = status.enabled,
3918 live = status.live,
3919 "route.open refused"
3920 );
3921 self.route_outages
3925 .record_not_serving(module_id, "supervised_not_registered");
3926 let message = if let Some(pid) = status.waiting_for_previous_pid {
3927 format!("waiting: previous instance (pid {pid}) is still running but hidden; it cannot be identified")
3928 } else {
3929 format!(
3930 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
3931 status.state, status.enabled, status.live
3932 )
3933 };
3934 control_error_frame(frame, code, message)
3935 }
3936
3937 async fn handle_route_open(
3938 &self,
3939 ctx: &RouteCtx,
3940 frame: Frame,
3941 request: RouteOpenRequest,
3942 ) -> Result<Vec<Frame>, RouterError> {
3943 let RouteOpenRequest {
3944 target,
3945 mut identity,
3946 consumer_identity,
3947 consumer_capabilities,
3948 role_versions,
3949 admission_facts,
3950 scope,
3951 } = request;
3952 let target_module_id = target_module_id(&target).to_string();
3953 if self
3954 .registry
3955 .get_module_by_connection(ctx.connection_id)
3956 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3957 .is_some()
3958 {
3959 return Ok(vec![control_error_frame(
3960 &frame,
3961 "invalid_request",
3962 "module connections cannot open client routes",
3963 )?]);
3964 }
3965 debug!(
3966 connection_id = ctx.connection_id.get(),
3967 corr = frame.header.corr,
3968 module_id = %target_module_id,
3969 "handling route.open"
3970 );
3971
3972 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
3978 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
3979 self.observe_route_open_refusal(
3980 ctx,
3981 &target_module_id,
3982 "invalid_role_versions",
3983 error_codes::INVALID_REQUEST,
3984 );
3985 return Ok(vec![control_error_body_frame(
3986 &frame,
3987 ErrorBody {
3988 code: error_codes::INVALID_REQUEST.to_string(),
3989 message: error.to_string(),
3990 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
3991 },
3992 )?]);
3993 }
3994
3995 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
3996 if status.waiting_for_previous_pid.is_some() {
3997 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3998 ctx,
3999 &frame,
4000 &target_module_id,
4001 "module_warming",
4002 &status,
4003 )?]);
4004 }
4005 }
4006
4007 let Some(registration) = self
4026 .registry
4027 .get_module(&target_module_id)
4028 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4029 else {
4030 if let Some((status, warming)) =
4031 self.supervisor_status(&target_module_id, frame.header.corr)?
4032 {
4033 if status.protocol == ModuleProtocol::None {
4040 return Ok(vec![self.route_open_refusal_frame(
4041 ctx,
4042 &frame,
4043 &target_module_id,
4044 "protocol_none",
4045 error_codes::MODULE_NO_PROTOCOL,
4046 format!(
4047 "module_id '{target_module_id}' is declared protocol: none; \
4048 it speaks no subc wire and serves no routes"
4049 ),
4050 )?]);
4051 }
4052 let code = if warming {
4053 "module_warming"
4054 } else {
4055 "target_unavailable"
4056 };
4057 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4058 ctx,
4059 &frame,
4060 &target_module_id,
4061 code,
4062 &status,
4063 )?]);
4064 }
4065 if let Some(removed_ago_ms) =
4066 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4067 {
4068 return Ok(vec![self.route_open_refusal_frame(
4069 ctx,
4070 &frame,
4071 &target_module_id,
4072 "removed",
4073 error_codes::MODULE_REMOVED,
4074 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4075 )?]);
4076 }
4077 return Ok(vec![self.route_open_refusal_frame(
4078 ctx,
4079 &frame,
4080 &target_module_id,
4081 "not_registered",
4082 error_codes::UNKNOWN_MODULE,
4083 format!("module_id '{target_module_id}' is not registered"),
4084 )?]);
4085 };
4086
4087 if !registration.ready {
4091 self.counters
4092 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4093 info!(
4094 target: "control",
4095 code = error_codes::MODULE_WARMING,
4096 module_id = ?target_module_id,
4097 connection_id = ctx.connection_id.get(),
4098 reason = "declared_not_ready",
4099 "route.open refused"
4100 );
4101 self.route_outages
4104 .record_not_serving(&target_module_id, "declared_not_ready");
4105 return Ok(vec![control_error_body_frame(
4106 &frame,
4107 ErrorBody {
4108 code: error_codes::MODULE_WARMING.to_string(),
4109 message: format!(
4110 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4111 ),
4112 detail: Some(serde_json::json!({
4113 "reason": "declared_not_ready"
4114 })),
4115 },
4116 )?]);
4117 }
4118
4119 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4138 self.counters
4139 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4140 info!(
4141 target: "control",
4142 code = error_codes::MODULE_WARMING,
4143 module_id = ?target_module_id,
4144 connection_id = ctx.connection_id.get(),
4145 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4146 capability = %capability,
4147 "route.open refused"
4148 );
4149 return Ok(vec![control_error_body_frame(
4150 &frame,
4151 ErrorBody {
4152 code: error_codes::MODULE_WARMING.to_string(),
4153 message: format!(
4154 "module_id '{target_module_id}' requires capability '{capability}', \
4155 which no registered module provides; retry"
4156 ),
4157 detail: Some(serde_json::json!({
4158 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4159 "capability": capability,
4160 })),
4161 },
4162 )?]);
4163 }
4164
4165 if !target_has_required_role(&target, ®istration.manifest.provides) {
4166 return Ok(vec![self.route_open_refusal_frame(
4167 ctx,
4168 &frame,
4169 &target_module_id,
4170 "role_not_provided",
4171 "target_unavailable",
4172 format!("module_id '{target_module_id}' does not provide the requested target"),
4173 )?]);
4174 }
4175
4176 if registration.state != ChannelState::Active {
4177 return Ok(vec![self.route_open_refusal_frame(
4178 ctx,
4179 &frame,
4180 &target_module_id,
4181 "registration_not_active",
4182 "target_unavailable",
4183 format!("module_id '{target_module_id}' is not active"),
4184 )?]);
4185 }
4186
4187 if self
4188 .forwarding
4189 .module_is_draining(&target_module_id)
4190 .map_err(RouterError::Forwarding)?
4191 {
4192 return Ok(vec![self.route_open_refusal_frame(
4193 ctx,
4194 &frame,
4195 &target_module_id,
4196 "reloading",
4197 "module_reloading",
4198 format!("module_id '{target_module_id}' is reloading"),
4199 )?]);
4200 }
4201
4202 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4203 process_liveness.process_live(&target_module_id) == Some(false)
4204 }) {
4205 if process_liveness.process_replacing(&target_module_id) {
4212 return Ok(vec![self.route_open_refusal_frame(
4213 ctx,
4214 &frame,
4215 &target_module_id,
4216 "reloading",
4217 "module_reloading",
4218 format!("module_id '{target_module_id}' is reloading"),
4219 )?]);
4220 }
4221 return Ok(vec![self.route_open_refusal_frame(
4222 ctx,
4223 &frame,
4224 &target_module_id,
4225 "supervisor_not_live",
4226 "target_unavailable",
4227 format!("module_id '{target_module_id}' is not live"),
4228 )?]);
4229 }
4230
4231 if !self
4232 .forwarding
4233 .has_live_module_connection(&target_module_id)
4234 .map_err(RouterError::Forwarding)?
4235 {
4236 return Ok(vec![self.route_open_refusal_frame(
4237 ctx,
4238 &frame,
4239 &target_module_id,
4240 "no_forwarding_connection",
4241 "target_unavailable",
4242 format!("module_id '{target_module_id}' has no live forwarding connection"),
4243 )?]);
4244 }
4245
4246 if let Some(error) =
4247 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4248 {
4249 self.observe_route_open_refusal(
4250 ctx,
4251 &target_module_id,
4252 "op_not_allowed",
4253 "op_not_allowed",
4254 );
4255 return Ok(vec![error]);
4256 }
4257
4258 let principal = match self.route_open_principal(&frame, consumer_identity)? {
4259 Ok(principal) => principal,
4260 Err(error) => {
4261 self.observe_route_open_refusal(
4262 ctx,
4263 &target_module_id,
4264 "bad_consumer_identity",
4265 "bad_consumer_identity",
4266 );
4267 return Ok(vec![error]);
4268 }
4269 };
4270
4271 if let Principal::Reserved {
4275 module_id: opening_module_id,
4276 } = &principal
4277 {
4278 if let Some(opening_registration) = self
4279 .registry
4280 .get_module(opening_module_id)
4281 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4282 {
4283 if let Some(capability) =
4284 denied_capability(&opening_registration.manifest, ®istration.manifest)
4285 {
4286 warn!(
4287 opening_module_id,
4288 target_module_id,
4289 capability,
4290 "refusing route.open because an attested capability deny edge matches"
4291 );
4292 return Ok(vec![self.route_open_refusal_frame(
4293 ctx,
4294 &frame,
4295 &target_module_id,
4296 "capability_deny_edge",
4297 "capability_forbidden",
4298 format!(
4299 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4300 ),
4301 )?]);
4302 }
4303 }
4304 }
4305
4306 if admission_facts.is_some() {
4307 let carrier_matches = matches!(
4308 &principal,
4309 Principal::Reserved { module_id }
4310 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4311 );
4312 if !carrier_matches {
4313 return Ok(vec![self.route_open_refusal_frame(
4314 ctx,
4315 &frame,
4316 &target_module_id,
4317 "admission_facts_carrier_not_permitted",
4318 "admission_facts_not_permitted",
4319 "admission facts may only be carried by the configured reserved module",
4320 )?]);
4321 }
4322
4323 let target_allowed = self
4324 .admission_facts_targets
4325 .as_ref()
4326 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4327 if !target_allowed {
4328 return Ok(vec![self.route_open_refusal_frame(
4329 ctx,
4330 &frame,
4331 &target_module_id,
4332 "admission_facts_target_not_listed",
4333 "admission_facts_target_not_allowed",
4334 format!(
4335 "admission facts are not permitted for target module_id '{target_module_id}'"
4336 ),
4337 )?]);
4338 }
4339
4340 }
4344
4345 let (bound_scope, mut scope_stamp) = match &scope {
4351 None => (None, None),
4352 Some(selector) => {
4353 let owner_configured = match &selector.owner {
4354 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4360 _ => false,
4361 };
4362 let admitted = self
4363 .scopes
4364 .read()
4365 .unwrap_or_else(|poisoned| poisoned.into_inner())
4366 .admit(&principal, &target_module_id, selector, owner_configured)
4367 .and_then(|admission| {
4368 crate::scopes::check_target_flow_support(
4369 &admission.stamp,
4370 &target_module_id,
4371 registration.manifest.capabilities.as_ref(),
4372 )?;
4373 crate::scopes::check_target_agent_run_support(
4374 &admission.stamp,
4375 &target_module_id,
4376 registration.manifest.capabilities.as_ref(),
4377 )?;
4378 Ok(admission)
4379 });
4380 match admitted {
4381 Ok(admission) => (
4382 Some(BoundScope {
4383 owner: admission.owner,
4384 scope_ref: admission.stamp.scope_ref.clone(),
4385 tag: admission.tag,
4386 }),
4387 Some(admission.stamp),
4388 ),
4389 Err(refusal) => {
4390 return Ok(vec![self.route_open_refusal_frame(
4391 ctx,
4392 &frame,
4393 &target_module_id,
4394 refusal.code,
4395 refusal.code,
4396 refusal.message,
4397 )?]);
4398 }
4399 }
4400 }
4401 };
4402
4403 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4425 Ok(project_root) => project_root,
4426 Err(err) => {
4427 return Ok(vec![control_error_frame(
4428 &frame,
4429 "invalid_project_root",
4430 err.to_string(),
4431 )?])
4432 }
4433 };
4434 identity.project_root = project_root.as_path().to_path_buf();
4435
4436 let _concurrency_guard = match self
4447 .route_bind_concurrency
4448 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4449 {
4450 Ok(guard) => guard,
4451 Err(in_flight) => {
4452 return Ok(vec![self.route_open_target_capacity_refusal(
4453 ctx,
4454 &frame,
4455 &target_module_id,
4456 in_flight,
4457 )?]);
4458 }
4459 };
4460
4461 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4464 RouteBindAdmission::Admitted { guard, probe } => {
4465 if probe {
4466 info!(
4467 module_id = %target_module_id,
4468 connection_id = ctx.connection_id.get(),
4469 "route.bind breaker half-open: admitting one probe"
4470 );
4471 }
4472 guard
4473 }
4474 RouteBindAdmission::Refused {
4475 consecutive_timeouts,
4476 retry_in,
4477 probe_in_flight,
4478 } => {
4479 return Ok(vec![self.route_open_breaker_refusal_frame(
4480 ctx,
4481 &frame,
4482 &target_module_id,
4483 consecutive_timeouts,
4484 retry_in,
4485 probe_in_flight,
4486 )?]);
4487 }
4488 };
4489
4490 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4495 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4496 let pending = match self
4497 .forwarding
4498 .begin_route_bind_relay_for(
4499 ctx.connection_id,
4500 ctx.egress.clone(),
4501 response_version(&frame),
4502 frame.header.corr,
4503 &target_module_id,
4504 principal.clone(),
4505 bound_scope,
4506 Some(project_root),
4507 relay_deadline,
4508 )
4509 .await
4510 {
4511 Ok(pending) => pending,
4512 Err(err) => {
4513 return Ok(vec![self.route_open_refusal_frame(
4514 ctx,
4515 &frame,
4516 &target_module_id,
4517 "relay_reservation_failed",
4518 forwarding_error_code(&err),
4519 err.to_string(),
4520 )?])
4521 }
4522 };
4523 let crate::forwarding::PendingRouteBindRelay {
4524 endpoint,
4525 module_sink,
4526 negotiated_ver,
4527 client_channel,
4528 client_epoch,
4529 module_channel,
4530 module_epoch,
4531 corr: relay_corr,
4532 receiver,
4533 } = pending;
4534 let mut reservation =
4535 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4536
4537 let relay_registration = self
4542 .registry
4543 .get_module_by_connection(endpoint.connection_id)
4544 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4545 let relay_capabilities = relay_registration
4546 .as_ref()
4547 .and_then(|registration| registration.manifest.capabilities.as_ref());
4548 if let Some(selector) = &scope {
4549 if let Err(refusal) = self
4552 .scopes
4553 .read()
4554 .unwrap_or_else(|poisoned| poisoned.into_inner())
4555 .check_bind_proof(selector)
4556 {
4557 reservation.release_and_disarm();
4558 return Ok(vec![self.route_open_refusal_frame(
4559 ctx,
4560 &frame,
4561 &target_module_id,
4562 refusal.code,
4563 refusal.code,
4564 refusal.message,
4565 )?]);
4566 }
4567 }
4568 if let Some(stamp) = scope_stamp
4569 .as_ref()
4570 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4571 {
4572 if let Err(refusal) = crate::scopes::check_target_flow_support(
4573 stamp,
4574 &target_module_id,
4575 relay_capabilities,
4576 )
4577 .and_then(|()| {
4578 crate::scopes::check_target_agent_run_support(
4579 stamp,
4580 &target_module_id,
4581 relay_capabilities,
4582 )
4583 }) {
4584 reservation.release_and_disarm();
4585 return Ok(vec![self.route_open_refusal_frame(
4586 ctx,
4587 &frame,
4588 &target_module_id,
4589 refusal.code,
4590 refusal.code,
4591 refusal.message,
4592 )?]);
4593 }
4594 }
4595
4596 let call_mode = if relay_capabilities
4599 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4600 {
4601 Some(derive_call_mode(scope_stamp.as_ref()))
4602 } else {
4603 if let Some(stamp) = scope_stamp.as_mut() {
4604 stamp.expires_at_ms = None;
4605 }
4606 None
4607 };
4608
4609 debug!(
4610 connection_id = ctx.connection_id.get(),
4611 client_channel,
4612 client_epoch,
4613 module_channel,
4614 module_epoch,
4615 "reserved route handle pair"
4616 );
4617 let principal_label = principal_label(&principal);
4620 let relay = ModuleControlRequest::RouteBind {
4621 route_channel: module_channel,
4622 epoch: module_epoch,
4623 target,
4624 identity,
4625 principal: Some(principal),
4626 consumer_capabilities,
4627 role_versions,
4628 admission_facts,
4629 scope: scope_stamp,
4630 call_mode,
4631 };
4632 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4633 RouterError::backend(
4634 0,
4635 frame.header.corr,
4636 format!("failed to encode route.bind request: {err}"),
4637 )
4638 })?;
4639 let relay_frame = Frame::build_with_version(
4640 negotiated_ver,
4641 FrameType::Request,
4642 control_flags(),
4643 0,
4644 0,
4645 relay_corr,
4646 relay_body,
4647 )
4648 .map_err(RouterError::FrameBuild)?;
4649
4650 if let Err(err) = module_sink.send(relay_frame).await {
4651 reservation.release_and_disarm();
4652 return Ok(vec![self.route_open_refusal_frame(
4653 ctx,
4654 &frame,
4655 &target_module_id,
4656 "relay_send_failed",
4657 "target_unavailable",
4658 err.to_string(),
4659 )?]);
4660 }
4661
4662 #[cfg(test)]
4663 if let Some(hold) = &self.route_bind_relay_sent_hold {
4664 hold.wait().await;
4668 hold.wait().await;
4669 }
4670
4671 if self
4676 .forwarding
4677 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4678 .map_err(RouterError::Forwarding)?
4679 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4680 {
4681 self.send_abandoned_route_bind_goodbye(
4682 &module_sink,
4683 negotiated_ver,
4684 module_channel,
4685 module_epoch,
4686 );
4687 }
4688
4689 match timeout_at(relay_deadline, receiver).await {
4690 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4691 reservation.disarm();
4692 if breaker.record_accepted() {
4693 info!(
4694 module_id = %target_module_id,
4695 "route.bind breaker closed: the probe was accepted"
4696 );
4697 }
4698 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
4699 Ok(Vec::new())
4700 }
4701 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4702 reservation.release_and_disarm();
4703 breaker.record_inconclusive();
4707 let scope_code = match body.code.as_str() {
4711 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4712 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4713 _ => None,
4714 };
4715 if let Some(code) = scope_code {
4716 self.observe_route_open_refusal(
4717 ctx,
4718 &target_module_id,
4719 "scope_changed_before_commit",
4720 code,
4721 );
4722 return Ok(vec![control_error_body_frame(&frame, body)?]);
4723 }
4724 self.counters
4725 .increment_route_open_refused("module_rejected");
4726 info!(
4727 target: "control",
4728 code = "module_rejected",
4729 module_code = ?body.code,
4730 module_id = ?target_module_id,
4731 connection_id = ctx.connection_id.get(),
4732 "route.open refused"
4733 );
4734 Ok(vec![control_error_body_frame(&frame, body)?])
4735 }
4736 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
4737 reservation.release_and_disarm();
4738 breaker.record_inconclusive();
4739 tracing::warn!(
4746 module_id = %target_module_id,
4747 "route.bind relay abandoned: {message}"
4748 );
4749 Ok(vec![self.route_open_refusal_frame(
4750 ctx,
4751 &frame,
4752 &target_module_id,
4753 "relay_abandoned",
4754 "target_unavailable",
4755 message,
4756 )?])
4757 }
4758 Ok(Err(_)) => {
4759 reservation.release_and_disarm();
4760 breaker.record_inconclusive();
4761 Ok(vec![self.route_open_refusal_frame(
4762 ctx,
4763 &frame,
4764 &target_module_id,
4765 "relay_waiter_canceled",
4766 "target_unavailable",
4767 "route.bind relay waiter was canceled before the module responded",
4768 )?])
4769 }
4770 Err(_) => {
4771 reservation.release_and_disarm();
4772 if let Some(opened) = breaker.record_timeout(
4776 self.route_bind_breaker_threshold,
4777 self.route_bind_breaker_cooldown,
4778 ) {
4779 warn!(
4780 module_id = %target_module_id,
4781 consecutive_timeouts = opened.consecutive_timeouts,
4782 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
4783 reopened_after_probe = opened.reopened_after_probe,
4784 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
4785 );
4786 }
4787 tracing::warn!(
4795 module_id = %target_module_id,
4796 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
4797 "route.bind relay timed out: module did not ack within budget"
4798 );
4799 Ok(vec![self.route_open_refusal_frame(
4800 ctx,
4801 &frame,
4802 &target_module_id,
4803 "relay_timed_out",
4804 "module_timeout",
4805 format!(
4806 "module_id '{target_module_id}' did not answer route.bind within {:?}",
4807 route_bind_relay_timeout
4808 ),
4809 )?])
4810 }
4811 }
4812 }
4813
4814 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4815 let response = ClientControlResponse::SupervisorSpawnSnapshot {
4816 snapshot: self.supervisor.spawn_snapshot(),
4817 };
4818 Ok(vec![control_response_body_frame(
4819 &frame,
4820 &response,
4821 "ClientControlResponse::SupervisorSpawnSnapshot",
4822 )?])
4823 }
4824
4825 fn handle_supervisor_spawn_subscribe(
4826 &self,
4827 ctx: &RouteCtx,
4828 frame: Frame,
4829 since: Option<SpawnCursor>,
4830 ) -> Result<Vec<Frame>, RouterError> {
4831 match self.supervisor.subscribe_spawns(
4832 ctx.connection_id,
4833 frame.header.corr,
4834 response_version(&frame),
4835 since,
4836 ctx.egress.clone(),
4837 ) {
4838 Ok(()) => Ok(Vec::new()),
4839 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
4840 Ok(vec![control_error_body_frame(
4841 &frame,
4842 ErrorBody {
4843 code: "spawn_cursor_incarnation_mismatch".to_string(),
4844 message: "spawn cursor belongs to a different daemon incarnation"
4845 .to_string(),
4846 detail: Some(serde_json::json!({
4847 "current_daemon_incarnation": current
4848 })),
4849 },
4850 )?])
4851 }
4852 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
4853 &frame,
4854 ErrorBody {
4855 code: "spawn_cursor_too_old".to_string(),
4856 message: "spawn cursor predates the retained event ring".to_string(),
4857 detail: Some(serde_json::json!({
4858 "oldest_retained_cursor": oldest
4859 })),
4860 },
4861 )?]),
4862 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
4863 0,
4864 frame.header.corr,
4865 format!("failed to open supervisor spawn subscription: {error}"),
4866 )),
4867 }
4868 }
4869
4870 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4871 let generation = self
4872 .registry
4873 .generation()
4874 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4875 let mut modules = Vec::new();
4876 for module in self.supervisor.list() {
4877 let status = module.status_for_control("list").map_err(|err| {
4878 RouterError::backend(
4879 0,
4880 frame.header.corr,
4881 format!("failed to read supervisor status: {err}"),
4882 )
4883 })?;
4884 let (configured, _) = module.configuration().map_err(|err| {
4885 RouterError::backend(
4886 0,
4887 frame.header.corr,
4888 format!("failed to read module configuration: {err}"),
4889 )
4890 })?;
4891 let image = module.running_image_agreement().await;
4893 let resources = Some(module.child_resource_usage());
4896 let pending_reload = Some(reload_verdict(
4897 &configured.program,
4898 status.spawned_from.as_deref(),
4899 image,
4900 ));
4901 modules.push(
4904 SupervisorEntry::new(
4905 status.module_id,
4906 status.state.to_string(),
4907 status.enabled,
4908 status.live,
4909 status.health.status,
4910 )
4911 .with_launch_nonce_env(Some(!cfg!(unix)))
4912 .with_launch_nonce_source(status.launch_nonce_source)
4913 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
4914 .with_protocol(status.protocol)
4915 .with_pending_reload(pending_reload)
4916 .with_last_probe_ms(status.health.last_probe_ms)
4917 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
4918 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
4919 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
4920 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
4921 .with_restart_count(Some(status.restart_count))
4922 .with_max_restarts(Some(status.max_restarts))
4923 .with_lifetime_restarts(Some(status.lifetime_restarts))
4924 .with_spawn_generation(Some(status.spawn_generation))
4925 .with_restart_window_secs(Some(status.restart_window.as_secs()))
4926 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
4927 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
4928 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
4929 .with_resources(resources),
4930 );
4931 }
4932 let response = ClientControlResponse::SupervisorList {
4933 generation,
4934 modules,
4935 };
4936 Ok(vec![control_response_body_frame(
4937 &frame,
4938 &response,
4939 "ClientControlResponse::SupervisorList",
4940 )?])
4941 }
4942
4943 fn handle_supervisor_stderr_tail(
4944 &self,
4945 frame: Frame,
4946 module_id: String,
4947 max_lines: Option<u32>,
4948 max_bytes: Option<u32>,
4949 ) -> Result<Vec<Frame>, RouterError> {
4950 let Some(module) = self.supervisor.get(&module_id) else {
4951 return Ok(vec![control_error_frame(
4952 &frame,
4953 "unknown_module",
4954 format!("module_id '{module_id}' is not supervised"),
4955 )?]);
4956 };
4957
4958 let snapshot = module.stderr_tail(
4959 max_lines.map(|value| value as usize),
4960 max_bytes.map(|value| value as usize),
4961 );
4962
4963 let response = ClientControlResponse::SupervisorStderrTail {
4964 module_id,
4965 tail: StderrTail {
4966 capture: match snapshot.capture {
4967 CaptureState::Captured => StderrCaptureState::Captured,
4968 CaptureState::Incomplete { reason } => {
4969 StderrCaptureState::Incomplete { reason }
4970 }
4971 CaptureState::NotCaptured { reason } => {
4972 StderrCaptureState::NotCaptured { reason }
4973 }
4974 },
4975 entries: snapshot
4976 .entries
4977 .into_iter()
4978 .map(|entry| match entry {
4979 TailEntry::Line {
4980 text,
4981 truncated,
4982 at_ms,
4983 } => StderrTailEntry::Line {
4984 text,
4985 truncated,
4986 at_ms,
4987 },
4988 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
4989 })
4990 .collect(),
4991 dropped_lines: snapshot.dropped_lines,
4992 },
4993 };
4994 Ok(vec![control_response_body_frame(
4995 &frame,
4996 &response,
4997 "ClientControlResponse::SupervisorStderrTail",
4998 )?])
4999 }
5000
5001 async fn handle_supervisor_terminals(
5002 &self,
5003 frame: Frame,
5004 module_id: String,
5005 ) -> Result<Vec<Frame>, RouterError> {
5006 let Some(module) = self.supervisor.get(&module_id) else {
5007 return Ok(vec![control_error_frame(
5008 &frame,
5009 "unknown_module",
5010 format!("module_id '{module_id}' is not supervised"),
5011 )?]);
5012 };
5013
5014 let terminals = module
5017 .read_durable_terminal_history()
5018 .await
5019 .map_err(|error| {
5020 RouterError::backend(
5021 0,
5022 frame.header.corr,
5023 format!("failed to read terminal history: {error}"),
5024 )
5025 })?;
5026 let response = ClientControlResponse::SupervisorTerminals {
5027 module_id,
5028 terminals,
5029 };
5030 Ok(vec![control_response_body_frame(
5031 &frame,
5032 &response,
5033 "ClientControlResponse::SupervisorTerminals",
5034 )?])
5035 }
5036
5037 fn handle_supervisor_routes(
5038 &self,
5039 frame: Frame,
5040 module_id: Option<String>,
5041 ) -> Result<Vec<Frame>, RouterError> {
5042 let modules = self
5043 .forwarding
5044 .route_census(module_id.as_deref())
5045 .map_err(RouterError::Forwarding)?
5046 .into_iter()
5047 .map(|(module_id, routes)| SupervisorRouteModule {
5048 module_id,
5049 routes: routes
5050 .into_iter()
5051 .map(|route| SupervisorRoute {
5052 consumer: match route.principal {
5053 Principal::Reserved { module_id } => {
5054 SupervisorRouteConsumer::Reserved { module_id }
5055 }
5056 Principal::Direct | Principal::Unverified => {
5057 SupervisorRouteConsumer::Direct {
5058 connection_id: route.goodbye_target.connection_id.get(),
5059 }
5060 }
5061 },
5062 age_ms: Instant::now()
5063 .saturating_duration_since(route.bound_at)
5064 .as_millis()
5065 .try_into()
5066 .unwrap_or(u64::MAX),
5067 draining: route.draining,
5068 drain_reason: route.drain_reason,
5069 })
5070 .collect(),
5071 })
5072 .collect();
5073 let response = ClientControlResponse::SupervisorRoutes { modules };
5074 Ok(vec![control_response_body_frame(
5075 &frame,
5076 &response,
5077 "ClientControlResponse::SupervisorRoutes",
5078 )?])
5079 }
5080
5081 async fn handle_supervisor_provenance(
5082 &self,
5083 frame: Frame,
5084 module_id: Option<String>,
5085 ) -> Result<Vec<Frame>, RouterError> {
5086 let mut selected = if let Some(module_id) = module_id {
5087 let Some(module) = self.supervisor.get(&module_id) else {
5088 return Ok(vec![control_error_frame(
5089 &frame,
5090 "unknown_module",
5091 format!("module_id '{module_id}' is not supervised"),
5092 )?]);
5093 };
5094 vec![module]
5095 } else {
5096 self.supervisor.list()
5097 };
5098
5099 let mut modules = Vec::with_capacity(selected.len());
5100 for module in selected.drain(..) {
5101 let (status, observed_image) = module
5102 .status_and_running_image_agreement()
5103 .await
5104 .map_err(|err| {
5105 RouterError::backend(
5106 0,
5107 frame.header.corr,
5108 format!("failed to read supervisor status: {err}"),
5109 )
5110 })?;
5111 let module_declared = self
5112 .registry
5113 .get_module(&status.module_id)
5114 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5115 .and_then(|registration| registration.manifest.provenance)
5116 .map(|build| ModuleDeclaredProvenance::Reported { build })
5117 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5118 #[cfg(test)]
5119 let running_image = match &self.provenance_probe_override {
5120 Some(result) => result.clone(),
5121 None => observed_image,
5122 };
5123 #[cfg(not(test))]
5124 let running_image = observed_image;
5125 modules.push(SupervisorModuleProvenance {
5126 module_id: status.module_id,
5127 module_declared,
5128 daemon_observed: SupervisorObservedProcess {
5129 launch_nonce_source: status.launch_nonce_source,
5130 pid: status.pid,
5131 spawned_at_ms: status.spawned_at_ms,
5132 spawned_from: status.spawned_from,
5133 running_image,
5134 },
5135 });
5136 }
5137 let daemon = SupervisorDaemonProvenance {
5138 daemon_build: self.daemon_provenance.build.clone(),
5139 daemon_observed: DaemonObservedProcess {
5140 pid: self.daemon_provenance.pid,
5141 started_at_ms: self
5142 .daemon_provenance
5143 .start_clock
5144 .map(|clock| clock.started_at_ms())
5145 .or(self.daemon_provenance.started_at_ms),
5146 running_image: self
5147 .daemon_provenance
5148 .probe
5149 .observe(
5150 self.daemon_provenance.pid,
5151 self.daemon_provenance.executable_path.as_deref(),
5152 self.daemon_provenance.executable_identity.clone(),
5153 self.daemon_provenance.process_start_time,
5154 )
5155 .await,
5156 },
5157 };
5158 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5159 Ok(vec![control_response_body_frame(
5160 &frame,
5161 &response,
5162 "ClientControlResponse::SupervisorProvenance",
5163 )?])
5164 }
5165
5166 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5167 self.refresh_capability_requirements();
5168 let generation = self
5169 .registry
5170 .generation()
5171 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5172 let modules = self
5173 .supervisor
5174 .list()
5175 .into_iter()
5176 .map(|module| {
5177 let status = module.status_for_control("health").map_err(|err| {
5178 RouterError::backend(
5179 0,
5180 frame.header.corr,
5181 format!("failed to read supervisor health: {err}"),
5182 )
5183 })?;
5184 let module_id = status.module_id;
5185 let capability_detail = self
5186 .capability_evaluator
5187 .required_problem_detail(&module_id);
5188 Ok(SupervisorHealthEntry {
5189 module_id,
5190 status: status.health.status,
5191 detail: append_capability_problem_detail(
5192 status.health.detail,
5193 capability_detail,
5194 ),
5195 metrics: status.health.metrics,
5196 consecutive_failures: status.health.consecutive_failures,
5197 late_answer_count: status.health.late_answer_count,
5198 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5199 last_action: status.health.last_action,
5200 last_action_ms: status.health.last_action_ms,
5201 last_probe_ms: status.health.last_probe_ms,
5202 })
5203 })
5204 .collect::<Result<Vec<_>, RouterError>>()?;
5205 let response = ClientControlResponse::SupervisorHealth {
5206 generation,
5207 modules,
5208 };
5209 Ok(vec![control_response_body_frame(
5210 &frame,
5211 &response,
5212 "ClientControlResponse::SupervisorHealth",
5213 )?])
5214 }
5215
5216 async fn handle_supervisor_restart(
5217 &self,
5218 frame: Frame,
5219 module_id: String,
5220 drain_timeout_ms: Option<u64>,
5221 requester: Principal,
5222 ) -> Result<Vec<Frame>, RouterError> {
5223 let operation_lock = self.supervisor.operation_lock();
5224 let _operation_guard = operation_lock.lock().await;
5225 let Some(module) = self.supervisor.get(&module_id) else {
5226 return Ok(vec![control_error_frame(
5227 &frame,
5228 "unknown_module",
5229 format!("module_id '{module_id}' is not supervised"),
5230 )?]);
5231 };
5232
5233 self.route_outages.mark_operator_action(&module_id);
5234 if let Err(err) = module
5235 .restart_requested(drain_timeout_ms, Some(requester))
5236 .await
5237 {
5238 self.route_outages
5239 .operator_action_ended_unrefused(&module_id);
5240 let (code, message) = match err {
5241 crate::supervise::SuperviseError::Disabled { .. } => {
5242 ("module_disabled", err.to_string())
5243 }
5244 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5245 ("swap_in_progress", err.to_string())
5246 }
5247 _ => (
5248 "target_unavailable",
5249 format!("failed to restart module_id '{module_id}': {err}"),
5250 ),
5251 };
5252 return Ok(vec![control_error_frame(&frame, code, message)?]);
5253 }
5254
5255 let response = ClientControlResponse::SupervisorAck {
5256 module_id,
5257 applied: true,
5258 };
5259 Ok(vec![control_response_body_frame(
5260 &frame,
5261 &response,
5262 "ClientControlResponse::SupervisorAck",
5263 )?])
5264 }
5265
5266 async fn handle_supervisor_swap(
5270 &self,
5271 frame: Frame,
5272 module_id: String,
5273 ready_timeout_ms: Option<u64>,
5274 requester: Principal,
5275 ) -> Result<Vec<Frame>, RouterError> {
5276 let module = {
5283 let operation_lock = self.supervisor.operation_lock();
5284 let _operation_guard = operation_lock.lock().await;
5285 self.supervisor.get(&module_id)
5286 };
5287 let Some(module) = module else {
5288 return Ok(vec![control_error_frame(
5289 &frame,
5290 "unknown_module",
5291 format!("module_id '{module_id}' is not supervised"),
5292 )?]);
5293 };
5294
5295 self.route_outages.mark_operator_action(&module_id);
5296 if let Err(err) = module
5297 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5298 .await
5299 {
5300 self.route_outages
5301 .operator_action_ended_unrefused(&module_id);
5302 use crate::supervise::SuperviseError;
5303 let message = err.to_string();
5304 let error = match err {
5305 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5306 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5307 code: "swap_refused".to_string(),
5308 message,
5309 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5310 },
5311 SuperviseError::SwapFailed {
5312 arm,
5313 candidate_exit,
5314 ..
5315 } => ErrorBody {
5316 code: "swap_failed".to_string(),
5317 message,
5318 detail: Some(serde_json::json!({
5319 "arm": arm.as_str(),
5320 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5321 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5322 })),
5323 },
5324 _ => ErrorBody::new(
5325 "target_unavailable",
5326 format!("failed to swap module_id '{module_id}': {message}"),
5327 ),
5328 };
5329 return Ok(vec![control_error_body_frame(&frame, error)?]);
5330 }
5331 self.route_outages
5335 .operator_action_ended_unrefused(&module_id);
5336
5337 let response = ClientControlResponse::SupervisorAck {
5338 module_id,
5339 applied: true,
5340 };
5341 Ok(vec![control_response_body_frame(
5342 &frame,
5343 &response,
5344 "ClientControlResponse::SupervisorAck",
5345 )?])
5346 }
5347
5348 async fn handle_supervisor_reload(
5349 &self,
5350 frame: Frame,
5351 module_id: String,
5352 requester: Principal,
5353 ) -> Result<Vec<Frame>, RouterError> {
5354 let operation_lock = self.supervisor.operation_lock();
5355 let _operation_guard = operation_lock.lock().await;
5356 let Some(module) = self.supervisor.get(&module_id) else {
5357 return Ok(vec![control_error_frame(
5358 &frame,
5359 "unknown_module",
5360 format!("module_id '{module_id}' is not supervised"),
5361 )?]);
5362 };
5363
5364 self.route_outages.mark_operator_action(&module_id);
5365 if let Err(err) = module.reload_requested(Some(requester)).await {
5366 self.route_outages
5367 .operator_action_ended_unrefused(&module_id);
5368 let (code, message) = match err {
5369 crate::supervise::SuperviseError::Disabled { .. } => {
5370 ("module_disabled", err.to_string())
5371 }
5372 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5373 ("swap_in_progress", err.to_string())
5374 }
5375 _ => (
5376 "reload_failed",
5377 format!("failed to reload module_id '{module_id}': {err}"),
5378 ),
5379 };
5380 return Ok(vec![control_error_frame(&frame, code, message)?]);
5381 }
5382
5383 let response = ClientControlResponse::SupervisorAck {
5384 module_id,
5385 applied: true,
5386 };
5387 Ok(vec![control_response_body_frame(
5388 &frame,
5389 &response,
5390 "ClientControlResponse::SupervisorAck",
5391 )?])
5392 }
5393
5394 async fn handle_supervisor_rescan(
5395 &self,
5396 frame: Frame,
5397 preview: bool,
5398 requester: Principal,
5399 ) -> Result<Vec<Frame>, RouterError> {
5400 let Some(context) = self.rescan.clone() else {
5401 return Ok(vec![control_error_frame(
5402 &frame,
5403 "rescan_unavailable",
5404 "the daemon was not started with a reloadable config path".to_string(),
5405 )?]);
5406 };
5407
5408 let operation_lock = self.supervisor.operation_lock();
5409 let _operation_guard = operation_lock.lock().await;
5410 let loaded = match crate::daemon_config::load_with_activity(&context.config_path) {
5411 Ok(config) => config,
5412 Err(err) => {
5413 return Ok(vec![control_error_frame(
5414 &frame,
5415 "invalid_daemon_config",
5416 format!("supervisor rescan rejected daemon config: {err}"),
5417 )?])
5418 }
5419 };
5420 let Some((config, activity_config)) = loaded else {
5428 return Ok(vec![control_error_frame(
5429 &frame,
5430 "invalid_daemon_config",
5431 format!(
5432 "daemon config not found at {}; refusing to rescan (an absent config would \
5433 retire every supervised module)",
5434 context.config_path.display()
5435 ),
5436 )?]);
5437 };
5438 let (
5439 configured_port,
5440 storage_config,
5441 admission_facts_carrier_module_id,
5442 admission_facts_targets,
5443 scope_authority_owners,
5444 privileged_operator_prompt,
5445 modules,
5446 reserved_capabilities,
5447 ) = (
5448 config.port,
5449 config.storage,
5450 config.admission_facts_carrier_module_id,
5451 config.admission_facts_targets,
5452 config.scope_authority_owners,
5453 config.privileged_operator_prompt,
5454 config.modules,
5455 config.reserved_capabilities,
5456 );
5457
5458 let mut restart_required = Vec::new();
5467 for section in RestartRequiredSection::ALL {
5468 let changed = match section {
5469 RestartRequiredSection::Port => configured_port != context.configured_port,
5470 RestartRequiredSection::Storage => storage_config != context.storage_config,
5471 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5472 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5473 }
5474 RestartRequiredSection::AdmissionFactsTargets => {
5475 admission_facts_targets != context.admission_facts_targets
5476 }
5477 RestartRequiredSection::ScopeAuthorityOwners => {
5478 scope_authority_owners != context.scope_authority_owners
5479 }
5480 RestartRequiredSection::PrivilegedOperatorPrompt => {
5481 privileged_operator_prompt != context.privileged_operator_prompt
5482 }
5483 };
5484 if changed {
5485 restart_required.push(section.label().to_string());
5486 }
5487 }
5488 if !restart_required.is_empty() {
5489 warn!(
5490 config_path = %context.config_path.display(),
5491 sections = %restart_required.join(", "),
5492 "daemon config changed outside the modules section; restart the daemon to apply those changes"
5493 );
5494 }
5495
5496 for configured in &modules {
5497 if let Err(err) = validate_spec(&configured.module_spec()) {
5498 return Ok(vec![control_error_frame(
5499 &frame,
5500 "invalid_daemon_config",
5501 format!("supervisor rescan rejected daemon config: {err}"),
5502 )?]);
5503 }
5504 }
5505
5506 let configured_capabilities = modules
5507 .iter()
5508 .map(|module| (module.module_id.clone(), module.enabled))
5509 .collect::<Vec<_>>();
5510 let preview_capability_warnings = if preview {
5511 let (_, registrations) = self.runtime_capability_snapshot()?;
5512 let current_modules = self
5513 .supervisor
5514 .list()
5515 .into_iter()
5516 .map(|module| module.module_id().to_string())
5517 .collect::<BTreeSet<_>>();
5518 let resulting_modules = configured_capabilities.clone();
5519 let removed = current_modules
5520 .into_iter()
5521 .filter(|module_id| {
5522 !resulting_modules
5523 .iter()
5524 .any(|(configured_id, _)| configured_id == module_id)
5525 })
5526 .collect::<Vec<_>>();
5527 self.capability_evaluator.preview_removal_warnings(
5528 resulting_modules,
5529 &removed,
5530 ®istrations,
5531 )
5532 } else {
5533 Vec::new()
5534 };
5535 let result = match self
5536 .reconcile_supervised_modules(&context.supervisor, modules, preview, requester.clone())
5537 .await
5538 {
5539 Ok(result) => result,
5540 Err(message) => {
5541 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5542 }
5543 };
5544 if !preview {
5545 if let Some(ledger) = &self.activity {
5546 ledger.module_rescanned(&result.added, &result.removed, requester);
5547 }
5548 if let Err(code) = self.configure_activity(activity_config).await {
5549 warn!(
5550 code,
5551 "activity retention could not be applied during rescan"
5552 );
5553 }
5554 self.capability_evaluator
5555 .configure(configured_capabilities, reserved_capabilities);
5556 self.capability_evaluator.wake_deadline_loop();
5557 self.refresh_capability_requirements();
5558 }
5559 let mut result = result;
5560 result.restart_required = restart_required;
5561 result.capability_warnings = preview_capability_warnings;
5562 let response = ClientControlResponse::SupervisorRescan { result };
5563 Ok(vec![control_response_body_frame(
5564 &frame,
5565 &response,
5566 "ClientControlResponse::SupervisorRescan",
5567 )?])
5568 }
5569
5570 async fn handle_supervisor_release_reserved(
5571 &self,
5572 frame: Frame,
5573 module_id: String,
5574 ) -> Result<Vec<Frame>, RouterError> {
5575 let Some(context) = self.rescan.clone() else {
5576 return Ok(vec![control_error_frame(
5577 &frame,
5578 "release_unavailable",
5579 "reserved-id release requires a daemon started with a reloadable config path",
5580 )?]);
5581 };
5582 let operation_lock = self.supervisor.operation_lock();
5583 let _operation_guard = operation_lock.lock().await;
5584 let loaded = match crate::daemon_config::load(&context.config_path) {
5585 Ok(Some(config)) => config,
5586 Ok(None) => {
5587 return Ok(vec![control_error_frame(
5588 &frame,
5589 "invalid_daemon_config",
5590 format!(
5591 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5592 context.config_path.display()
5593 ),
5594 )?])
5595 }
5596 Err(err) => {
5597 return Ok(vec![control_error_frame(
5598 &frame,
5599 "invalid_daemon_config",
5600 format!("unable to verify reserved-id release against daemon config: {err}"),
5601 )?])
5602 }
5603 };
5604 if loaded
5605 .modules
5606 .iter()
5607 .any(|configured| configured.module_id == module_id)
5608 {
5609 return Ok(vec![control_error_frame(
5610 &frame,
5611 "reserved_module_configured",
5612 format!(
5613 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5614 ),
5615 )?]);
5616 }
5617 if !self.supervisor.release_retained_reserved_gate(&module_id) {
5618 return Ok(vec![control_error_frame(
5619 &frame,
5620 "reserved_gate_not_retained",
5621 format!(
5622 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
5623 ),
5624 )?]);
5625 }
5626
5627 let response = ClientControlResponse::SupervisorAck {
5628 module_id,
5629 applied: true,
5630 };
5631 Ok(vec![control_response_body_frame(
5632 &frame,
5633 &response,
5634 "ClientControlResponse::SupervisorAck",
5635 )?])
5636 }
5637
5638 async fn reconcile_supervised_modules(
5647 &self,
5648 supervisor: &Supervisor,
5649 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
5650 preview: bool,
5651 requester: Principal,
5652 ) -> Result<SupervisorRescanResult, String> {
5653 let mut current = BTreeMap::new();
5654 for module in self.supervisor.list() {
5655 let (spec, health) = module.configuration().map_err(|err| {
5656 format!(
5657 "failed to read configuration for module_id '{}': {err}",
5658 module.module_id()
5659 )
5660 })?;
5661 let enabled = module
5662 .status()
5663 .map_err(|err| {
5664 format!(
5665 "failed to read status for module_id '{}': {err}",
5666 module.module_id()
5667 )
5668 })?
5669 .enabled;
5670 current.insert(
5671 module.module_id().to_string(),
5672 (module, spec, health, enabled),
5673 );
5674 }
5675 let configured = configured_modules
5676 .into_iter()
5677 .map(|module| (module.module_id.clone(), module))
5678 .collect::<BTreeMap<_, _>>();
5679
5680 let added = configured
5681 .keys()
5682 .filter(|module_id| !current.contains_key(*module_id))
5683 .cloned()
5684 .collect::<Vec<_>>();
5685 let removed = current
5686 .keys()
5687 .filter(|module_id| !configured.contains_key(*module_id))
5688 .cloned()
5689 .collect::<Vec<_>>();
5690 let mut changed_pending_reload = Vec::new();
5691 let mut configuration_changes = BTreeSet::new();
5692 let mut enabled_changes = BTreeSet::new();
5693 let mut unchanged = 0_u32;
5694
5695 for (module_id, configured_module) in &configured {
5696 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5697 else {
5698 continue;
5699 };
5700 let launch_changed = *current_spec != configured_module.module_spec();
5704 let configuration_changed =
5705 launch_changed || *current_health != configured_module.health;
5706 let enabled_changed = *current_enabled != configured_module.enabled;
5707 if configuration_changed {
5708 configuration_changes.insert(module_id.clone());
5709 }
5710 if launch_changed {
5711 changed_pending_reload.push(module_id.clone());
5712 }
5713 if enabled_changed {
5714 enabled_changes.insert(module_id.clone());
5715 }
5716 if !configuration_changed && !enabled_changed {
5717 unchanged = unchanged.saturating_add(1);
5718 }
5719 }
5720
5721 if preview {
5727 return Ok(SupervisorRescanResult {
5728 added,
5729 removed,
5730 changed_pending_reload,
5731 enabled_changes: enabled_changes.iter().cloned().collect(),
5732 unchanged,
5733 preview: true,
5734 restart_required: Vec::new(),
5738 capability_warnings: Vec::new(),
5739 });
5740 }
5741
5742 for module_id in &removed {
5743 let module = ¤t
5744 .get(module_id)
5745 .expect("removed module came from current supervisor state")
5746 .0;
5747 module
5748 .retire_requested(Some(requester.clone()))
5749 .await
5750 .map_err(|err| {
5751 format!("failed to retire module_id '{module_id}' during rescan: {err}")
5752 })?;
5753 self.supervisor.record_rescan_removal(module_id);
5787 self.supervisor.retire(module_id);
5788 self.route_outages.forget(module_id);
5789 }
5790
5791 for module_id in configured.keys() {
5792 let Some((module, _, _, _)) = current.get(module_id) else {
5793 continue;
5794 };
5795 let configured_module = configured
5796 .get(module_id)
5797 .expect("configured module id came from configured map");
5798 if configuration_changes.contains(module_id) {
5799 module
5800 .update_configuration(
5801 configured_module.module_spec(),
5802 configured_module.health.clone(),
5803 configured_module.drain_timeout_ms,
5804 )
5805 .await
5806 .map_err(|err| {
5807 format!(
5808 "failed to update module_id '{module_id}' configuration during rescan: {err}"
5809 )
5810 })?;
5811 }
5812 if enabled_changes.contains(module_id) {
5813 self.route_outages.mark_operator_action(module_id);
5816 module
5817 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
5818 .await
5819 .map_err(|err| {
5820 self.route_outages.operator_action_ended_unrefused(module_id);
5821 format!(
5822 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
5823 configured_module.enabled
5824 )
5825 })?;
5826 }
5827 }
5828
5829 for module_id in &added {
5830 let configured_module = configured
5831 .get(module_id)
5832 .expect("added module id came from configured map");
5833 supervisor
5834 .supervise_configured_requested(
5835 configured_module.module_spec(),
5836 configured_module.enabled,
5837 configured_module.health.clone(),
5838 configured_module.drain_timeout_ms,
5839 configured_module.restart,
5840 Some(requester.clone()),
5841 )
5842 .map_err(|err| {
5843 format!("failed to add module_id '{module_id}' during rescan: {err}")
5844 })?;
5845 }
5846
5847 Ok(SupervisorRescanResult {
5848 added,
5849 removed,
5850 changed_pending_reload,
5851 enabled_changes: enabled_changes.iter().cloned().collect(),
5852 unchanged,
5853 preview: false,
5854 restart_required: Vec::new(),
5857 capability_warnings: Vec::new(),
5858 })
5859 }
5860
5861 async fn handle_supervisor_set_enabled(
5862 &self,
5863 frame: Frame,
5864 module_id: String,
5865 enabled: bool,
5866 requester: Principal,
5867 ) -> Result<Vec<Frame>, RouterError> {
5868 let operation_lock = self.supervisor.operation_lock();
5869 let _operation_guard = operation_lock.lock().await;
5870 let Some(module) = self.supervisor.get(&module_id) else {
5871 return Ok(vec![control_error_frame(
5872 &frame,
5873 "unknown_module",
5874 format!("module_id '{module_id}' is not supervised"),
5875 )?]);
5876 };
5877
5878 self.route_outages.mark_operator_action(&module_id);
5881 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
5882 Ok(applied) => applied,
5883 Err(err) => {
5884 self.route_outages
5885 .operator_action_ended_unrefused(&module_id);
5886 return Ok(vec![control_error_frame(
5887 &frame,
5888 "target_unavailable",
5889 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
5890 )?]);
5891 }
5892 };
5893 if !applied {
5894 self.route_outages
5897 .operator_action_ended_unrefused(&module_id);
5898 }
5899
5900 self.capability_evaluator.wake_deadline_loop();
5901 self.refresh_capability_requirements();
5902 let response = ClientControlResponse::SupervisorAck { module_id, applied };
5903 Ok(vec![control_response_body_frame(
5904 &frame,
5905 &response,
5906 "ClientControlResponse::SupervisorAck",
5907 )?])
5908 }
5909
5910 async fn handle_supervisor_health_probe(
5911 &self,
5912 frame: Frame,
5913 module_id: String,
5914 ) -> Result<Vec<Frame>, RouterError> {
5915 self.refresh_capability_requirements();
5916 let Some(registration) = self
5917 .registry
5918 .get_module(&module_id)
5919 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5920 else {
5921 return Ok(vec![control_error_frame(
5922 &frame,
5923 "unknown_module",
5924 format!("module_id '{module_id}' is not registered"),
5925 )?]);
5926 };
5927
5928 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
5941 {
5942 return Ok(vec![control_error_frame(
5943 &frame,
5944 "health_not_advertised",
5945 format!("module_id '{module_id}' did not advertise health.check"),
5946 )?]);
5947 }
5948
5949 let deadline = Instant::now() + self.health_probe_timeout;
5950 let pending = match self.forwarding.begin_module_control_rpc_for(
5951 &module_id,
5952 MODULE_CONTROL_OP_HEALTH_CHECK,
5953 deadline,
5954 ) {
5955 Ok(pending) => pending,
5956 Err(err) => {
5957 return Ok(vec![control_error_frame(
5958 &frame,
5959 forwarding_error_code(&err),
5960 err.to_string(),
5961 )?])
5962 }
5963 };
5964
5965 let PendingModuleControlRpc {
5966 endpoint,
5967 module_sink,
5968 negotiated_ver,
5969 corr: probe_corr,
5970 receiver,
5971 } = pending;
5972 let mut guard =
5973 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
5974 let probe_body =
5975 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5976 RouterError::backend(
5977 0,
5978 frame.header.corr,
5979 format!("failed to encode health.check request: {err}"),
5980 )
5981 })?;
5982 let probe_frame = Frame::build_with_version(
5983 negotiated_ver,
5984 FrameType::Request,
5985 control_flags(),
5986 0,
5987 0,
5988 probe_corr,
5989 probe_body,
5990 )
5991 .map_err(RouterError::FrameBuild)?;
5992
5993 if let Err(err) = module_sink.send(probe_frame).await {
5994 return Ok(vec![control_error_frame(
5995 &frame,
5996 "target_unavailable",
5997 err.to_string(),
5998 )?]);
5999 }
6000
6001 match timeout_at(deadline, receiver).await {
6002 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6003 guard.disarm();
6004 let Some(report) = response.health_report() else {
6005 return Ok(vec![control_error_frame(
6006 &frame,
6007 "invalid_control_body",
6008 "health.check RPC returned a non-health response",
6009 )?]);
6010 };
6011 let HealthReport {
6016 status,
6017 detail,
6018 metrics,
6019 } = report;
6020 let capability_detail = self
6021 .capability_evaluator
6022 .required_problem_detail(&module_id);
6023 let response = ClientControlResponse::SupervisorHealthProbe {
6024 module_id,
6025 status,
6026 detail: append_capability_problem_detail(detail, capability_detail),
6027 metrics,
6028 };
6029 Ok(vec![control_response_body_frame(
6030 &frame,
6031 &response,
6032 "ClientControlResponse::SupervisorHealthProbe",
6033 )?])
6034 }
6035 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6036 guard.disarm();
6037 Ok(vec![control_error_body_frame(&frame, body)?])
6038 }
6039 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6040 guard.disarm();
6041 Ok(vec![control_error_frame(
6042 &frame,
6043 "target_unavailable",
6044 message,
6045 )?])
6046 }
6047 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6048 guard.disarm();
6049 Ok(vec![control_error_frame(
6050 &frame,
6051 "invalid_control_body",
6052 message,
6053 )?])
6054 }
6055 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6056 guard.disarm();
6057 Ok(vec![control_error_frame(
6058 &frame,
6059 "invalid_control_body",
6060 format!("expected module-control op '{expected}', got '{actual}'"),
6061 )?])
6062 }
6063 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6064 guard.disarm();
6065 Ok(vec![control_error_frame(
6066 &frame,
6067 "module_timeout",
6068 format!(
6069 "module_id '{module_id}' answered health.check after {:?}",
6070 self.health_probe_timeout
6071 ),
6072 )?])
6073 }
6074 Ok(Err(_)) => Ok(vec![control_error_frame(
6075 &frame,
6076 "target_unavailable",
6077 "health.check waiter was canceled before the module responded",
6078 )?]),
6079 Err(_) => Ok(vec![control_error_frame(
6080 &frame,
6081 "module_timeout",
6082 format!(
6083 "module_id '{module_id}' did not answer health.check within {:?}",
6084 self.health_probe_timeout
6085 ),
6086 )?]),
6087 }
6088 }
6089
6090 fn supervisor_status(
6091 &self,
6092 module_id: &str,
6093 corr: u64,
6094 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6095 self.supervisor
6096 .get(module_id)
6097 .map(|module| {
6098 let warming = module.is_warming_for_control("status").map_err(|err| {
6099 RouterError::backend(
6100 0,
6101 corr,
6102 format!(
6103 "failed to read supervisor warming state for module_id '{module_id}': {err}"
6104 ),
6105 )
6106 })?;
6107 module.status_for_control("status").map_err(|err| {
6108 RouterError::backend(
6109 0,
6110 corr,
6111 format!(
6112 "failed to read supervisor status for module_id '{module_id}': {err}"
6113 ),
6114 )
6115 }).map(|status| (status, warming))
6116 })
6117 .transpose()
6118 }
6119
6120 fn guard_module_control_op(
6121 &self,
6122 frame: &Frame,
6123 module_id: &str,
6124 op: &str,
6125 ) -> Result<Option<Frame>, RouterError> {
6126 if self.module_grants_op(module_id, op, frame.header.corr)? {
6127 return Ok(None);
6128 }
6129
6130 Ok(Some(control_error_frame(
6131 frame,
6132 "op_not_allowed",
6133 format!("module_id '{module_id}' did not grant control op '{op}'"),
6134 )?))
6135 }
6136
6137 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6138 let Some(registration) = self
6139 .registry
6140 .get_module(module_id)
6141 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6142 else {
6143 return Ok(false);
6144 };
6145 Ok(module_registration_grants_op(®istration.control_ops, op))
6146 }
6147
6148 fn handle_status_update(
6149 &self,
6150 endpoint: ModuleEndpointId,
6151 frame: Frame,
6152 ) -> Result<Vec<Frame>, RouterError> {
6153 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6154 Ok(update) => update,
6155 Err(err) => {
6156 if is_known_module_push_op(&frame.body) {
6161 return Ok(vec![control_error_frame(
6162 &frame,
6163 "invalid_control_body",
6164 format!("malformed module control push body: {err}"),
6165 )?]);
6166 }
6167 return Ok(Vec::new());
6168 }
6169 };
6170
6171 match update {
6172 ModuleControlPush::RouteStatus {
6173 route_channel,
6174 route_epoch,
6175 status,
6176 } => {
6177 self.forwarding
6178 .cache_status(endpoint, route_channel, route_epoch, status)
6179 .map_err(RouterError::Forwarding)?;
6180 }
6181 }
6182 Ok(Vec::new())
6183 }
6184
6185 fn handle_route_poll(
6186 &self,
6187 ctx: &RouteCtx,
6188 frame: Frame,
6189 route_channel: u16,
6190 route_epoch: u32,
6191 kind: PollKind,
6192 ) -> Result<Vec<Frame>, RouterError> {
6193 let snapshot = self
6194 .forwarding
6195 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6196 .map_err(RouterError::Forwarding)?;
6197 let response = match (kind, snapshot) {
6198 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6199 ClientControlResponse::RoutePoll {
6200 route_channel,
6201 route_epoch,
6202 status,
6203 live: None,
6204 }
6205 }
6206 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6207 route_channel,
6208 route_epoch,
6209 status: None,
6210 live: None,
6211 },
6212 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6213 let live = self
6239 .process_liveness
6240 .as_ref()
6241 .and_then(|source| source.process_live(&module_id))
6242 .unwrap_or(true);
6243 ClientControlResponse::RoutePoll {
6244 route_channel,
6245 route_epoch,
6246 status: None,
6247 live: Some(live),
6248 }
6249 }
6250 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6251 route_channel,
6252 route_epoch,
6253 status: None,
6254 live: Some(false),
6255 },
6256 };
6257
6258 Ok(vec![control_response_body_frame(
6259 &frame,
6260 &response,
6261 "ClientControlResponse::RoutePoll",
6262 )?])
6263 }
6264
6265 pub(crate) fn observe_module_control_completion(
6266 &self,
6267 completion: ModuleControlRpcCompletion,
6268 ) -> bool {
6269 match completion {
6270 ModuleControlRpcCompletion::Unknown => false,
6271 ModuleControlRpcCompletion::Settled => true,
6272 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6273 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6274 info!(
6275 module_id = %module_id,
6276 latency_ms,
6277 "late health.check answer proves the module is alive"
6278 );
6279 match self
6280 .supervisor
6281 .record_late_health_answer(&module_id, latency_ms)
6282 {
6283 Ok(true) => {}
6284 Ok(false) => debug!(
6285 module_id = %module_id,
6286 latency_ms,
6287 "late health.check answer has no active supervisor snapshot"
6288 ),
6289 Err(err) => warn!(
6290 module_id = %module_id,
6291 latency_ms,
6292 error = %err,
6293 "failed to record late health.check answer"
6294 ),
6295 }
6296 true
6297 }
6298 }
6299 }
6300
6301 fn refuse_to_end_module_connection_for_a_client(
6322 &self,
6323 module_connection_id: ConnectionId,
6324 corr: u64,
6325 err: ForwardingError,
6326 ) -> Result<(), RouterError> {
6327 if let ForwardingError::ConnectionClosing { connection_id } = err {
6328 if connection_id != module_connection_id {
6329 warn!(
6330 module_connection_id = module_connection_id.get(),
6331 client_connection_id = connection_id.get(),
6332 corr,
6333 "dropping a route.bind response for a closing client; the module connection keeps serving"
6334 );
6335 return Ok(());
6336 }
6337 }
6338 Err(RouterError::Forwarding(err))
6339 }
6340
6341 fn handle_module_relay_response(
6342 &self,
6343 connection_id: ConnectionId,
6344 frame: Frame,
6345 ) -> Result<Vec<Frame>, RouterError> {
6346 let mut secondary_error = None;
6347 let outcome = match frame.header.ty {
6348 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6349 Ok(probe) if probe.op == "route.bind" => {
6350 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6351 Ok(ModuleControlResponse::RouteBindAck {}) => {
6352 RouteBindRelayOutcome::Accepted
6353 }
6354 Ok(other) => {
6355 let message =
6356 format!("route.bind response carried unexpected body: {other:?}");
6357 secondary_error = Some(control_error_frame(
6358 &frame,
6359 "invalid_control_body",
6360 message.clone(),
6361 )?);
6362 RouteBindRelayOutcome::ModuleGone(message)
6363 }
6364 Err(err) => {
6365 let message = format!("malformed route.bind response body: {err}");
6366 secondary_error = Some(control_error_frame(
6367 &frame,
6368 "invalid_control_body",
6369 message.clone(),
6370 )?);
6371 RouteBindRelayOutcome::ModuleGone(message)
6372 }
6373 }
6374 }
6375 Ok(probe) => {
6376 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6377 {
6378 Ok(response) => ModuleControlRpcOutcome::Response(response),
6379 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6380 "malformed {} response body: {err}",
6381 probe.op
6382 )),
6383 };
6384 let completion = self
6385 .forwarding
6386 .complete_module_control_rpc(
6387 connection_id,
6388 frame.header.corr,
6389 Some(&probe.op),
6390 outcome,
6391 )
6392 .map_err(RouterError::Forwarding)?;
6393 if !self.observe_module_control_completion(completion) {
6394 debug!(
6395 connection_id = connection_id.get(),
6396 corr = frame.header.corr,
6397 op = %probe.op,
6398 "dropping late or unknown module-control RPC response"
6399 );
6400 }
6401 return Ok(Vec::new());
6402 }
6403 Err(err) => {
6404 if let Some(expected_op) = self
6405 .forwarding
6406 .pending_module_control_op(connection_id, frame.header.corr)
6407 .map_err(RouterError::Forwarding)?
6408 {
6409 let completion = self
6410 .forwarding
6411 .complete_module_control_rpc(
6412 connection_id,
6413 frame.header.corr,
6414 None,
6415 ModuleControlRpcOutcome::MalformedResponse(format!(
6416 "malformed {expected_op} response body: {err}"
6417 )),
6418 )
6419 .map_err(RouterError::Forwarding)?;
6420 if !self.observe_module_control_completion(completion) {
6421 debug!(
6422 connection_id = connection_id.get(),
6423 corr = frame.header.corr,
6424 "dropping late malformed module-control RPC response"
6425 );
6426 }
6427 return Ok(Vec::new());
6428 }
6429 let message = format!("malformed route.bind response body: {err}");
6430 secondary_error = Some(control_error_frame(
6431 &frame,
6432 "invalid_control_body",
6433 message.clone(),
6434 )?);
6435 RouteBindRelayOutcome::ModuleGone(message)
6436 }
6437 },
6438 FrameType::Error => {
6439 if self
6440 .forwarding
6441 .pending_module_control_op(connection_id, frame.header.corr)
6442 .map_err(RouterError::Forwarding)?
6443 .is_some()
6444 {
6445 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6446 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
6447 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6448 "malformed module-control ERROR body: {err}"
6449 )),
6450 };
6451 let completion = self
6452 .forwarding
6453 .complete_module_control_rpc(
6454 connection_id,
6455 frame.header.corr,
6456 None,
6457 outcome,
6458 )
6459 .map_err(RouterError::Forwarding)?;
6460 if !self.observe_module_control_completion(completion) {
6461 debug!(
6462 connection_id = connection_id.get(),
6463 corr = frame.header.corr,
6464 "dropping late or unknown module-control RPC error"
6465 );
6466 }
6467 return Ok(Vec::new());
6468 }
6469 match serde_json::from_slice::<ErrorBody>(&frame.body) {
6470 Ok(body) => RouteBindRelayOutcome::Rejected(body),
6471 Err(err) => {
6472 let message = format!("malformed route.bind ERROR body: {err}");
6473 secondary_error = Some(control_error_frame(
6474 &frame,
6475 "invalid_control_body",
6476 message.clone(),
6477 )?);
6478 RouteBindRelayOutcome::ModuleGone(message)
6479 }
6480 }
6481 }
6482 ty => {
6483 return Ok(vec![control_error_frame(
6484 &frame,
6485 "unsupported_control_frame",
6486 format!("unsupported module channel-0 frame {ty:?}"),
6487 )?])
6488 }
6489 };
6490
6491 let settled =
6492 self.forwarding
6493 .complete_pending_relay(connection_id, frame.header.corr, outcome);
6494 let completion = match settled {
6495 Ok(completion) => completion,
6496 Err(err) => {
6497 self.refuse_to_end_module_connection_for_a_client(
6498 connection_id,
6499 frame.header.corr,
6500 err,
6501 )?;
6502 return Ok(secondary_error.into_iter().collect());
6503 }
6504 };
6505 if let Some(target) = completion.abandoned.as_ref() {
6506 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6507 }
6508 if !completion.settled {
6509 debug!(
6510 connection_id = connection_id.get(),
6511 corr = frame.header.corr,
6512 frame_type = ?frame.header.ty,
6513 "dropping late or unknown route.bind relay response"
6514 );
6515 }
6516 Ok(secondary_error.into_iter().collect())
6517 }
6518
6519 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6520 debug!(connection_id = connection_id.get(), "handling GOODBYE");
6521 self.cleanup_connection_with_end_reason(
6525 connection_id,
6526 RegistrationEndReason::ExplicitGoodbye,
6527 )
6528 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6529 Ok(Vec::new())
6530 }
6531}
6532
6533impl Default for ControlHandler {
6534 fn default() -> Self {
6535 Self::new(Arc::new(Registry::default()))
6536 }
6537}
6538
6539impl crate::supervise::SwapPromotionObserver for ControlHandler {
6540 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6541 self.apply_registration_capabilities(registration);
6542 }
6543}
6544
6545fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6546 CapabilityRequirementStatus {
6547 consumer: status.consumer,
6548 capability: status.capability,
6549 need: match status.need {
6550 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6551 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6552 },
6553 verdict: status.verdict.as_str().to_string(),
6554 episode_seq: status.episode_seq,
6555 config_satisfiable: status.config_satisfiable,
6556 runtime_available: status.runtime_available,
6557 detail: status.detail,
6558 }
6559}
6560
6561fn append_capability_problem_detail(
6562 detail: Option<String>,
6563 capability_detail: Option<String>,
6564) -> Option<String> {
6565 match (detail, capability_detail) {
6566 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
6567 (Some(detail), None) => Some(detail),
6568 (None, Some(capability_detail)) => Some(capability_detail),
6569 (None, None) => None,
6570 }
6571}
6572
6573fn subc_ops() -> Vec<String> {
6574 SUBC_CONTROL_OPS
6575 .iter()
6576 .map(|op| (*op).to_string())
6577 .collect()
6578}
6579
6580fn module_subc_ops() -> Vec<String> {
6581 SUBC_CONTROL_OPS
6582 .iter()
6583 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
6584 .map(|op| (*op).to_string())
6585 .collect()
6586}
6587
6588#[cfg(test)]
6589fn module_baseline_control_ops() -> Vec<String> {
6590 MODULE_BASELINE_CONTROL_OPS
6591 .iter()
6592 .map(|op| (*op).to_string())
6593 .collect()
6594}
6595
6596fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
6597 let mut seen = HashSet::new();
6598 let mut effective = Vec::new();
6599 for op in MODULE_BASELINE_CONTROL_OPS {
6600 if seen.insert((*op).to_string()) {
6601 effective.push((*op).to_string());
6602 }
6603 }
6604 for op in declared.unwrap_or_default() {
6605 if seen.insert(op.clone()) {
6606 effective.push(op);
6607 }
6608 }
6609 effective
6610}
6611
6612fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
6613 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
6614}
6615
6616fn target_module_id(target: &RouteTarget) -> &str {
6617 match target {
6618 RouteTarget::ToolProvider { module_id }
6619 | RouteTarget::ManagementSurface { module_id }
6620 | RouteTarget::InternalService { module_id, .. } => module_id,
6621 }
6622}
6623
6624fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
6625 roles.iter().any(|role| match (target, role) {
6626 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
6627 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
6628 (
6629 RouteTarget::InternalService { service_id, .. },
6630 ProviderRole::InternalService {
6631 service_id: provided,
6632 ..
6633 },
6634 ) => service_id == provided,
6635 _ => false,
6636 })
6637}
6638
6639fn is_routable_role(role: &ProviderRole) -> bool {
6640 matches!(
6641 role,
6642 ProviderRole::ToolProvider { .. }
6643 | ProviderRole::ManagementSurface { .. }
6644 | ProviderRole::InternalService { .. }
6645 )
6646}
6647
6648#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6649enum ControlRequestBodyError {
6650 UnknownOp,
6651 InvalidBody,
6652}
6653
6654#[derive(Debug, Deserialize)]
6655struct ControlOpProbe {
6656 op: String,
6657}
6658
6659const MODULE_PUSH_OPS: &[&str] = &["route.status"];
6662
6663fn is_known_module_push_op(body: &[u8]) -> bool {
6664 serde_json::from_slice::<ControlOpProbe>(body)
6665 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
6666 .unwrap_or(false)
6667}
6668
6669fn is_known_module_request_op(body: &[u8]) -> bool {
6670 serde_json::from_slice::<ControlOpProbe>(body)
6671 .map(|probe| is_module_to_subc_op(&probe.op))
6672 .unwrap_or(false)
6673}
6674
6675fn is_module_to_subc_op(op: &str) -> bool {
6676 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
6677}
6678
6679fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
6680 debug!(
6681 op = %op,
6682 connection_id = connection_id.get(),
6683 corr,
6684 "control dispatch"
6685 );
6686}
6687
6688fn log_slow_control_dispatch(
6689 dispatch_started_at: Option<StdInstant>,
6690 op: &'static str,
6691 connection_id: ConnectionId,
6692 corr: u64,
6693) {
6694 let Some(dispatch_started_at) = dispatch_started_at else {
6695 return;
6696 };
6697 let elapsed = dispatch_started_at.elapsed();
6698 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
6699 warn!(
6700 op = %op,
6701 connection_id = connection_id.get(),
6702 corr,
6703 elapsed_ms = elapsed.as_millis() as u64,
6704 "slow control dispatch"
6705 );
6706 }
6707}
6708
6709#[cfg(test)]
6711type TestCallerLookup =
6712 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
6713
6714fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
6716 match request {
6717 ClientControlRequest::SupervisorRestart { module_id, .. } => {
6718 Some(PrivilegedOperation::Restart { module_id })
6719 }
6720 ClientControlRequest::SupervisorSwap { module_id, .. } => {
6721 Some(PrivilegedOperation::Swap { module_id })
6722 }
6723 ClientControlRequest::SupervisorReload { module_id } => {
6724 Some(PrivilegedOperation::Reload { module_id })
6725 }
6726 ClientControlRequest::SupervisorRescan { preview } => {
6727 if *preview {
6728 None
6729 } else {
6730 Some(PrivilegedOperation::Rescan)
6731 }
6732 }
6733 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
6734 Some(PrivilegedOperation::SetEnabled {
6735 module_id,
6736 enabled: *enabled,
6737 })
6738 }
6739 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
6740 Some(PrivilegedOperation::ReleaseReserved { module_id })
6741 }
6742 ClientControlRequest::ServerDescribe {}
6743 | ClientControlRequest::CatalogList { .. }
6744 | ClientControlRequest::RouteOpen { .. }
6745 | ClientControlRequest::RoutePoll { .. }
6746 | ClientControlRequest::SupervisorList {}
6747 | ClientControlRequest::SupervisorHealth {}
6748 | ClientControlRequest::SupervisorHealthProbe { .. }
6749 | ClientControlRequest::SupervisorRoutes { .. }
6750 | ClientControlRequest::SupervisorProvenance { .. }
6751 | ClientControlRequest::SupervisorStderrTail { .. }
6752 | ClientControlRequest::SupervisorTerminals { .. }
6753 | ClientControlRequest::SupervisorSpawnSnapshot {}
6754 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
6755 }
6756}
6757
6758fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
6759 match request {
6760 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
6761 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
6762 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
6763 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
6764 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
6765 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
6766 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
6767 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
6768 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
6769 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
6770 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
6771 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
6772 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
6773 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
6774 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
6775 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
6776 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
6777 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
6778 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
6779 }
6780}
6781
6782fn principal_label(principal: &Principal) -> String {
6783 match principal {
6784 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
6785 Principal::Direct => "direct".to_string(),
6786 other => format!("{other:?}"),
6787 }
6788}
6789
6790fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
6791 match request {
6792 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
6793 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
6794 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
6795 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
6796 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
6797 }
6798}
6799
6800fn parse_client_control_request(
6801 body: &[u8],
6802) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
6803 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
6804 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6805 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
6806 ControlRequestBodyError::InvalidBody
6807 }
6808 Ok(_) => ControlRequestBodyError::UnknownOp,
6809 Err(_) => ControlRequestBodyError::InvalidBody,
6810 };
6811 (err, classification)
6812 })
6813}
6814
6815fn parse_module_control_request_from_module(
6816 body: &[u8],
6817) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
6818 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
6819 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6820 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
6821 Ok(_) => ControlRequestBodyError::UnknownOp,
6822 Err(_) => ControlRequestBodyError::InvalidBody,
6823 };
6824 (err, classification)
6825 })
6826}
6827
6828#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
6829enum ProviderRoleKind {
6830 ToolProvider,
6831 PipelineStage,
6832 ManagementSurface,
6833 InternalService,
6834}
6835
6836fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
6837 match role {
6838 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
6839 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
6840 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
6841 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
6842 }
6843}
6844
6845fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
6846 roles.iter().map(provider_role_kind).collect()
6847}
6848
6849const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
6852
6853#[derive(Debug, Default, PartialEq, Eq)]
6855struct ScopeOutcomeCounts {
6856 created: usize,
6857 replaced: usize,
6858 updated: usize,
6859 unchanged: usize,
6860 refused: usize,
6861}
6862
6863impl ScopeOutcomeCounts {
6864 fn of(results: &[ScopeRecordResult]) -> Self {
6865 let mut counts = Self::default();
6866 for result in results {
6867 let slot = match result.outcome {
6868 ScopeRecordOutcome::Created => &mut counts.created,
6869 ScopeRecordOutcome::Replaced => &mut counts.replaced,
6870 ScopeRecordOutcome::Updated => &mut counts.updated,
6871 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
6872 ScopeRecordOutcome::Refused => &mut counts.refused,
6873 };
6874 *slot += 1;
6875 }
6876 counts
6877 }
6878}
6879
6880#[cfg(test)]
6881mod scope_outcome_count_tests {
6882 use super::*;
6883
6884 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
6885 ScopeRecordResult {
6886 scope_ref: "r".to_string(),
6887 scope_epoch: 1,
6888 outcome,
6889 code: None,
6890 message: None,
6891 version: None,
6892 parent_state: None,
6893 }
6894 }
6895
6896 #[test]
6899 fn every_outcome_is_counted_in_its_own_field() {
6900 let results = [
6901 result(ScopeRecordOutcome::Created),
6902 result(ScopeRecordOutcome::Created),
6903 result(ScopeRecordOutcome::Replaced),
6904 result(ScopeRecordOutcome::Updated),
6905 result(ScopeRecordOutcome::Unchanged),
6906 result(ScopeRecordOutcome::Refused),
6907 result(ScopeRecordOutcome::Refused),
6908 result(ScopeRecordOutcome::Refused),
6909 ];
6910 assert_eq!(
6911 ScopeOutcomeCounts::of(&results),
6912 ScopeOutcomeCounts {
6913 created: 2,
6914 replaced: 1,
6915 updated: 1,
6916 unchanged: 1,
6917 refused: 3,
6918 }
6919 );
6920 }
6921}
6922
6923fn capability_census_trigger(
6927 old: Option<&CapabilityDeclarations>,
6928 new: Option<&CapabilityDeclarations>,
6929) -> bool {
6930 let old_provides = old
6931 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
6932 .unwrap_or_default();
6933 let old_denies = old
6934 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
6935 .unwrap_or_default();
6936 let new = new.cloned().unwrap_or(CapabilityDeclarations {
6937 provides: Vec::new(),
6938 requires: Vec::new(),
6939 must_never_reach: Vec::new(),
6940 });
6941
6942 new.provides
6943 .iter()
6944 .any(|capability| !old_provides.contains(capability))
6945 || new
6946 .must_never_reach
6947 .iter()
6948 .any(|capability| !old_denies.contains(capability))
6949}
6950
6951fn denied_capability<'a>(
6954 opening_manifest: &'a ModuleManifest,
6955 target_manifest: &ModuleManifest,
6956) -> Option<&'a str> {
6957 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
6958 let target_capabilities = target_manifest.capabilities.as_ref()?;
6959 opening_capabilities
6960 .must_never_reach
6961 .iter()
6962 .find(|denied| {
6963 target_capabilities
6964 .provides
6965 .iter()
6966 .any(|provided| provided == *denied)
6967 })
6968 .map(String::as_str)
6969}
6970
6971fn catalog_update_frozen_field_message(
6972 registered: &ModuleManifest,
6973 provides: &[ProviderRole],
6974) -> Option<String> {
6975 let old_has_provides = !registered.provides.is_empty();
6976 let new_has_provides = !provides.is_empty();
6977 if old_has_provides != new_has_provides {
6978 return Some(format!(
6979 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
6980 registered.module_id
6981 ));
6982 }
6983
6984 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
6985 return Some(format!(
6986 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
6987 registered.module_id
6988 ));
6989 }
6990
6991 let registered_concurrency = manifest_concurrency(registered);
6992 let mut candidate = registered.clone();
6993 candidate.provides = provides.to_vec();
6994 let candidate_concurrency = manifest_concurrency(&candidate);
6995 if candidate_concurrency != registered_concurrency {
6996 return Some(format!(
6997 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
6998 registered.module_id, registered_concurrency, candidate_concurrency
6999 ));
7000 }
7001
7002 None
7005}
7006
7007fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7008 manifest.provides.iter().any(is_routable_role)
7009}
7010
7011fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7017 manifest
7018 .provides
7019 .iter()
7020 .find_map(|provider| match provider {
7021 ProviderRole::ToolProvider { concurrency, .. }
7022 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7023 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7024 })
7025 .unwrap_or(Concurrency::ModuleManaged)
7026}
7027
7028fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7035 let has_management_surface = manifest
7036 .provides
7037 .iter()
7038 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7039 if !has_management_surface {
7040 return false;
7041 }
7042 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7043 return false;
7044 };
7045 let Some(provides) = raw
7046 .get("manifest")
7047 .and_then(|manifest| manifest.get("provides"))
7048 .and_then(serde_json::Value::as_array)
7049 else {
7050 return false;
7051 };
7052 provides.iter().any(|role| {
7057 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7058 && role.get("concurrency").is_none()
7059 })
7060}
7061
7062fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7063 if peer_version != PROTOCOL_VERSION {
7064 return Err(format!(
7065 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7066 ));
7067 }
7068 Ok(PROTOCOL_VERSION)
7069}
7070
7071fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7072 Frame::build_with_version(
7073 response_version(frame),
7074 FrameType::Pong,
7075 frame.header.flags,
7076 0,
7077 0,
7078 frame.header.corr,
7079 Vec::new(),
7080 )
7081 .map_err(RouterError::FrameBuild)
7082}
7083
7084pub(crate) fn control_error_frame(
7085 frame: &Frame,
7086 code: &'static str,
7087 message: impl Into<String>,
7088) -> Result<Frame, RouterError> {
7089 control_error_body_frame(
7090 frame,
7091 ErrorBody {
7092 code: code.to_string(),
7093 message: message.into(),
7094 detail: None,
7095 },
7096 )
7097}
7098
7099fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7100 let body = serde_json::to_vec(&error).map_err(|err| {
7101 RouterError::backend(
7102 0,
7103 frame.header.corr,
7104 format!("failed to encode control ERROR: {err}"),
7105 )
7106 })?;
7107
7108 Frame::build_with_version(
7109 response_version(frame),
7110 FrameType::Error,
7111 control_flags(),
7112 0,
7113 0,
7114 frame.header.corr,
7115 body,
7116 )
7117 .map_err(RouterError::FrameBuild)
7118}
7119
7120fn control_response_body_frame<T: Serialize>(
7121 frame: &Frame,
7122 reply: &T,
7123 label: &'static str,
7124) -> Result<Frame, RouterError> {
7125 let body = serde_json::to_vec(reply).map_err(|err| {
7126 RouterError::backend(
7127 0,
7128 frame.header.corr,
7129 format!("failed to encode {label}: {err}"),
7130 )
7131 })?;
7132
7133 Frame::build_with_version(
7134 response_version(frame),
7135 FrameType::Response,
7136 control_flags(),
7137 0,
7138 0,
7139 frame.header.corr,
7140 body,
7141 )
7142 .map_err(RouterError::FrameBuild)
7143}
7144
7145fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7175 match err {
7176 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7177 ForwardingError::NoModuleConnection => "target_unavailable",
7178 ForwardingError::ModuleReloading { .. } => "module_reloading",
7179 ForwardingError::ClientRouteChannelExhausted { .. }
7180 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7181 ForwardingError::StaleModuleEndpoint
7182 | ForwardingError::UnknownReservation { .. }
7183 | ForwardingError::ConnectionClosing { .. }
7184 | ForwardingError::ClientEgressClosed { .. }
7185 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7186 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7189 ForwardingError::RelayCorrelationExhausted
7190 | ForwardingError::RouteOpenBuild(_)
7191 | ForwardingError::Poisoned => "forwarding_error",
7192 }
7193}
7194
7195fn response_version(frame: &Frame) -> u8 {
7196 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7197 frame.header.ver
7198 } else {
7199 PROTOCOL_VERSION
7200 }
7201}
7202
7203fn control_flags() -> Flags {
7204 Flags::new(false, Priority::Passive, false)
7205}
7206
7207fn send_goodbye_target_best_effort(
7212 counters: &DaemonCounters,
7213 target: &GoodbyeTarget,
7214 context: &'static str,
7215) {
7216 let Ok(frame) = Frame::build_with_version(
7217 target.negotiated_ver,
7218 FrameType::Goodbye,
7219 control_flags(),
7220 target.channel,
7221 target.epoch,
7222 0,
7223 Vec::new(),
7224 ) else {
7225 return;
7226 };
7227 crate::forwarding::send_module_route_goodbye(
7228 counters,
7229 &target.sink,
7230 frame,
7231 target.module_id.as_deref(),
7232 context,
7233 );
7234}
7235
7236pub(crate) fn send_route_control_pushes(
7237 forwarding: &ForwardingTable,
7238 routes: Vec<EndpointRoute>,
7239 push: ClientControlPush,
7240) {
7241 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7242 for route in routes {
7243 let target = route.goodbye_target;
7244 if let Some((existing, channels)) = targets
7245 .iter_mut()
7246 .find(|(existing, _)| existing.connection_id == target.connection_id)
7247 {
7248 debug_assert_eq!(
7249 existing.negotiated_ver, target.negotiated_ver,
7250 "one connection cannot negotiate multiple frame versions"
7251 );
7252 if !channels.contains(&target.channel) {
7253 channels.push(target.channel);
7254 }
7255 continue;
7256 }
7257 let channel = target.channel;
7258 targets.push((target, vec![channel]));
7259 }
7260 for (target, mut channels) in targets {
7261 channels.sort_unstable();
7262 let mut push = push.clone();
7263 match &mut push {
7264 ClientControlPush::RouteClosing {
7265 channels: covered, ..
7266 }
7267 | ClientControlPush::RouteClosed {
7268 channels: covered, ..
7269 } => *covered = channels,
7270 }
7271 let body = match serde_json::to_vec(&push) {
7272 Ok(body) => body,
7273 Err(err) => {
7274 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7275 continue;
7276 }
7277 };
7278 let frame = match Frame::build_with_version(
7279 target.negotiated_ver,
7280 FrameType::Push,
7281 control_flags(),
7282 0,
7283 0,
7284 0,
7285 body.clone(),
7286 ) {
7287 Ok(frame) => frame,
7288 Err(err) => {
7289 warn!(
7290 route_channel = target.channel,
7291 error = %err,
7292 "failed to build route lifecycle control PUSH frame"
7293 );
7294 continue;
7295 }
7296 };
7297 if let Err(err) = target.sink.try_send(frame) {
7298 if target.close_on_delivery_failure() {
7299 warn!(
7300 target_connection_id = target.connection_id.get(),
7301 route_channel = target.channel,
7302 error = %err,
7303 "route lifecycle control PUSH was not delivered to client; closing target connection"
7304 );
7305 let _ = forwarding.escalate_client_delivery_failure(
7306 target.connection_id,
7307 target.channel,
7308 target.epoch,
7309 CloseReason::new(
7310 "route_lifecycle_push_delivery_failed",
7311 format!(
7312 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7313 target.channel
7314 ),
7315 ),
7316 crate::forwarding::UndeliveredFrame {
7317 module_id: target.module_id.as_deref(),
7318 sink: &target.sink,
7319 },
7320 );
7321 }
7322 }
7323 }
7324}
7325
7326#[cfg(test)]
7327mod tests {
7328 mod activity_read {
7329 include!("activity_read_tests.rs");
7330 }
7331
7332 mod lifecycle_activity {
7333 include!("control_lifecycle_tests.rs");
7334 }
7335 #[cfg(unix)]
7336 #[tokio::test]
7337 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
7338 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
7339 let path = dir.join("subc.jsonc");
7340 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
7341 "program":"/bin/sleep","args":["60"],"protocol":"none",
7342 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
7343 }}}).to_string()).unwrap();
7344 let mut configured = crate::daemon_config::load(&path)
7345 .unwrap()
7346 .unwrap()
7347 .modules
7348 .pop()
7349 .unwrap();
7350 let registry = std::sync::Arc::new(crate::Registry::default());
7351 let handle = crate::SupervisorHandle::new();
7352 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
7353 .with_handle(handle.clone());
7354 let module = supervisor
7355 .supervise_configured_with_health(
7356 configured.module_spec(),
7357 true,
7358 configured.health.clone(),
7359 None,
7360 configured.restart,
7361 )
7362 .unwrap();
7363 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
7364 let before = module.status().unwrap().pid;
7365 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
7366 configured.health.cadence = std::time::Duration::from_secs(3600);
7367 let health_only = handler
7368 .reconcile_supervised_modules(
7369 &supervisor,
7370 vec![configured.clone()],
7371 false,
7372 Principal::Direct,
7373 )
7374 .await
7375 .unwrap();
7376 assert!(
7377 health_only.changed_pending_reload.is_empty(),
7378 "health policy is already applied live"
7379 );
7380 assert_eq!(module.status().unwrap().pid, before);
7381 assert_eq!(
7382 module.configuration().unwrap().1.http,
7383 configured.health.http
7384 );
7385 configured.args = vec!["61".into()];
7386 let launch = handler
7387 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
7388 .await
7389 .unwrap();
7390 assert_eq!(launch.changed_pending_reload, ["stock"]);
7391 assert_eq!(
7392 module.status().unwrap().pid,
7393 before,
7394 "a launch edit is stored until reload"
7395 );
7396 module.drain().await.unwrap();
7397 }
7398 use cortexkit_test_support::ScratchDir;
7399 use std::{
7400 collections::BTreeMap,
7401 fmt,
7402 path::PathBuf,
7403 sync::{Arc, Mutex},
7404 time::Duration,
7405 };
7406
7407 use serde_json::{json, Value};
7408 use subc_protocol::{
7409 manifest::{
7410 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
7411 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
7412 },
7413 session::HealthStatus,
7414 FrameType,
7415 };
7416
7417 use super::*;
7418 use crate::{
7419 forwarding::{DataRoute, DataRouteState},
7420 registry::ChannelState,
7421 router::FrameSink,
7422 stderr_tail::DEFAULT_MAX_LINE_BYTES,
7423 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
7424 RouteCtx, Router,
7425 };
7426 use tokio::{
7427 sync::mpsc,
7428 time::{sleep, Instant},
7429 };
7430 use tracing::{
7431 field::{Field, Visit},
7432 Event, Subscriber,
7433 };
7434 use tracing_subscriber::{layer::Context, prelude::*, Layer};
7435
7436 fn fake_aft_stub_path() -> PathBuf {
7455 let mut path = std::env::current_exe().expect("current_exe available in tests");
7456 path.pop(); path.pop(); path.push(if cfg!(windows) {
7459 "fake-aft-stub.exe"
7460 } else {
7461 "fake-aft-stub"
7462 });
7463 assert!(
7464 path.exists(),
7465 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
7466 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
7467 path.display()
7468 );
7469 path
7470 }
7471
7472 fn client_retries(code: &str) -> bool {
7477 subc_protocol::error_codes::is_retryable_route_open(code)
7478 }
7479
7480 #[test]
7491 fn retryability_of_forwarding_codes_matches_the_failure() {
7492 let transient = [
7495 ForwardingError::NoModuleConnection,
7496 ForwardingError::ModuleReloading {
7497 module_id: "m".into(),
7498 },
7499 ForwardingError::StaleModuleEndpoint,
7500 ForwardingError::UnknownReservation {
7501 client_channel: 1,
7502 module_channel: 1,
7503 },
7504 ForwardingError::ConnectionClosing {
7505 connection_id: ConnectionId::new(1),
7506 },
7507 ForwardingError::ClientEgressClosed {
7508 connection_id: ConnectionId::new(1),
7509 },
7510 ForwardingError::ModuleEgressUnavailable {
7511 connection_id: ConnectionId::new(1),
7512 },
7513 ];
7514 for err in transient {
7515 let code = forwarding_error_code(&err);
7516 assert!(
7517 client_retries(code),
7518 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
7519 );
7520 }
7521
7522 let permanent = [
7527 ForwardingError::ConnectionRoleConflict {
7528 connection_id: ConnectionId::new(1),
7529 },
7530 ForwardingError::ClientRouteChannelExhausted {
7531 connection_id: ConnectionId::new(1),
7532 },
7533 ForwardingError::ModuleRouteChannelExhausted {
7534 endpoint: ModuleEndpointId {
7535 connection_id: ConnectionId::new(1),
7536 generation: 1,
7537 },
7538 },
7539 ForwardingError::RelayCorrelationExhausted,
7540 ForwardingError::RouteOpenBuild("x".into()),
7541 ForwardingError::Poisoned,
7542 ];
7543 for err in permanent {
7544 let code = forwarding_error_code(&err);
7545 assert!(
7546 !client_retries(code),
7547 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
7548 );
7549 }
7550 }
7551
7552 #[tokio::test]
7580 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
7581 let handler = ControlHandler::default();
7582 let frame =
7583 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
7584
7585 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
7589 assert_eq!(
7590 stamped,
7591 Principal::Direct,
7592 "a caller that proved nothing must not be stamped as a supervised module"
7593 );
7594
7595 let forged = handler
7600 .route_open_principal(
7601 &frame,
7602 Some(ConsumerIdentity {
7603 module_id: "aft".to_string(),
7604 launch_nonce: "not-a-real-nonce".to_string(),
7605 }),
7606 )
7607 .unwrap();
7608 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
7609 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
7610 }
7611
7612 #[test]
7629 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
7630 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
7631 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
7632 let ClientControlRequest::RouteOpen {
7633 consumer_identity, ..
7634 } = parsed
7635 else {
7636 panic!("route.open body must parse as RouteOpen");
7637 };
7638 assert_eq!(
7639 consumer_identity,
7640 Some(ConsumerIdentity {
7641 module_id: "aft".to_string(),
7642 launch_nonce: "n".to_string(),
7643 }),
7644 "the wire field name must reach the value route_open_principal reads"
7645 );
7646 }
7647
7648 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
7649 ModuleManifest::builder(module_id, "0.1.0")
7650 .protocol_ver(protocol_ver)
7651 .provides(vec![ProviderRole::ToolProvider {
7652 tools: vec![Tool {
7653 name: "read".to_string(),
7654 description: None,
7655 execution_mode: ExecutionMode::Pure,
7656 schema: json!({"type": "object"}),
7657 }],
7658 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
7659 concurrency: Concurrency::ModuleManaged,
7660 emits_push: true,
7661 sub_supervises: true,
7662 }])
7663 .build()
7664 }
7665
7666 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
7667 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
7668 }
7669
7670 fn hello_frame_with_control_ops(
7671 module_id: &str,
7672 protocol_ver: u8,
7673 corr: u64,
7674 control_ops: Option<Vec<String>>,
7675 ) -> Frame {
7676 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
7677 }
7678
7679 fn hello_frame_with_nonce(
7680 module_id: &str,
7681 protocol_ver: u8,
7682 corr: u64,
7683 launch_nonce: Option<&str>,
7684 ) -> Frame {
7685 hello_frame_full(
7686 module_id,
7687 protocol_ver,
7688 corr,
7689 None,
7690 launch_nonce.map(ToOwned::to_owned),
7691 )
7692 }
7693
7694 fn hello_frame_full(
7695 module_id: &str,
7696 protocol_ver: u8,
7697 corr: u64,
7698 control_ops: Option<Vec<String>>,
7699 launch_nonce: Option<String>,
7700 ) -> Frame {
7701 let body = serde_json::to_vec(&ModuleHelloBody {
7702 manifest: manifest(module_id, protocol_ver),
7703 protocol_ver,
7704 control_ops,
7705 launch_nonce,
7706 })
7707 .unwrap();
7708 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7709 }
7710
7711 fn non_routable_hello_frame_with_control_ops(
7712 module_id: &str,
7713 corr: u64,
7714 control_ops: Option<Vec<String>>,
7715 ) -> Frame {
7716 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
7717 manifest.provides.clear();
7718 let body = serde_json::to_vec(&ModuleHelloBody {
7719 manifest,
7720 protocol_ver: PROTOCOL_VERSION,
7721 control_ops,
7722 launch_nonce: None,
7723 })
7724 .unwrap();
7725 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7726 }
7727
7728 fn capability_grammar_hello_frame(
7729 capabilities: Value,
7730 runtime_computed: Option<Value>,
7731 corr: u64,
7732 ) -> Frame {
7733 let mut body = serde_json::to_value(ModuleHelloBody {
7734 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
7735 protocol_ver: PROTOCOL_VERSION,
7736 control_ops: None,
7737 launch_nonce: None,
7738 })
7739 .expect("HELLO body serializes");
7740 body["manifest"]["capabilities"] = capabilities;
7741 if let Some(runtime_computed) = runtime_computed {
7742 body["runtime_computed"] = runtime_computed;
7743 }
7744 Frame::build(
7745 FrameType::Hello,
7746 control_flags(),
7747 0,
7748 0,
7749 corr,
7750 serde_json::to_vec(&body).expect("HELLO body reserializes"),
7751 )
7752 .expect("HELLO frame builds")
7753 }
7754
7755 fn channel_request(channel: u16, corr: u64) -> Frame {
7756 Frame::build(
7757 FrameType::Request,
7758 Flags::new(true, Priority::Interactive, false),
7759 channel,
7760 0,
7761 corr,
7762 b"opaque".to_vec(),
7763 )
7764 .unwrap()
7765 }
7766
7767 fn route_ctx(
7768 connection_id: ConnectionId,
7769 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
7770 let (tx, rx) = mpsc::channel(8);
7771 (
7772 RouteCtx {
7773 connection_id,
7774 egress: FrameSink::new(tx),
7775 },
7776 rx,
7777 )
7778 }
7779
7780 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
7781 serde_json::from_slice(&frame.body).unwrap()
7782 }
7783
7784 async fn hello_via_sink(
7789 handler: &ControlHandler,
7790 ctx: &RouteCtx,
7791 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7792 hello: Frame,
7793 ) -> Frame {
7794 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
7795 assert!(
7796 replies.is_empty(),
7797 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
7798 );
7799 let ack = rx
7800 .try_recv()
7801 .expect("HELLO_ACK is queued on the module sink")
7802 .frame;
7803 assert_eq!(ack.header.ty, FrameType::HelloAck);
7804 ack
7805 }
7806
7807 fn parse_error(frame: &Frame) -> Value {
7808 serde_json::from_slice(&frame.body).unwrap()
7809 }
7810
7811 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
7812 serde_json::from_slice(&frame.body).unwrap()
7813 }
7814
7815 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
7816 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
7817 route_channel,
7818 route_epoch: 0,
7819 kind,
7820 })
7821 .unwrap();
7822 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7823 }
7824
7825 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
7826 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
7827 module_id: module_id.to_string(),
7828 })
7829 .unwrap();
7830 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7831 }
7832
7833 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
7834 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
7835 }
7836
7837 fn route_open_frame_with_consumer_capabilities(
7838 corr: u64,
7839 module_id: &str,
7840 project_root: ScratchDir,
7841 consumer_capabilities: Option<Vec<String>>,
7842 ) -> Frame {
7843 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7844 target: RouteTarget::ToolProvider {
7845 module_id: module_id.to_string(),
7846 },
7847 identity: BindIdentity::new(
7848 project_root.path().to_path_buf(),
7849 "unit".to_string(),
7850 "session".to_string(),
7851 ),
7852 consumer_identity: None,
7853 consumer_capabilities,
7854 role_versions: None,
7855 admission_facts: None,
7856 scope: None,
7857 })
7858 .unwrap();
7859 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7860 }
7861
7862 fn route_open_frame_with_role_versions(
7863 corr: u64,
7864 module_id: &str,
7865 project_root: ScratchDir,
7866 role_versions: Option<BTreeMap<String, String>>,
7867 ) -> Frame {
7868 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7869 target: RouteTarget::ToolProvider {
7870 module_id: module_id.to_string(),
7871 },
7872 identity: BindIdentity::new(
7873 project_root.path().to_path_buf(),
7874 "unit".to_string(),
7875 format!("session-{corr}"),
7876 ),
7877 consumer_identity: None,
7878 consumer_capabilities: None,
7879 role_versions,
7880 admission_facts: None,
7881 scope: None,
7882 })
7883 .unwrap();
7884 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7885 }
7886
7887 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
7888 entries
7889 .iter()
7890 .map(|(role, version)| (role.to_string(), version.to_string()))
7891 .collect()
7892 }
7893
7894 fn route_open_frame_with_admission_facts(
7895 corr: u64,
7896 module_id: &str,
7897 project_root: ScratchDir,
7898 consumer_identity: Option<subc_control::ConsumerIdentity>,
7899 facts: Option<Value>,
7900 ) -> Frame {
7901 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7902 target: RouteTarget::ToolProvider {
7903 module_id: module_id.to_string(),
7904 },
7905 identity: BindIdentity::new(
7906 project_root.path().to_path_buf(),
7907 "unit".to_string(),
7908 format!("session-{corr}"),
7909 ),
7910 consumer_identity,
7911 consumer_capabilities: None,
7912 role_versions: None,
7913 admission_facts: facts,
7914 scope: None,
7915 })
7916 .unwrap();
7917 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7918 }
7919
7920 #[derive(Clone, Default)]
7921 struct EventCapture {
7922 events: Arc<Mutex<Vec<CapturedEvent>>>,
7923 }
7924
7925 #[derive(Clone, Debug)]
7926 struct CapturedEvent {
7927 target: String,
7928 level: tracing::Level,
7929 fields: BTreeMap<String, String>,
7930 }
7931
7932 impl EventCapture {
7933 fn events(&self) -> Vec<CapturedEvent> {
7934 self.events.lock().unwrap().clone()
7935 }
7936 }
7937
7938 impl<S> Layer<S> for EventCapture
7939 where
7940 S: Subscriber,
7941 {
7942 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
7943 let mut visitor = EventFieldVisitor::default();
7944 event.record(&mut visitor);
7945 self.events.lock().unwrap().push(CapturedEvent {
7946 target: event.metadata().target().to_string(),
7947 level: *event.metadata().level(),
7948 fields: visitor.fields,
7949 });
7950 }
7951 }
7952
7953 #[derive(Default)]
7954 struct EventFieldVisitor {
7955 fields: BTreeMap<String, String>,
7956 }
7957
7958 impl Visit for EventFieldVisitor {
7959 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
7960 self.fields
7961 .insert(field.name().to_string(), format!("{value:?}"));
7962 }
7963 }
7964
7965 fn health_response(corr: u64, status: HealthStatus) -> Frame {
7966 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
7967 status,
7968 detail: Some("warming".to_string()),
7969 metrics: Some(json!({"queue_depth": 3})),
7970 })
7971 .unwrap();
7972 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7973 }
7974
7975 fn route_bind_ack(corr: u64) -> Frame {
7976 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
7977 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7978 }
7979
7980 fn unique_project_root(label: &str) -> ScratchDir {
7981 ScratchDir::new(label)
7982 }
7983
7984 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
7985 match parse_route_poll(frame) {
7986 ClientControlResponse::RoutePoll {
7987 status: None,
7988 live: Some(live),
7989 ..
7990 } => assert_eq!(live, expected_live),
7991 other => panic!("unexpected route.poll response: {other:?}"),
7992 }
7993 }
7994
7995 fn bind_liveness_route(
7996 registry: &Registry,
7997 forwarding: &ForwardingTable,
7998 module_id: &str,
7999 ) -> (RouteCtx, u16, u32) {
8000 let module_connection = ConnectionId::new(101);
8001 let client_connection = ConnectionId::new(202);
8002 let registration = registry
8003 .register_with_control_ops(
8004 manifest(module_id, PROTOCOL_VERSION),
8005 PROTOCOL_VERSION,
8006 module_connection,
8007 module_baseline_control_ops(),
8008 )
8009 .unwrap();
8010 let (module_tx, _module_rx) = mpsc::channel(8);
8011 let endpoint = forwarding
8012 .register_module_connection(
8013 module_connection,
8014 module_id.to_string(),
8015 PROTOCOL_VERSION,
8016 manifest_concurrency(®istration.manifest),
8017 FrameSink::new(module_tx),
8018 )
8019 .unwrap();
8020 let (client_ctx, _client_rx) = route_ctx(client_connection);
8021 let pending = forwarding
8022 .begin_route_bind_relay_for_test(
8023 client_connection,
8024 client_ctx.egress.clone(),
8025 1,
8026 module_id,
8027 )
8028 .unwrap();
8029 assert_eq!(pending.endpoint, endpoint);
8030 let route_channel = pending.client_channel;
8031 let route_epoch = pending.client_epoch;
8032 forwarding
8033 .complete_pending_relay(
8034 module_connection,
8035 pending.corr,
8036 RouteBindRelayOutcome::Accepted,
8037 )
8038 .unwrap();
8039 (client_ctx, route_channel, route_epoch)
8040 }
8041
8042 struct FakeProcessLiveness {
8043 live: Option<bool>,
8044 }
8045
8046 impl ModuleProcessLiveness for FakeProcessLiveness {
8047 fn process_live(&self, _module_id: &str) -> Option<bool> {
8048 self.live
8049 }
8050 }
8051
8052 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8053 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8054 {
8055 let registry = Arc::new(Registry::default());
8056 let supervisor_handle = SupervisorHandle::new();
8057 let supervisor = Supervisor::new_for_test(
8058 Arc::clone(®istry),
8059 RestartPolicy::new(1, Duration::from_millis(10)),
8060 )
8061 .with_handle(supervisor_handle.clone());
8062 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8063 let module = supervisor
8064 .spawn(ModuleSpec {
8065 module_id: "stderr-tail-wire".to_string(),
8066 program: fake_aft_stub_path(),
8067 args: Vec::new(),
8068 env: vec![
8069 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8070 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8071 ],
8072 reserved: false,
8073 reserved_prefixes: Vec::new(),
8074 protocol: ModuleProtocol::Subc,
8075 overlap: Default::default(),
8076 })
8077 .unwrap();
8078
8079 let deadline = Instant::now() + Duration::from_secs(5);
8080 loop {
8081 let tail = module.stderr_tail(None, None);
8082 if tail
8083 .entries
8084 .iter()
8085 .any(|entry| matches!(entry, TailEntry::ProcessStart))
8086 && tail.entries.iter().any(|entry| {
8087 matches!(
8088 entry,
8089 TailEntry::Line {
8090 truncated: true,
8091 ..
8092 }
8093 )
8094 })
8095 {
8096 break;
8097 }
8098 assert!(
8099 Instant::now() < deadline,
8100 "module did not produce a truncated line and restart boundary: {tail:?}"
8101 );
8102 sleep(Duration::from_millis(10)).await;
8103 }
8104
8105 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8106 let request = ClientControlRequest::SupervisorStderrTail {
8107 module_id: "stderr-tail-wire".to_string(),
8108 max_lines: None,
8109 max_bytes: None,
8110 };
8111 let frame = Frame::build(
8112 FrameType::Request,
8113 control_flags(),
8114 0,
8115 0,
8116 1,
8117 serde_json::to_vec(&request).unwrap(),
8118 )
8119 .unwrap();
8120 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8121 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8122 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8123 serde_json::from_slice(&responses[0].body).unwrap()
8124 else {
8125 panic!("expected supervisor.stderr_tail response");
8126 };
8127
8128 assert!(
8129 tail.entries
8130 .iter()
8131 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8132 "the control response lost the restart boundary"
8133 );
8134 let Some(StderrTailEntry::Line {
8135 text,
8136 truncated,
8137 at_ms,
8138 }) = tail.entries.iter().find(|entry| {
8139 matches!(
8140 entry,
8141 StderrTailEntry::Line {
8142 truncated: true,
8143 ..
8144 }
8145 )
8146 })
8147 else {
8148 panic!("the control response lost the truncated line");
8149 };
8150 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8151 assert!(*truncated);
8152 assert!(
8153 at_ms.is_some(),
8154 "the control response lost the line's capture time"
8155 );
8156 }
8157
8158 #[tokio::test(flavor = "current_thread")]
8163 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8164 let dir = ScratchDir::new("terminals-off-worker");
8165 let journal_path = dir.join("terminals.jsonl");
8166 let registry = Arc::new(Registry::default());
8167 let supervisor_handle = SupervisorHandle::new();
8168 let supervisor =
8169 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8170 .with_handle(supervisor_handle.clone())
8171 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8172 let module = supervisor
8173 .spawn(ModuleSpec {
8174 module_id: "terminal-off-worker".to_string(),
8175 program: fake_aft_stub_path(),
8176 args: Vec::new(),
8177 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8178 reserved: false,
8179 reserved_prefixes: Vec::new(),
8180 protocol: ModuleProtocol::Subc,
8181 overlap: Default::default(),
8182 })
8183 .unwrap();
8184 let deadline = Instant::now() + Duration::from_secs(5);
8185 while module.terminal_history().entries.len() != 2 {
8186 assert!(Instant::now() < deadline, "module did not record two exits");
8187 sleep(Duration::from_millis(10)).await;
8188 }
8189
8190 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8191 let handler =
8192 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
8193 let frame = Frame::build(
8194 FrameType::Request,
8195 control_flags(),
8196 0,
8197 0,
8198 1,
8199 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8200 module_id: "terminal-off-worker".to_string(),
8201 })
8202 .unwrap(),
8203 )
8204 .unwrap();
8205 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8206 let spawned_at = std::time::Instant::now();
8207 let read = tokio::spawn({
8208 let handler = Arc::clone(&handler);
8209 async move { handler.handle_control_frame(&ctx, frame).await }
8210 });
8211 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8214 .await
8215 .unwrap()
8216 .expect("the history read reached its pause");
8217 let elapsed = spawned_at.elapsed();
8218 assert!(
8219 elapsed < Duration::from_secs(2) && !read.is_finished(),
8220 "this task could not run while the history read was paused \
8221 (resumed after {elapsed:?}, read finished: {})",
8222 read.is_finished()
8223 );
8224
8225 drop(release);
8226 let responses = read.await.unwrap().unwrap();
8227 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8228 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8229 panic!("expected supervisor.terminals response");
8230 };
8231 assert_eq!(terminals.entries.len(), 2);
8232 assert_eq!(terminals.journal_skipped_lines, 0);
8233 assert_eq!(terminals.journal_read_errors, 0);
8234 }
8235
8236 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8237 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8238 let registry = Arc::new(Registry::default());
8239 let supervisor_handle = SupervisorHandle::new();
8240 let supervisor =
8241 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8242 .with_handle(supervisor_handle.clone());
8243 let module = supervisor
8244 .spawn(ModuleSpec {
8245 module_id: "terminal-golden".to_string(),
8246 program: fake_aft_stub_path(),
8247 args: Vec::new(),
8248 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8249 reserved: false,
8250 reserved_prefixes: Vec::new(),
8251 protocol: ModuleProtocol::Subc,
8252 overlap: Default::default(),
8253 })
8254 .unwrap();
8255
8256 let deadline = Instant::now() + Duration::from_secs(5);
8257 while module.terminal_history().entries.len() != 2 {
8258 assert!(
8259 Instant::now() < deadline,
8260 "module did not retain two terminal exits: {:?}",
8261 module.terminal_history()
8262 );
8263 sleep(Duration::from_millis(10)).await;
8264 }
8265
8266 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8267 let request = ClientControlRequest::SupervisorTerminals {
8268 module_id: "terminal-golden".to_string(),
8269 };
8270 let frame = Frame::build(
8271 FrameType::Request,
8272 control_flags(),
8273 0,
8274 0,
8275 1,
8276 serde_json::to_vec(&request).unwrap(),
8277 )
8278 .unwrap();
8279 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8280 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8281 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8282 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8283 panic!("expected supervisor.terminals response");
8284 };
8285 assert_eq!(terminals.entries.len(), 2);
8286 assert_eq!(terminals.dropped, 0);
8287
8288 let mut rendered = serde_json::to_value(response).unwrap();
8289 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
8292 for (index, entry) in rendered["entries"]
8293 .as_array_mut()
8294 .expect("terminal response entries array")
8295 .iter_mut()
8296 .enumerate()
8297 {
8298 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
8299 }
8300
8301 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8302 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
8303 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
8304 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8305 std::fs::write(&golden_path, &serialized).unwrap();
8306 }
8307 let expected: Value =
8308 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
8309 assert_eq!(rendered, expected);
8310 }
8311
8312 #[test]
8313 fn hello_registers_manifest_and_returns_ack() {
8314 let registry = Arc::new(Registry::default());
8315 let handler = ControlHandler::new(Arc::clone(®istry));
8316 let conn = ConnectionId::new(1);
8317
8318 let responses = handler
8319 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
8320 .unwrap();
8321
8322 assert_eq!(responses.len(), 1);
8323 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8324 assert_eq!(responses[0].header.channel, 0);
8325 assert_eq!(responses[0].header.corr, 7);
8326 let ack = parse_ack(&responses[0]);
8327 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8328 assert!(ack
8329 .subc_capabilities
8330 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
8331 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
8332 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
8333 assert!(ack
8334 .subc_ops
8335 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
8336 assert!(ack
8337 .subc_ops
8338 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
8339
8340 let registration = registry.get_module("aft").unwrap().unwrap();
8341 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
8342 assert_eq!(registration.state, ChannelState::Active);
8343 assert_eq!(registration.connection_id, conn);
8344 assert_eq!(registration.control_ops, module_baseline_control_ops());
8345 }
8346
8347 #[test]
8348 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
8349 let invalid_identifiers = [
8350 ("case_change", "credentials-Provider/v1"),
8351 ("leading_zero", "credentials-provider/v01"),
8352 ("trailing_hyphen", "credentials-provider-/v1"),
8353 ("consecutive_hyphens", "credentials--provider/v1"),
8354 ("uppercase", "Credentials-provider/v1"),
8355 ("missing_v", "credentials-provider/1"),
8356 ("whitespace", "credentials provider/v1"),
8357 ("zero_version", "credentials-provider/v0"),
8358 ("out_of_range_version", "credentials-provider/v4294967296"),
8359 (
8360 "overlength_name",
8361 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
8362 ),
8363 ];
8364 let mut cases = invalid_identifiers
8365 .into_iter()
8366 .map(|(name, identifier)| {
8367 (
8368 format!("identifier_{name}"),
8369 "capabilities.provides[0]".to_string(),
8370 identifier.to_string(),
8371 json!({ "provides": [identifier] }),
8372 None,
8373 )
8374 })
8375 .collect::<Vec<_>>();
8376 cases.extend([
8377 (
8378 "unknown_need".to_string(),
8379 "capabilities.requires[0].need".to_string(),
8380 "deferred".to_string(),
8381 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
8382 None,
8383 ),
8384 (
8385 "duplicate_provides".to_string(),
8386 "capabilities.provides[1]".to_string(),
8387 "credentials-provider/v1".to_string(),
8388 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
8389 None,
8390 ),
8391 (
8392 "duplicate_must_never_reach".to_string(),
8393 "capabilities.must_never_reach[1]".to_string(),
8394 "credentials-provider/v1".to_string(),
8395 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
8396 None,
8397 ),
8398 (
8399 "duplicate_requires_same_need".to_string(),
8400 "capabilities.requires[1]".to_string(),
8401 "credentials-provider/v1".to_string(),
8402 json!({ "requires": [
8403 { "capability": "credentials-provider/v1", "need": "required" },
8404 { "capability": "credentials-provider/v1", "need": "required" }
8405 ] }),
8406 None,
8407 ),
8408 (
8409 "duplicate_requires_conflicting_need".to_string(),
8410 "capabilities.requires[1]".to_string(),
8411 "credentials-provider/v1".to_string(),
8412 json!({ "requires": [
8413 { "capability": "credentials-provider/v1", "need": "required" },
8414 { "capability": "credentials-provider/v1", "need": "optional" }
8415 ] }),
8416 None,
8417 ),
8418 (
8419 "capabilities_root_pointer".to_string(),
8420 "runtime_computed[0]".to_string(),
8421 "/capabilities".to_string(),
8422 json!({}),
8423 Some(json!(["/capabilities"])),
8424 ),
8425 (
8426 "capabilities_descendant_pointer".to_string(),
8427 "runtime_computed[0]".to_string(),
8428 "/capabilities/provides".to_string(),
8429 json!({}),
8430 Some(json!(["/capabilities/provides"])),
8431 ),
8432 (
8433 "malformed_pointer_without_leading_slash".to_string(),
8434 "runtime_computed[0]".to_string(),
8435 "capabilities".to_string(),
8436 json!({}),
8437 Some(json!(["capabilities"])),
8438 ),
8439 (
8440 "malformed_pointer_escape".to_string(),
8441 "runtime_computed[0]".to_string(),
8442 "/roles/~2/tools".to_string(),
8443 json!({}),
8444 Some(json!(["/roles/~2/tools"])),
8445 ),
8446 (
8447 "unknown_capabilities_field".to_string(),
8448 "capabilities.future".to_string(),
8449 "<array>".to_string(),
8450 json!({ "future": [] }),
8451 None,
8452 ),
8453 ]);
8454
8455 for (index, (name, field, value, capabilities, runtime_computed)) in
8456 cases.into_iter().enumerate()
8457 {
8458 let registry = Arc::new(Registry::default());
8459 let handler = ControlHandler::new(Arc::clone(®istry));
8460 let response = handler
8461 .handle_control(
8462 ConnectionId::new((index + 1) as u64),
8463 capability_grammar_hello_frame(
8464 capabilities,
8465 runtime_computed,
8466 index as u64 + 1,
8467 ),
8468 )
8469 .expect("invalid HELLO returns a refusal");
8470
8471 assert_eq!(response.len(), 1, "{name} must emit one refusal");
8472 let error = parse_error(&response[0]);
8473 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
8474 let message = error["message"]
8475 .as_str()
8476 .expect("error message is a string");
8477 assert!(
8478 message.contains(&field),
8479 "{name}: field missing from {message}"
8480 );
8481 assert!(
8482 message.contains(&value),
8483 "{name}: value missing from {message}"
8484 );
8485 assert_eq!(
8486 registry
8487 .active_registration_count()
8488 .expect("registry reads"),
8489 0,
8490 "{name}: refused HELLO must not create a catalog entry"
8491 );
8492 }
8493 }
8494
8495 #[test]
8496 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
8497 let registry = Arc::new(Registry::default());
8498 let handler = ControlHandler::new(Arc::clone(®istry));
8499 let capabilities = json!({
8500 "provides": ["credentials-provider/v1"],
8501 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
8502 "must_never_reach": ["federation-transport/v1"]
8503 });
8504 let response = handler
8505 .handle_control(
8506 ConnectionId::new(99),
8507 capability_grammar_hello_frame(
8508 capabilities.clone(),
8509 Some(json!(["/roles/0/tools"])),
8510 99,
8511 ),
8512 )
8513 .expect("valid HELLO registers");
8514 assert_eq!(response[0].header.ty, FrameType::HelloAck);
8515
8516 let request = Frame::build(
8517 FrameType::Request,
8518 control_flags(),
8519 0,
8520 0,
8521 100,
8522 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8523 .expect("catalog request serializes"),
8524 )
8525 .expect("catalog request frame builds");
8526 let response = handler
8527 .handle_catalog_list(request, None)
8528 .expect("catalog list succeeds");
8529 let ClientControlResponse::CatalogList { modules, .. } =
8530 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
8531 else {
8532 panic!("catalog request must return catalog.list");
8533 };
8534 assert_eq!(modules.len(), 1);
8535 assert_eq!(
8536 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
8537 capabilities
8538 );
8539 }
8540
8541 #[test]
8542 fn catalog_list_mirrors_management_operation_description() {
8543 let registry = Arc::new(Registry::default());
8544 let handler = ControlHandler::new(Arc::clone(®istry));
8545 let description = "List managed records and return their identifiers and metadata.";
8546 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
8547 manifest.provides = vec![ProviderRole::ManagementSurface {
8548 operations: vec![ManagementOperation {
8549 name: "records.list".to_string(),
8550 kind: ManagementOperationKind::Query,
8551 description: Some(description.to_string()),
8552 }],
8553 config_schema: json!({"type": "object"}),
8554 observability: vec![ObservabilitySurface {
8555 name: "records.stats".to_string(),
8556 kind: ObservabilityKind::Snapshot,
8557 }],
8558 identity_scope: vec![IdentityScope::Project],
8559 concurrency: Concurrency::ModuleManaged,
8560 }];
8561 registry
8562 .register_with_control_ops(
8563 manifest,
8564 PROTOCOL_VERSION,
8565 ConnectionId::new(99),
8566 Vec::new(),
8567 )
8568 .expect("described management manifest registers");
8569
8570 let request = Frame::build(
8571 FrameType::Request,
8572 control_flags(),
8573 0,
8574 0,
8575 100,
8576 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8577 .expect("catalog request serializes"),
8578 )
8579 .expect("catalog request frame builds");
8580 let response = handler
8581 .handle_catalog_list(request, None)
8582 .expect("catalog list succeeds");
8583 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8584 assert_eq!(
8585 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
8586 "catalog.list must preserve the declared operation description verbatim"
8587 );
8588 }
8589
8590 #[test]
8591 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
8592 let registry = Arc::new(Registry::default());
8593 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8594 [("vault".to_string(), true), ("squatter".to_string(), true)],
8595 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8596 );
8597 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
8598 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8599 provides: vec!["credentials-provider/v1".to_string()],
8600 requires: Vec::new(),
8601 must_never_reach: Vec::new(),
8602 });
8603 let frame = Frame::build(
8604 FrameType::Hello,
8605 control_flags(),
8606 0,
8607 0,
8608 77,
8609 serde_json::to_vec(&ModuleHelloBody {
8610 manifest: squatter,
8611 protocol_ver: PROTOCOL_VERSION,
8612 control_ops: None,
8613 launch_nonce: None,
8614 })
8615 .expect("HELLO serializes"),
8616 )
8617 .expect("HELLO frame builds");
8618 let response = handler
8619 .handle_control(ConnectionId::new(77), frame)
8620 .expect("reserved claim receives a typed refusal");
8621 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8622 assert_eq!(
8623 registry
8624 .active_registration_count()
8625 .expect("registry reads"),
8626 0,
8627 "a reserved capability refusal must not leave a catalog entry"
8628 );
8629 }
8630
8631 #[test]
8632 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
8633 for settlement in ["timeout", "inconclusive", "drop"] {
8634 let breakers = RouteBindBreakers::default();
8635 let RouteBindAdmission::Admitted {
8636 guard: mut old,
8637 probe: false,
8638 } = breakers.admit("prov")
8639 else {
8640 panic!("ordinary relay admitted")
8641 };
8642 let RouteBindAdmission::Admitted {
8643 guard: mut opener, ..
8644 } = breakers.admit("prov")
8645 else {
8646 panic!("second relay admitted")
8647 };
8648 assert!(
8649 !opener
8650 .record_timeout(1, Duration::ZERO)
8651 .unwrap()
8652 .reopened_after_probe
8653 );
8654 let RouteBindAdmission::Admitted {
8655 guard: mut probe,
8656 probe: true,
8657 } = breakers.admit("prov")
8658 else {
8659 panic!("one cooldown probe admitted")
8660 };
8661 match settlement {
8662 "timeout" => assert!(
8663 !old.record_timeout(1, Duration::ZERO)
8664 .unwrap()
8665 .reopened_after_probe
8666 ),
8667 "inconclusive" => old.record_inconclusive(),
8668 "drop" => drop(old),
8669 _ => unreachable!(),
8670 }
8671 assert!(
8672 matches!(
8673 breakers.admit("prov"),
8674 RouteBindAdmission::Refused {
8675 probe_in_flight: true,
8676 ..
8677 }
8678 ),
8679 "{settlement} of a pre-open relay cannot release the real probe"
8680 );
8681 assert!(
8682 probe
8683 .record_timeout(1, Duration::ZERO)
8684 .unwrap()
8685 .reopened_after_probe
8686 );
8687 assert!(matches!(
8688 breakers.admit("prov"),
8689 RouteBindAdmission::Admitted { probe: true, .. }
8690 ));
8691 }
8692 let breakers = RouteBindBreakers::default();
8693 let admit = || match breakers.admit("prov") {
8694 RouteBindAdmission::Admitted { guard, .. } => guard,
8695 _ => panic!("relay admitted"),
8696 };
8697 admit().record_timeout(1, Duration::ZERO);
8698 let mut old_probe = admit();
8699 breakers.reset_for_new_module_connection("prov");
8700 admit().record_timeout(1, Duration::ZERO);
8701 let _new_probe = admit();
8702 old_probe.record_inconclusive();
8703 assert!(matches!(
8704 breakers.admit("prov"),
8705 RouteBindAdmission::Refused {
8706 probe_in_flight: true,
8707 ..
8708 }
8709 ));
8710 }
8711
8712 #[tokio::test]
8713 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
8714 for candidate in [false, true] {
8715 let registry = Arc::new(Registry::default());
8716 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8717 [("vault".to_string(), true), ("squatter".to_string(), true)],
8718 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8719 );
8720 let conn = ConnectionId::new(77);
8721 let (ctx, mut rx) = route_ctx(conn);
8722 let initial = capability_manifest("squatter", &[], &[]);
8723 if candidate {
8724 registry
8725 .register_candidate_with_control_ops(
8726 initial.clone(),
8727 PROTOCOL_VERSION,
8728 conn,
8729 module_baseline_control_ops(),
8730 )
8731 .unwrap();
8732 handler
8733 .forwarding
8734 .register_candidate_module_connection(
8735 conn,
8736 "squatter".to_string(),
8737 PROTOCOL_VERSION,
8738 manifest_concurrency(&initial),
8739 ctx.egress.clone(),
8740 )
8741 .unwrap();
8742 } else {
8743 hello_via_sink(
8744 &handler,
8745 &ctx,
8746 &mut rx,
8747 hello_frame_with_manifest(initial.clone(), 1),
8748 )
8749 .await;
8750 }
8751 let response = handler
8752 .handle_control_frame(
8753 &ctx,
8754 catalog_update_with_capabilities_frame(
8755 2,
8756 capability_manifest("squatter", &["credentials-provider/v1"], &[])
8757 .capabilities
8758 .unwrap(),
8759 ),
8760 )
8761 .await
8762 .unwrap();
8763 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8764 assert_eq!(
8765 registry
8766 .get_module_by_connection(conn)
8767 .unwrap()
8768 .unwrap()
8769 .manifest,
8770 initial
8771 );
8772 }
8773 }
8774
8775 #[test]
8776 fn server_describe_surfaces_required_capability_verdict_fields() {
8777 let registry = Arc::new(Registry::default());
8778 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8779 [
8780 ("consumer".to_string(), true),
8781 ("provider".to_string(), false),
8782 ],
8783 BTreeMap::new(),
8784 );
8785 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
8786 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8787 provides: Vec::new(),
8788 requires: vec![subc_protocol::manifest::CapabilityRequirement {
8789 capability: "credentials-provider/v1".to_string(),
8790 need: subc_protocol::manifest::CapabilityNeed::Required,
8791 }],
8792 must_never_reach: Vec::new(),
8793 });
8794 let hello = Frame::build(
8795 FrameType::Hello,
8796 control_flags(),
8797 0,
8798 0,
8799 78,
8800 serde_json::to_vec(&ModuleHelloBody {
8801 manifest: consumer,
8802 protocol_ver: PROTOCOL_VERSION,
8803 control_ops: None,
8804 launch_nonce: None,
8805 })
8806 .expect("HELLO serializes"),
8807 )
8808 .expect("HELLO frame builds");
8809 handler
8810 .handle_control(ConnectionId::new(78), hello)
8811 .expect("consumer registers");
8812 let describe = Frame::build(
8813 FrameType::Request,
8814 control_flags(),
8815 0,
8816 0,
8817 79,
8818 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
8819 .expect("request serializes"),
8820 )
8821 .expect("describe frame builds");
8822 let response = handler
8823 .handle_server_describe(describe)
8824 .expect("server.describe succeeds");
8825 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
8826 let requirement = &rendered["capability_requirements"][0];
8827 assert_eq!(requirement["consumer"], "consumer");
8828 assert_eq!(requirement["verdict"], "never_provided");
8829 assert_eq!(requirement["episode_seq"], 1);
8830 assert_eq!(requirement["config_satisfiable"], false);
8831 assert_eq!(requirement["runtime_available"], false);
8832 assert!(requirement["detail"]
8833 .as_str()
8834 .expect("detail string")
8835 .contains("credentials-provider/v1"));
8836 }
8837
8838 #[test]
8839 fn catalog_list_omits_capabilities_for_legacy_manifest() {
8840 let registry = Arc::new(Registry::default());
8841 let handler = ControlHandler::new(Arc::clone(®istry));
8842 let hello = handler
8843 .handle_control(
8844 ConnectionId::new(101),
8845 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
8846 )
8847 .expect("legacy HELLO registers");
8848 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
8849
8850 let request = Frame::build(
8851 FrameType::Request,
8852 control_flags(),
8853 0,
8854 0,
8855 102,
8856 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8857 .expect("catalog request serializes"),
8858 )
8859 .expect("catalog request frame builds");
8860 let response = handler
8861 .handle_catalog_list(request, None)
8862 .expect("catalog list succeeds");
8863 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8864 assert!(
8865 body["modules"][0].get("capabilities").is_none(),
8866 "legacy manifest must retain an absent capabilities field on catalog.list"
8867 );
8868 }
8869
8870 #[test]
8871 fn hello_ack_omits_storage_when_no_storage_config() {
8872 let registry = Arc::new(Registry::default());
8873 let handler = ControlHandler::new(Arc::clone(®istry));
8874 let responses = handler
8875 .handle_control(
8876 ConnectionId::new(1),
8877 hello_frame("aft", PROTOCOL_VERSION, 7),
8878 )
8879 .unwrap();
8880 let ack = parse_ack(&responses[0]);
8881 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
8882 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
8883 }
8884
8885 #[tokio::test]
8886 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
8887 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
8888 let registry = Arc::new(Registry::default());
8889 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
8890 let responses = handler
8891 .handle_control(
8892 ConnectionId::new(1),
8893 hello_frame("aft", PROTOCOL_VERSION, 7),
8894 )
8895 .unwrap();
8896 let ack = parse_ack(&responses[0]);
8897 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
8898
8899 let described = handler
8900 .handle_control_frame(
8901 &route_ctx(ConnectionId::new(2)).0,
8902 Frame::build(
8903 FrameType::Request,
8904 control_flags(),
8905 0,
8906 0,
8907 9,
8908 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8909 )
8910 .unwrap(),
8911 )
8912 .await
8913 .unwrap();
8914 let ClientControlResponse::ServerDescribe { machine_id, .. } =
8915 serde_json::from_slice(&described[0].body).unwrap()
8916 else {
8917 panic!("server.describe answered with another shape");
8918 };
8919 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
8920 }
8921
8922 #[test]
8923 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
8924 let registry = Arc::new(Registry::default());
8927 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
8928 crate::daemon_config::StorageConfig::Sqlite {
8929 data_home: std::path::PathBuf::from("/data"),
8930 },
8931 ));
8932
8933 let responses = handler
8934 .handle_control(
8935 ConnectionId::new(1),
8936 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
8937 )
8938 .unwrap();
8939 let ack = parse_ack(&responses[0]);
8940 assert_eq!(
8941 ack.storage,
8942 Some(serde_json::json!({
8943 "module_id": "alfonso-routing",
8944 "storage_namespace": "default",
8945 "isolation": { "kind": "module" },
8946 "backend": {
8947 "backend": "sqlite",
8948 "path": "/data/cortexkit/alfonso-routing/store.db"
8949 }
8950 })),
8951 "the delivered descriptor is the module's own sqlite store path"
8952 );
8953 }
8954
8955 #[test]
8956 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
8957 let registry = Arc::new(Registry::default());
8958 let handler = ControlHandler::new(Arc::clone(®istry));
8959 let conn = ConnectionId::new(1);
8960 let responses = handler
8961 .handle_control(
8962 conn,
8963 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8964 )
8965 .unwrap();
8966 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8967 let registration = registry.get_module("aft").unwrap().unwrap();
8968 assert_eq!(registration.control_ops, module_baseline_control_ops());
8969
8970 let frame =
8971 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
8972 assert!(handler
8973 .guard_module_control_op(&frame, "aft", "route.bind")
8974 .unwrap()
8975 .is_none());
8976 let error = handler
8977 .guard_module_control_op(&frame, "aft", "test.synthetic")
8978 .unwrap()
8979 .expect("synthetic ungranted op should be rejected");
8980 assert_eq!(error.header.ty, FrameType::Error);
8981 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
8982 }
8983
8984 #[test]
8985 fn hello_control_ops_some_adds_optional_grants() {
8986 let registry = Arc::new(Registry::default());
8987 let handler = ControlHandler::new(Arc::clone(®istry));
8988 handler
8989 .handle_control(
8990 ConnectionId::new(1),
8991 hello_frame_with_control_ops(
8992 "aft",
8993 PROTOCOL_VERSION,
8994 7,
8995 Some(vec![
8996 "future.synthetic".to_string(),
8997 "route.bind".to_string(),
8998 ]),
8999 ),
9000 )
9001 .unwrap();
9002 let registration = registry.get_module("aft").unwrap().unwrap();
9003 assert_eq!(
9004 registration.control_ops,
9005 vec![
9006 "route.bind".to_string(),
9007 "route.status".to_string(),
9008 "future.synthetic".to_string(),
9009 ]
9010 );
9011 let frame =
9012 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
9013 assert!(handler
9014 .guard_module_control_op(&frame, "aft", "future.synthetic")
9015 .unwrap()
9016 .is_none());
9017 }
9018
9019 #[tokio::test]
9020 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
9021 let registry = Arc::new(Registry::default());
9022 let forwarding = Arc::new(ForwardingTable::default());
9023 let handler =
9024 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9025 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
9026 hello_via_sink(
9027 &handler,
9028 &module_ctx,
9029 &mut module_rx,
9030 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9031 )
9032 .await;
9033
9034 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
9035 let responses = handler
9036 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
9037 .await
9038 .unwrap();
9039 assert_eq!(responses.len(), 1);
9040 assert_eq!(responses[0].header.ty, FrameType::Error);
9041 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
9042 assert!(module_rx.try_recv().is_err());
9043 }
9044
9045 #[tokio::test]
9046 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
9047 let registry = Arc::new(Registry::default());
9048 let forwarding = Arc::new(ForwardingTable::default());
9049 let handler =
9050 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9051 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
9052 hello_via_sink(
9053 &handler,
9054 &module_ctx,
9055 &mut module_rx,
9056 hello_frame_with_control_ops(
9057 "aft",
9058 PROTOCOL_VERSION,
9059 7,
9060 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9061 ),
9062 )
9063 .await;
9064
9065 let project_root = unique_project_root("demux");
9066 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
9067 let route_handler = handler.clone();
9068 let route_task = tokio::spawn(async move {
9069 route_handler
9070 .handle_control_frame(
9071 &route_client_ctx,
9072 route_open_frame(100, "aft", project_root),
9073 )
9074 .await
9075 .unwrap()
9076 });
9077 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9078 .await
9079 .unwrap()
9080 .unwrap();
9081 assert!(matches!(
9082 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
9083 ModuleControlRequest::RouteBind { .. }
9084 ));
9085
9086 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
9087 let health_handler = handler.clone();
9088 let health_task = tokio::spawn(async move {
9089 health_handler
9090 .handle_control_frame(
9091 &health_client_ctx,
9092 supervisor_health_probe_frame(101, "aft"),
9093 )
9094 .await
9095 .unwrap()
9096 });
9097 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9098 .await
9099 .unwrap()
9100 .unwrap();
9101 assert_eq!(
9102 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9103 ModuleControlRequest::HealthCheck {}
9104 );
9105
9106 handler
9107 .handle_control_frame(
9108 &module_ctx,
9109 health_response(health_frame.header.corr, HealthStatus::Degraded),
9110 )
9111 .await
9112 .unwrap();
9113 let health_response = health_task.await.unwrap();
9114 assert_eq!(health_response.len(), 1);
9115 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9116 ClientControlResponse::SupervisorHealthProbe {
9117 module_id,
9118 status,
9119 detail,
9120 metrics,
9121 } => {
9122 assert_eq!(module_id, "aft");
9123 assert_eq!(status, HealthStatus::Degraded);
9124 assert_eq!(detail.as_deref(), Some("warming"));
9125 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
9126 }
9127 other => panic!("unexpected health response: {other:?}"),
9128 }
9129
9130 handler
9131 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9132 .await
9133 .unwrap();
9134 let route_response = route_task.await.unwrap();
9135 assert!(route_response.is_empty());
9136 let published = route_client_rx.recv().await.unwrap();
9137 assert!(matches!(
9138 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9139 ClientControlResponse::RouteOpen { .. }
9140 ));
9141 }
9142
9143 async fn relay_route_open(
9148 handler: &ControlHandler,
9149 client_connection: ConnectionId,
9150 client_egress: &FrameSink,
9151 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9152 corr: u64,
9153 module_id: &str,
9154 project_root_label: &str,
9155 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
9156 let ctx = RouteCtx {
9157 connection_id: client_connection,
9158 egress: client_egress.clone(),
9159 };
9160 let handler = handler.clone();
9161 let project_root = unique_project_root(project_root_label);
9162 let module_id = module_id.to_string();
9163 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
9164 let task = tokio::spawn(async move {
9165 let _guard = tracing::dispatcher::set_default(&dispatch);
9166 handler
9167 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
9168 .await
9169 .unwrap()
9170 });
9171 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
9172 .await
9173 .expect("module receives the relayed route.bind")
9174 .expect("module egress is open");
9175 (task, bind.frame)
9176 }
9177
9178 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
9179 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
9180 ModuleControlRequest::RouteBind {
9181 route_channel,
9182 epoch,
9183 ..
9184 } => (route_channel, epoch),
9185 other => panic!("expected a route.bind request, got {other:?}"),
9186 }
9187 }
9188
9189 fn published_route(frame: &Frame) -> (u16, u32) {
9190 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
9191 ClientControlResponse::RouteOpen {
9192 route_channel,
9193 route_epoch,
9194 } => (route_channel, route_epoch),
9195 other => panic!("expected a route.open response, got {other:?}"),
9196 }
9197 }
9198
9199 #[tokio::test]
9219 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
9220 let registry = Arc::new(Registry::default());
9221 let forwarding = Arc::new(ForwardingTable::default());
9222 let handler =
9223 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9224
9225 let module_connection = ConnectionId::new(30);
9226 let (module_ctx, mut module_rx) = route_ctx(module_connection);
9227 hello_via_sink(
9228 &handler,
9229 &module_ctx,
9230 &mut module_rx,
9231 hello_frame("aft", PROTOCOL_VERSION, 7),
9232 )
9233 .await;
9234
9235 let dying_client = ConnectionId::new(31);
9236 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
9237
9238 let (first_task, first_bind) = relay_route_open(
9241 &handler,
9242 dying_client,
9243 &dying_ctx.egress,
9244 &mut module_rx,
9245 100,
9246 "aft",
9247 "closing-first",
9248 )
9249 .await;
9250 handler
9251 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
9252 .await
9253 .unwrap();
9254 assert!(first_task.await.unwrap().is_empty());
9255 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
9256
9257 let (second_task, second_bind) = relay_route_open(
9259 &handler,
9260 dying_client,
9261 &dying_ctx.egress,
9262 &mut module_rx,
9263 101,
9264 "aft",
9265 "closing-second",
9266 )
9267 .await;
9268 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
9269
9270 assert!(forwarding
9273 .escalate_client_delivery_failure(
9274 dying_client,
9275 first_channel,
9276 first_epoch,
9277 CloseReason::new(
9278 "module_to_client_delivery_failed",
9279 "client egress refused a module frame",
9280 ),
9281 crate::forwarding::UndeliveredFrame {
9282 module_id: None,
9283 sink: &dying_ctx.egress,
9284 },
9285 )
9286 .unwrap());
9287 assert!(!dying_ctx.egress.is_closed());
9288
9289 let ack = handler
9291 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
9292 .await;
9293 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
9294 if module_loop_error.is_some() {
9295 handler.cleanup_connection(module_connection).unwrap();
9299 }
9300 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9304 .await
9305 .ok()
9306 .flatten();
9307
9308 assert!(
9310 registry
9311 .get_module_by_connection(module_connection)
9312 .unwrap()
9313 .is_some(),
9314 "one client's closing connection ended the shared module connection: \
9315 {module_loop_error:?}"
9316 );
9317 let cotenant = ConnectionId::new(32);
9319 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
9320 let (cotenant_task, cotenant_bind) = relay_route_open(
9321 &handler,
9322 cotenant,
9323 &cotenant_ctx.egress,
9324 &mut module_rx,
9325 102,
9326 "aft",
9327 "closing-cotenant",
9328 )
9329 .await;
9330 handler
9331 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
9332 .await
9333 .unwrap();
9334 assert!(cotenant_task.await.unwrap().is_empty());
9335 let (cotenant_channel, cotenant_epoch) =
9336 published_route(&cotenant_rx.recv().await.unwrap());
9337 assert!(matches!(
9338 forwarding
9339 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
9340 .unwrap(),
9341 DataRoute::Client(DataRouteState::Bound(_))
9342 ));
9343
9344 let goodbye = post_ack_module_frame
9347 .expect("module receives a GOODBYE for the abandoned route channel");
9348 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9349 assert_eq!(goodbye.header.channel, abandoned_channel);
9350 assert_eq!(goodbye.header.epoch, abandoned_epoch);
9351
9352 assert!(dying_rx.try_recv().is_err());
9356 let second_response = second_task.await.unwrap();
9357 assert_eq!(second_response.len(), 1);
9358 assert_eq!(
9359 parse_error(&second_response[0])["code"],
9360 "target_unavailable"
9361 );
9362 }
9363
9364 async fn catalog_snapshot(
9367 handler: &ControlHandler,
9368 client_ctx: &RouteCtx,
9369 corr: u64,
9370 ) -> (u64, Vec<(String, bool)>) {
9371 let body =
9372 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
9373 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
9374 let reply = handler
9375 .handle_control_frame(client_ctx, request)
9376 .await
9377 .unwrap()
9378 .pop()
9379 .unwrap();
9380 let ClientControlResponse::CatalogList {
9381 generation,
9382 modules,
9383 ..
9384 } = serde_json::from_slice(&reply.body).unwrap()
9385 else {
9386 panic!("not a catalog.list reply");
9387 };
9388 (
9389 generation,
9390 modules
9391 .into_iter()
9392 .map(|entry| (entry.module_id, entry.ready))
9393 .collect(),
9394 )
9395 }
9396
9397 fn hello_held_before_forwarding() -> (
9403 ControlHandler,
9404 Arc<std::sync::Barrier>,
9405 std::thread::JoinHandle<Vec<Frame>>,
9406 RouteCtx,
9407 mpsc::Receiver<crate::router::OutboundFrame>,
9408 ) {
9409 let hold = Arc::new(std::sync::Barrier::new(2));
9410 let handler = ControlHandler::with_forwarding(
9411 Arc::new(Registry::default()),
9412 Arc::new(ForwardingTable::default()),
9413 )
9414 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
9415 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9416 let hello = {
9417 let handler = handler.clone();
9418 let module_ctx = module_ctx.clone();
9419 std::thread::spawn(move || {
9420 handler
9421 .handle_hello(
9422 module_ctx.connection_id,
9423 Some(module_ctx.egress.clone()),
9424 hello_frame("aft", PROTOCOL_VERSION, 7),
9425 )
9426 .unwrap()
9427 })
9428 };
9429 (handler, hold, hello, module_ctx, module_rx)
9430 }
9431
9432 #[tokio::test]
9441 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
9442 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
9443 let client = ConnectionId::new(41);
9444 let (client_ctx, mut client_rx) = route_ctx(client);
9445
9446 hold.wait();
9447 assert!(handler.registry.get_module("aft").unwrap().is_some());
9449 assert!(!handler
9450 .forwarding
9451 .has_live_module_connection("aft")
9452 .unwrap());
9453 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
9454 assert!(
9455 listed.is_empty(),
9456 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
9457 );
9458 let refused = handler
9459 .handle_control_frame(
9460 &client_ctx,
9461 route_open_frame(301, "aft", unique_project_root("listed-window")),
9462 )
9463 .await
9464 .unwrap();
9465 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
9466 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
9467 let duplicate = handler
9468 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
9469 .await
9470 .unwrap();
9471 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9472
9473 hold.wait();
9474 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9475 let ack = module_rx.try_recv().unwrap().frame;
9476 assert_eq!(ack.header.ty, FrameType::HelloAck);
9477 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9478 assert_eq!(listed, vec![("aft".to_string(), true)]);
9479
9480 let (open_task, bind) = relay_route_open(
9481 &handler,
9482 client,
9483 &client_ctx.egress,
9484 &mut module_rx,
9485 303,
9486 "aft",
9487 "listed-after",
9488 )
9489 .await;
9490 assert!(handler
9491 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9492 .await
9493 .unwrap()
9494 .is_empty());
9495 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
9496 published_route(&client_rx.try_recv().unwrap().frame);
9497 }
9498
9499 #[tokio::test]
9506 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
9507 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
9508 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9509 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9510
9511 hold.wait();
9512 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
9513 assert!(listed.is_empty());
9514 assert_eq!(
9515 during, before,
9516 "the generation moved while the module was not listed"
9517 );
9518
9519 hold.wait();
9520 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9521 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9522 assert_eq!(listed, vec![("aft".to_string(), true)]);
9523 assert_eq!(
9524 after,
9525 before + 1,
9526 "the module appeared without exactly one generation bump"
9527 );
9528 }
9529
9530 #[tokio::test]
9535 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
9536 let registry = Arc::new(Registry::default());
9537 let handler = ControlHandler::with_forwarding(
9538 Arc::clone(®istry),
9539 Arc::new(ForwardingTable::default()),
9540 );
9541 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9542 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9543 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9544 drop(module_rx);
9545
9546 let replies = handler
9547 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
9548 .await
9549 .unwrap();
9550 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
9551 assert!(registry.get_module("aft").unwrap().is_none());
9552 assert!(registry
9553 .get_module_by_connection(module_ctx.connection_id)
9554 .unwrap()
9555 .is_none());
9556 assert_eq!(
9557 catalog_snapshot(&handler, &client_ctx, 301).await,
9558 (before, Vec::new())
9559 );
9560 }
9561
9562 async fn route_open_held_after_bind_send() -> (
9566 ControlHandler,
9567 Arc<tokio::sync::Barrier>,
9568 RouteCtx,
9569 mpsc::Receiver<crate::router::OutboundFrame>,
9570 ) {
9571 let hold = Arc::new(tokio::sync::Barrier::new(2));
9572 let handler = ControlHandler::with_forwarding(
9573 Arc::new(Registry::default()),
9574 Arc::new(ForwardingTable::default()),
9575 )
9576 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
9577 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
9578 hello_via_sink(
9579 &handler,
9580 &module_ctx,
9581 &mut module_rx,
9582 hello_frame("aft", PROTOCOL_VERSION, 7),
9583 )
9584 .await;
9585 (handler, hold, module_ctx, module_rx)
9586 }
9587
9588 #[tokio::test]
9595 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
9596 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9597 let client = ConnectionId::new(41);
9598 let (client_ctx, mut client_rx) = route_ctx(client);
9599
9600 let (open_task, bind) = relay_route_open(
9601 &handler,
9602 client,
9603 &client_ctx.egress,
9604 &mut module_rx,
9605 200,
9606 "aft",
9607 "acked-before-mark",
9608 )
9609 .await;
9610 let (module_channel, module_epoch) = route_bind_channel(&bind);
9611 hold.wait().await;
9613
9614 assert!(handler
9616 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9617 .await
9618 .unwrap()
9619 .is_empty());
9620 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
9621
9622 hold.wait().await;
9623 assert!(
9624 open_task.await.unwrap().is_empty(),
9625 "route.open answers accepted"
9626 );
9627
9628 match module_rx.try_recv() {
9631 Err(mpsc::error::TryRecvError::Empty) => {}
9632 Ok(outbound) => panic!(
9633 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
9634 outbound.frame.header.ty,
9635 outbound.frame.header.channel,
9636 outbound.frame.header.epoch
9637 ),
9638 Err(err) => panic!("module egress closed: {err}"),
9639 }
9640
9641 let router = Router::with_control_handler(Arc::new(handler.clone()));
9644 let request = Frame::build(
9645 FrameType::Request,
9646 Flags::new(true, Priority::Interactive, false),
9647 client_channel,
9648 client_epoch,
9649 201,
9650 b"first request".to_vec(),
9651 )
9652 .unwrap();
9653 router
9654 .route_for_connection(&client_ctx, request)
9655 .await
9656 .unwrap();
9657 let delivered = module_rx.try_recv().unwrap().frame;
9658 assert_eq!(delivered.header.ty, FrameType::Request);
9659 assert_eq!(
9660 (delivered.header.channel, delivered.header.epoch),
9661 (module_channel, module_epoch)
9662 );
9663 assert_eq!(delivered.body, b"first request");
9664 }
9665
9666 #[tokio::test]
9672 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
9673 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9674 let client = ConnectionId::new(42);
9675 let (client_ctx, _client_rx) = route_ctx(client);
9676
9677 let (open_task, bind) = relay_route_open(
9678 &handler,
9679 client,
9680 &client_ctx.egress,
9681 &mut module_rx,
9682 300,
9683 "aft",
9684 "released-before-mark",
9685 )
9686 .await;
9687 let (module_channel, module_epoch) = route_bind_channel(&bind);
9688 hold.wait().await;
9689
9690 handler.cleanup_connection(client).unwrap();
9691 assert!(
9692 module_rx.try_recv().is_err(),
9693 "client teardown must not GOODBYE a bind route.open is still queueing"
9694 );
9695
9696 hold.wait().await;
9697 let response = open_task.await.unwrap();
9698 assert_eq!(response.len(), 1);
9699 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9700
9701 let goodbye = module_rx
9702 .try_recv()
9703 .expect("module is told to drop the binding the daemon abandoned")
9704 .frame;
9705 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9706 assert_eq!(
9707 (goodbye.header.channel, goodbye.header.epoch),
9708 (module_channel, module_epoch)
9709 );
9710 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9711 }
9712
9713 #[tokio::test]
9718 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
9719 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9720 let client = ConnectionId::new(43);
9721 let (client_ctx, client_rx) = route_ctx(client);
9722
9723 let (open_task, bind) = relay_route_open(
9724 &handler,
9725 client,
9726 &client_ctx.egress,
9727 &mut module_rx,
9728 400,
9729 "aft",
9730 "gone-client-before-mark",
9731 )
9732 .await;
9733 let (module_channel, module_epoch) = route_bind_channel(&bind);
9734 hold.wait().await;
9735
9736 drop(client_rx);
9737 assert!(client_ctx.egress.is_closed());
9738 handler
9739 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9740 .await
9741 .unwrap();
9742
9743 hold.wait().await;
9744 let response = open_task.await.unwrap();
9745 assert_eq!(response.len(), 1);
9746 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9747
9748 let goodbye = module_rx
9749 .try_recv()
9750 .expect("module is told to drop the binding it acked")
9751 .frame;
9752 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9753 assert_eq!(
9754 (goodbye.header.channel, goodbye.header.epoch),
9755 (module_channel, module_epoch)
9756 );
9757 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9758 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
9759 }
9760
9761 #[test]
9768 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
9769 let handler = ControlHandler::default();
9770 let module_connection = ConnectionId::new(30);
9771 let client_connection = ConnectionId::new(31);
9772
9773 handler
9774 .refuse_to_end_module_connection_for_a_client(
9775 module_connection,
9776 77,
9777 ForwardingError::ConnectionClosing {
9778 connection_id: client_connection,
9779 },
9780 )
9781 .expect("a closing client must never end the module connection");
9782
9783 assert!(matches!(
9784 handler.refuse_to_end_module_connection_for_a_client(
9785 module_connection,
9786 78,
9787 ForwardingError::ConnectionClosing {
9788 connection_id: module_connection,
9789 },
9790 ),
9791 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
9792 connection_id
9793 })) if connection_id == module_connection
9794 ));
9795 assert!(matches!(
9796 handler.refuse_to_end_module_connection_for_a_client(
9797 module_connection,
9798 79,
9799 ForwardingError::Poisoned,
9800 ),
9801 Err(RouterError::Forwarding(ForwardingError::Poisoned))
9802 ));
9803 assert!(matches!(
9804 handler.refuse_to_end_module_connection_for_a_client(
9805 module_connection,
9806 80,
9807 ForwardingError::StaleModuleEndpoint,
9808 ),
9809 Err(RouterError::Forwarding(
9810 ForwardingError::StaleModuleEndpoint
9811 ))
9812 ));
9813 }
9814
9815 #[tokio::test]
9828 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
9829 let registry = Arc::new(Registry::default());
9830 let forwarding = Arc::new(ForwardingTable::default());
9831 let supervisor = SupervisorHandle::new();
9832 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9833 let handler =
9834 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9835 .with_supervisor(supervisor);
9836
9837 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9838 hello_via_sink(
9839 &handler,
9840 &target_ctx,
9841 &mut target_rx,
9842 hello_frame("target", PROTOCOL_VERSION, 1),
9843 )
9844 .await;
9845
9846 let wrong_nonce = handler
9850 .handle_control_frame(
9851 &route_ctx(ConnectionId::new(91)).0,
9852 route_open_frame_with_admission_facts(
9853 20,
9854 "target",
9855 unique_project_root("admission-facts"),
9856 Some(subc_control::ConsumerIdentity {
9857 module_id: "fed".to_string(),
9858 launch_nonce: "not-the-real-nonce".to_string(),
9859 }),
9860 None,
9861 ),
9862 )
9863 .await
9864 .unwrap();
9865 assert_eq!(
9866 parse_error(&wrong_nonce[0])["code"],
9867 "bad_consumer_identity",
9868 "a mismatched launch nonce must be refused, not stamped Reserved"
9869 );
9870
9871 let never_spawned = handler
9875 .handle_control_frame(
9876 &route_ctx(ConnectionId::new(92)).0,
9877 route_open_frame_with_admission_facts(
9878 21,
9879 "target",
9880 unique_project_root("admission-facts"),
9881 Some(subc_control::ConsumerIdentity {
9882 module_id: "never-spawned".to_string(),
9883 launch_nonce: "any-nonce".to_string(),
9884 }),
9885 None,
9886 ),
9887 )
9888 .await
9889 .unwrap();
9890 assert_eq!(
9891 parse_error(&never_spawned[0])["code"],
9892 "bad_consumer_identity",
9893 "an unspawned module_id must be refused rather than accepted for lack of a record"
9894 );
9895 }
9896
9897 #[tokio::test]
9920 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
9921 let registry = Arc::new(Registry::default());
9922 let forwarding = Arc::new(ForwardingTable::default());
9923 let supervisor = SupervisorHandle::new();
9924 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9925 let handler =
9926 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9927 .with_supervisor(supervisor);
9928
9929 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
9930 hello_via_sink(
9931 &handler,
9932 &target_ctx,
9933 &mut target_rx,
9934 hello_frame("target", PROTOCOL_VERSION, 1),
9935 )
9936 .await;
9937
9938 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
9939 let route_handler = handler.clone();
9940 let route_task = tokio::spawn(async move {
9941 route_handler
9942 .handle_control_frame(
9943 &client_ctx,
9944 route_open_frame_with_admission_facts(
9945 30,
9946 "target",
9947 unique_project_root("admission-facts"),
9948 Some(subc_control::ConsumerIdentity {
9949 module_id: "fed".to_string(),
9950 launch_nonce: "fed-nonce".to_string(),
9951 }),
9952 None,
9953 ),
9954 )
9955 .await
9956 .unwrap()
9957 });
9958
9959 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9966 .await
9967 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
9968 .expect("module control channel closed before route.bind");
9969 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9970 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
9971 panic!("expected route.bind")
9972 };
9973 assert_eq!(
9974 principal,
9975 Some(Principal::Reserved {
9976 module_id: "fed".to_string()
9977 }),
9978 "a correctly attested consumer must be stamped Reserved for its own id"
9979 );
9980
9981 handler
9982 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9983 .await
9984 .unwrap();
9985 assert!(route_task.await.unwrap().is_empty());
9986 assert!(
9987 matches!(
9988 serde_json::from_slice::<ClientControlResponse>(
9989 &client_rx.recv().await.unwrap().body
9990 )
9991 .unwrap(),
9992 ClientControlResponse::RouteOpen { .. }
9993 ),
9994 "the route must actually open, not merely avoid an error"
9995 );
9996 }
9997
9998 #[tokio::test(start_paused = true)]
9999 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
10000 let registry = Arc::new(Registry::default());
10001 let forwarding = Arc::new(ForwardingTable::default());
10002 let supervisor = SupervisorHandle::new();
10003 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10004 let handler =
10005 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10006 .with_supervisor(supervisor);
10007
10008 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
10009 hello_via_sink(
10010 &handler,
10011 &target_ctx,
10012 &mut target_rx,
10013 hello_frame("target", PROTOCOL_VERSION, 1),
10014 )
10015 .await;
10016
10017 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
10018 let direct_handler = handler.clone();
10019 let direct_open = tokio::spawn(async move {
10020 direct_handler
10021 .handle_control_frame(
10022 &direct_ctx,
10023 route_open_frame(2, "target", unique_project_root("route-census-direct")),
10024 )
10025 .await
10026 .unwrap()
10027 });
10028 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10029 .await
10030 .expect("no direct route.bind within 5s")
10031 .expect("target control channel closed before direct route.bind");
10032 handler
10033 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
10034 .await
10035 .unwrap();
10036 assert!(direct_open.await.unwrap().is_empty());
10037 let _ = direct_rx.recv().await.unwrap();
10038
10039 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
10040 let reserved_handler = handler.clone();
10041 let reserved_open = tokio::spawn(async move {
10042 reserved_handler
10043 .handle_control_frame(
10044 &reserved_ctx,
10045 route_open_frame_with_admission_facts(
10046 3,
10047 "target",
10048 unique_project_root("admission-facts"),
10049 Some(ConsumerIdentity {
10050 module_id: "fed".to_string(),
10051 launch_nonce: "fed-nonce".to_string(),
10052 }),
10053 None,
10054 ),
10055 )
10056 .await
10057 .unwrap()
10058 });
10059 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10060 .await
10061 .expect("no reserved route.bind within 5s")
10062 .expect("target control channel closed before reserved route.bind");
10063 handler
10064 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
10065 .await
10066 .unwrap();
10067 assert!(reserved_open.await.unwrap().is_empty());
10068 let _ = reserved_rx.recv().await.unwrap();
10069
10070 forwarding
10071 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
10072 .unwrap();
10073 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
10074 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
10075 module_id: Some("target".to_string()),
10076 })
10077 .unwrap();
10078 let census_frame =
10079 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
10080 let response = handler
10081 .handle_control_frame(&census_ctx, census_frame)
10082 .await
10083 .unwrap()
10084 .pop()
10085 .unwrap();
10086 let actual: Value = serde_json::from_slice(&response.body).unwrap();
10087 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
10088 assert!(matches!(
10089 decoded,
10090 ClientControlResponse::SupervisorRoutes { .. }
10091 ));
10092 let routes = actual["modules"][0]["routes"].as_array().unwrap();
10093 assert_eq!(routes.len(), 2);
10094 assert!(routes.iter().all(|route| route["draining"] == true));
10095 assert!(
10098 routes.iter().all(|route| route["drain_reason"] == "reload"),
10099 "draining routes must name the drain's reason: {routes:?}"
10100 );
10101 assert!(routes.iter().any(|route| {
10102 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
10103 }));
10104 assert!(routes.iter().any(|route| {
10105 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
10106 }));
10107
10108 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10109 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
10110 if std::env::var_os("UPDATE_GOLDEN").is_some() {
10111 std::fs::write(
10112 &golden_path,
10113 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
10114 )
10115 .unwrap();
10116 }
10117 let expected: Value =
10118 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
10119 assert_eq!(actual, expected);
10120 }
10121
10122 async fn query_live_roots(
10123 handler: &ControlHandler,
10124 module_ctx: &RouteCtx,
10125 ) -> ModuleControlResponseToModule {
10126 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
10127 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
10128 let response = handler
10129 .handle_control_frame(module_ctx, frame)
10130 .await
10131 .unwrap()
10132 .pop()
10133 .unwrap();
10134 serde_json::from_slice(&response.body).unwrap()
10135 }
10136
10137 #[tokio::test(start_paused = true)]
10138 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
10139 let registry = Arc::new(Registry::default());
10140 let forwarding = Arc::new(ForwardingTable::default());
10141 let handler = ControlHandler::with_forwarding(registry, forwarding);
10142 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
10143 hello_via_sink(
10144 &handler,
10145 &target_ctx,
10146 &mut target_rx,
10147 hello_frame("target", PROTOCOL_VERSION, 1),
10148 )
10149 .await;
10150 let root = unique_project_root("live-roots-known");
10151 let path = ProjectRootId::from_path_allowing_missing(root.path())
10152 .unwrap()
10153 .as_path()
10154 .to_path_buf();
10155 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
10156 let open_handler = handler.clone();
10157 let opened = tokio::spawn(async move {
10158 open_handler
10159 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
10160 .await
10161 .unwrap()
10162 });
10163 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10164 .await
10165 .unwrap()
10166 .unwrap();
10167 handler
10168 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
10169 .await
10170 .unwrap();
10171 assert!(opened.await.unwrap().is_empty());
10172 let _ = client_rx.recv().await.unwrap();
10173
10174 let root = unique_project_root("live-roots-pending");
10175 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
10176 .unwrap()
10177 .as_path()
10178 .to_path_buf();
10179 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
10180 let open_handler = handler.clone();
10181 let pending = tokio::spawn(async move {
10182 open_handler
10183 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
10184 .await
10185 .unwrap()
10186 });
10187 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10188 .await
10189 .unwrap()
10190 .unwrap();
10191 let actual = query_live_roots(&handler, &target_ctx).await;
10192 let ModuleControlResponseToModule::LiveRoots {
10193 roots,
10194 unknown_root_bindings,
10195 total_bindings,
10196 } = actual
10197 else {
10198 panic!("expected live roots")
10199 };
10200 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
10201 assert_eq!(unknown_root_bindings, 0);
10202 assert_eq!(
10203 roots.len(),
10204 2,
10205 "root-known arm must retain each canonical root"
10206 );
10207 assert_eq!(
10208 total_bindings,
10209 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10210 );
10211 let counts = roots
10212 .iter()
10213 .map(|root| (root.project_root.clone(), root.bound, root.pending))
10214 .collect::<Vec<_>>();
10215 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
10216 expected.sort_by(|a, b| a.0.cmp(&b.0));
10217 assert_eq!(
10218 counts, expected,
10219 "roots must sort by path and count pending separately"
10220 );
10221 handler
10222 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
10223 .await
10224 .unwrap();
10225 assert!(pending.await.unwrap().is_empty());
10226 }
10227
10228 #[tokio::test(start_paused = true)]
10229 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
10230 let forwarding = Arc::new(ForwardingTable::default());
10231 let handler =
10232 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10233 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
10234 hello_via_sink(
10235 &handler,
10236 &target_ctx,
10237 &mut target_rx,
10238 hello_frame("target", PROTOCOL_VERSION, 1),
10239 )
10240 .await;
10241 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
10242 let pending = forwarding
10243 .begin_route_bind_relay_for_test(
10244 client_ctx.connection_id,
10245 client_ctx.egress.clone(),
10246 2,
10247 "target",
10248 )
10249 .unwrap();
10250 forwarding
10251 .complete_pending_relay(
10252 target_ctx.connection_id,
10253 pending.corr,
10254 RouteBindRelayOutcome::Accepted,
10255 )
10256 .unwrap();
10257 let actual = query_live_roots(&handler, &target_ctx).await;
10258 let ModuleControlResponseToModule::LiveRoots {
10259 roots,
10260 unknown_root_bindings,
10261 total_bindings,
10262 } = actual
10263 else {
10264 panic!("expected live roots")
10265 };
10266 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
10267 assert_eq!(
10268 unknown_root_bindings, 1,
10269 "unknown-root arm must not read as no bindings"
10270 );
10271 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
10272 assert_eq!(
10273 total_bindings,
10274 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10275 );
10276 }
10277
10278 #[tokio::test(start_paused = true)]
10286 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
10287 let forwarding = Arc::new(ForwardingTable::default());
10288 let handler =
10289 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10290 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
10291 let replies = handler
10292 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
10293 .await
10294 .unwrap();
10295 let queued_by_hello = module_rx.len();
10296
10297 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
10298 let open_handler = handler.clone();
10299 let open = tokio::spawn(async move {
10300 open_handler
10301 .handle_control_frame(
10302 &client_ctx,
10303 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
10304 )
10305 .await
10306 .unwrap()
10307 });
10308 let mut spins = 0;
10310 while module_rx.len() == queued_by_hello {
10311 spins += 1;
10312 assert!(spins < 10_000, "route.open never queued a route.bind");
10313 tokio::task::yield_now().await;
10314 }
10315
10316 for reply in replies {
10318 module_ctx.egress.send(reply).await.unwrap();
10319 }
10320
10321 let first = module_rx.recv().await.unwrap().frame;
10322 assert_eq!(
10323 first.header.ty,
10324 FrameType::HelloAck,
10325 "the first frame a registering module reads must be its HELLO_ACK"
10326 );
10327 assert_eq!(first.header.corr, 7);
10328 let second = module_rx.recv().await.unwrap().frame;
10329 assert_eq!(second.header.ty, FrameType::Request);
10330 assert!(
10331 matches!(
10332 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
10333 ModuleControlRequest::RouteBind { .. }
10334 ),
10335 "the route.bind follows the ack"
10336 );
10337 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
10338
10339 handler
10340 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
10341 .await
10342 .unwrap();
10343 assert!(open.await.unwrap().is_empty());
10344 let _ = client_rx.recv().await.unwrap();
10345 }
10346
10347 #[tokio::test(start_paused = true)]
10348 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
10349 let handler = ControlHandler::with_forwarding(
10350 Arc::new(Registry::default()),
10351 Arc::new(ForwardingTable::default()),
10352 );
10353 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
10354 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
10355 hello_via_sink(
10356 &handler,
10357 &first_ctx,
10358 &mut first_rx,
10359 hello_frame("first", PROTOCOL_VERSION, 1),
10360 )
10361 .await;
10362 hello_via_sink(
10363 &handler,
10364 &second_ctx,
10365 &mut second_rx,
10366 hello_frame("second", PROTOCOL_VERSION, 2),
10367 )
10368 .await;
10369 let root = unique_project_root("second-only");
10370 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
10371 let cloned = handler.clone();
10372 let open = tokio::spawn(async move {
10373 cloned
10374 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
10375 .await
10376 .unwrap()
10377 });
10378 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
10379 .await
10380 .unwrap()
10381 .unwrap();
10382 let first = query_live_roots(&handler, &first_ctx).await;
10383 let second = query_live_roots(&handler, &second_ctx).await;
10384 assert!(
10385 matches!(
10386 first,
10387 ModuleControlResponseToModule::LiveRoots {
10388 total_bindings: 0,
10389 ..
10390 }
10391 ),
10392 "cross-module scope must not expose another module's roots"
10393 );
10394 assert!(
10395 matches!(
10396 second,
10397 ModuleControlResponseToModule::LiveRoots {
10398 total_bindings: 1,
10399 ..
10400 }
10401 ),
10402 "second module must see its pending route"
10403 );
10404 handler
10405 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
10406 .await
10407 .unwrap();
10408 assert!(open.await.unwrap().is_empty());
10409 }
10410
10411 #[tokio::test(start_paused = true)]
10412 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
10413 let handler = ControlHandler::with_forwarding(
10414 Arc::new(Registry::default()),
10415 Arc::new(ForwardingTable::default()),
10416 );
10417 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
10418 hello_via_sink(
10419 &handler,
10420 &target_ctx,
10421 &mut target_rx,
10422 hello_frame("target", PROTOCOL_VERSION, 1),
10423 )
10424 .await;
10425 let actual = query_live_roots(&handler, &target_ctx).await;
10426 let ModuleControlResponseToModule::LiveRoots {
10427 roots,
10428 unknown_root_bindings,
10429 total_bindings,
10430 } = actual
10431 else {
10432 panic!("expected live roots")
10433 };
10434 assert!(roots.is_empty());
10435 assert_eq!(unknown_root_bindings, 0);
10436 assert_eq!(total_bindings, 0);
10437 assert_eq!(
10438 total_bindings,
10439 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10440 );
10441 }
10442
10443 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
10450 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10451 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
10452 let text = std::fs::read_to_string(&path)
10453 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
10454 let vectors: Vec<(String, Value)> = text
10455 .lines()
10456 .filter(|line| !line.trim().is_empty())
10457 .map(|line| {
10458 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
10459 let id = entry["corpus_id"]
10460 .as_str()
10461 .expect("every vector carries a corpus_id")
10462 .to_string();
10463 (id, entry["package"].clone())
10464 })
10465 .collect();
10466 assert_eq!(
10470 vectors.len(),
10471 3,
10472 "vendored fed corpus changed size; re-sync from subc-federation"
10473 );
10474
10475 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
10490 let richest = vectors
10491 .iter()
10492 .filter_map(|(_, package)| package.as_object())
10493 .map(|object| {
10494 object
10495 .keys()
10496 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
10497 .count()
10498 })
10499 .max()
10500 .unwrap_or(0);
10501 assert!(
10502 richest >= 2,
10503 "vendored corpus no longer carries a package with unmodelled fields, \
10504 so the relay test can no longer distinguish a verbatim relay from a lossy one"
10505 );
10506
10507 vectors
10508 }
10509
10510 #[tokio::test]
10518 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
10519 for (corpus_id, package) in fed_admission_facts_vectors() {
10520 let registry = Arc::new(Registry::default());
10521 let forwarding = Arc::new(ForwardingTable::default());
10522 let supervisor = SupervisorHandle::new();
10523 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10524 let handler =
10525 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10526 .with_supervisor(supervisor)
10527 .with_admission_facts_config(
10528 Some("fed".to_string()),
10529 Some(vec!["target".to_string()]),
10530 );
10531
10532 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10533 hello_via_sink(
10534 &handler,
10535 &target_ctx,
10536 &mut target_rx,
10537 hello_frame("target", PROTOCOL_VERSION, 1),
10538 )
10539 .await;
10540
10541 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
10542 let route_handler = handler.clone();
10543 let expected = package.clone();
10544 let route_task = tokio::spawn(async move {
10545 route_handler
10546 .handle_control_frame(
10547 &client_ctx,
10548 route_open_frame_with_admission_facts(
10549 20,
10550 "target",
10551 unique_project_root("admission-facts"),
10552 Some(subc_control::ConsumerIdentity {
10553 module_id: "fed".to_string(),
10554 launch_nonce: "fed-nonce".to_string(),
10555 }),
10556 Some(package),
10557 ),
10558 )
10559 .await
10560 .unwrap()
10561 });
10562
10563 let bind_frame = target_rx.recv().await.unwrap();
10564 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10565 let ModuleControlRequest::RouteBind {
10566 admission_facts, ..
10567 } = bind
10568 else {
10569 panic!("{corpus_id}: expected route.bind")
10570 };
10571 assert_eq!(
10572 admission_facts,
10573 Some(expected),
10574 "{corpus_id}: relay must not add, drop or reshape any field"
10575 );
10576
10577 handler
10578 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10579 .await
10580 .unwrap();
10581 route_task.await.unwrap();
10582 }
10583 }
10584
10585 #[tokio::test]
10586 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
10587 let registry = Arc::new(Registry::default());
10588 let forwarding = Arc::new(ForwardingTable::default());
10589 let supervisor = SupervisorHandle::new();
10590 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10591 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
10592 let handler =
10593 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10594 .with_supervisor(supervisor)
10595 .with_admission_facts_config(
10596 Some("fed".to_string()),
10597 Some(vec!["target".to_string()]),
10598 );
10599
10600 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
10601 hello_via_sink(
10602 &handler,
10603 &target_ctx,
10604 &mut target_rx,
10605 hello_frame("target", PROTOCOL_VERSION, 1),
10606 )
10607 .await;
10608 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
10609 hello_via_sink(
10610 &handler,
10611 &other_ctx,
10612 &mut other_rx,
10613 hello_frame("other", PROTOCOL_VERSION, 2),
10614 )
10615 .await;
10616
10617 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
10618 let expected_facts = facts.clone();
10619 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
10620 let route_handler = handler.clone();
10621 let route_task = tokio::spawn(async move {
10622 route_handler
10623 .handle_control_frame(
10624 &client_ctx,
10625 route_open_frame_with_admission_facts(
10626 10,
10627 "target",
10628 unique_project_root("admission-facts"),
10629 Some(subc_control::ConsumerIdentity {
10630 module_id: "fed".to_string(),
10631 launch_nonce: "fed-nonce".to_string(),
10632 }),
10633 Some(facts.clone()),
10634 ),
10635 )
10636 .await
10637 .unwrap()
10638 });
10639 let bind_frame = target_rx.recv().await.unwrap();
10640 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10641 let ModuleControlRequest::RouteBind {
10642 admission_facts, ..
10643 } = bind
10644 else {
10645 panic!("expected route.bind")
10646 };
10647 assert_eq!(admission_facts, Some(expected_facts));
10648 handler
10649 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10650 .await
10651 .unwrap();
10652 assert!(route_task.await.unwrap().is_empty());
10653 assert!(matches!(
10654 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
10655 .unwrap(),
10656 ClientControlResponse::RouteOpen { .. }
10657 ));
10658
10659 let direct = handler
10660 .handle_control_frame(
10661 &route_ctx(ConnectionId::new(73)).0,
10662 route_open_frame_with_admission_facts(
10663 11,
10664 "target",
10665 unique_project_root("admission-facts"),
10666 None,
10667 Some(json!({"x": 1})),
10668 ),
10669 )
10670 .await
10671 .unwrap();
10672 assert_eq!(
10673 parse_error(&direct[0])["code"],
10674 "admission_facts_not_permitted"
10675 );
10676
10677 let different_reserved = handler
10678 .handle_control_frame(
10679 &route_ctx(ConnectionId::new(77)).0,
10680 route_open_frame_with_admission_facts(
10681 15,
10682 "target",
10683 unique_project_root("admission-facts"),
10684 Some(subc_control::ConsumerIdentity {
10685 module_id: "other".to_string(),
10686 launch_nonce: "other-nonce".to_string(),
10687 }),
10688 Some(json!({"x": 1})),
10689 ),
10690 )
10691 .await
10692 .unwrap();
10693 assert_eq!(
10694 parse_error(&different_reserved[0])["code"],
10695 "admission_facts_not_permitted"
10696 );
10697
10698 let other_target = handler
10699 .handle_control_frame(
10700 &route_ctx(ConnectionId::new(74)).0,
10701 route_open_frame_with_admission_facts(
10702 12,
10703 "other",
10704 unique_project_root("admission-facts"),
10705 Some(subc_control::ConsumerIdentity {
10706 module_id: "fed".to_string(),
10707 launch_nonce: "fed-nonce".to_string(),
10708 }),
10709 Some(json!({"x": 1})),
10710 ),
10711 )
10712 .await
10713 .unwrap();
10714 assert_eq!(
10715 parse_error(&other_target[0])["code"],
10716 "admission_facts_target_not_allowed"
10717 );
10718
10719 let nonexistent = handler
10720 .handle_control_frame(
10721 &route_ctx(ConnectionId::new(75)).0,
10722 route_open_frame_with_admission_facts(
10723 13,
10724 "missing",
10725 unique_project_root("admission-facts"),
10726 None,
10727 Some(json!({"x": 1})),
10728 ),
10729 )
10730 .await
10731 .unwrap();
10732 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
10733
10734 let described = handler
10735 .handle_control_frame(
10736 &route_ctx(ConnectionId::new(76)).0,
10737 Frame::build(
10738 FrameType::Request,
10739 control_flags(),
10740 0,
10741 0,
10742 14,
10743 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10744 )
10745 .unwrap(),
10746 )
10747 .await
10748 .unwrap();
10749 let ClientControlResponse::ServerDescribe { capabilities, .. } =
10750 serde_json::from_slice(&described[0].body).unwrap()
10751 else {
10752 panic!("expected server.describe response")
10753 };
10754 assert!(capabilities
10755 .iter()
10756 .any(|cap| cap == "admission_facts_relay_v1"));
10757 }
10758
10759 #[tokio::test]
10760 async fn admission_facts_without_configured_carrier_are_rejected() {
10761 let registry = Arc::new(Registry::default());
10762 let forwarding = Arc::new(ForwardingTable::default());
10763 let handler = ControlHandler::with_forwarding(registry, forwarding);
10764 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
10765 hello_via_sink(
10766 &handler,
10767 &target_ctx,
10768 &mut target_rx,
10769 hello_frame("target", PROTOCOL_VERSION, 1),
10770 )
10771 .await;
10772
10773 let responses = handler
10774 .handle_control_frame(
10775 &route_ctx(ConnectionId::new(79)).0,
10776 route_open_frame_with_admission_facts(
10777 16,
10778 "target",
10779 unique_project_root("admission-facts"),
10780 None,
10781 Some(json!({"x": 1})),
10782 ),
10783 )
10784 .await
10785 .unwrap();
10786 assert_eq!(
10787 parse_error(&responses[0])["code"],
10788 "admission_facts_not_permitted"
10789 );
10790 }
10791
10792 #[tokio::test]
10793 async fn route_open_relays_consumer_capabilities_verbatim() {
10794 let registry = Arc::new(Registry::default());
10795 let forwarding = Arc::new(ForwardingTable::default());
10796 let handler =
10797 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10798 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
10799 hello_via_sink(
10800 &handler,
10801 &module_ctx,
10802 &mut module_rx,
10803 hello_frame("aft", PROTOCOL_VERSION, 7),
10804 )
10805 .await;
10806
10807 let expected = vec!["elicitation".to_string(), "roots".to_string()];
10808 let expected_for_request = expected.clone();
10809 let project_root = unique_project_root("consumer-capabilities-present");
10810 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
10811 let route_handler = handler.clone();
10812 let route_task = tokio::spawn(async move {
10813 route_handler
10814 .handle_control_frame(
10815 &client_ctx,
10816 route_open_frame_with_consumer_capabilities(
10817 401,
10818 "aft",
10819 project_root,
10820 Some(expected_for_request),
10821 ),
10822 )
10823 .await
10824 .unwrap()
10825 });
10826 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10827 .await
10828 .unwrap()
10829 .unwrap();
10830 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10831 let ModuleControlRequest::RouteBind {
10832 consumer_capabilities,
10833 ..
10834 } = bind
10835 else {
10836 panic!("expected route.bind request, got {bind:?}");
10837 };
10838 assert_eq!(consumer_capabilities, Some(expected.clone()));
10839
10840 handler
10841 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10842 .await
10843 .unwrap();
10844 let route_response = route_task.await.unwrap();
10845 assert!(route_response.is_empty());
10846 let published = client_rx.recv().await.unwrap();
10847 assert!(matches!(
10848 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10849 ClientControlResponse::RouteOpen { .. }
10850 ));
10851 }
10852
10853 #[tokio::test]
10854 async fn route_open_without_consumer_capabilities_relays_none() {
10855 let registry = Arc::new(Registry::default());
10856 let forwarding = Arc::new(ForwardingTable::default());
10857 let handler =
10858 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10859 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
10860 hello_via_sink(
10861 &handler,
10862 &module_ctx,
10863 &mut module_rx,
10864 hello_frame("aft", PROTOCOL_VERSION, 7),
10865 )
10866 .await;
10867
10868 let project_root = unique_project_root("consumer-capabilities-absent");
10869 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
10870 let route_handler = handler.clone();
10871 let route_task = tokio::spawn(async move {
10872 route_handler
10873 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
10874 .await
10875 .unwrap()
10876 });
10877 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10878 .await
10879 .unwrap()
10880 .unwrap();
10881 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10882 let ModuleControlRequest::RouteBind {
10883 consumer_capabilities,
10884 ..
10885 } = bind
10886 else {
10887 panic!("expected route.bind request, got {bind:?}");
10888 };
10889 assert_eq!(consumer_capabilities, None);
10890
10891 handler
10892 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10893 .await
10894 .unwrap();
10895 let route_response = route_task.await.unwrap();
10896 assert!(route_response.is_empty());
10897 let published = client_rx.recv().await.unwrap();
10898 assert!(matches!(
10899 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10900 ClientControlResponse::RouteOpen { .. }
10901 ));
10902 }
10903
10904 async fn bind_role_versions_for(
10908 sent: Option<BTreeMap<String, String>>,
10909 connection: u64,
10910 ) -> Option<BTreeMap<String, String>> {
10911 let registry = Arc::new(Registry::default());
10912 let forwarding = Arc::new(ForwardingTable::default());
10913 let handler =
10914 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10915 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
10916 hello_via_sink(
10917 &handler,
10918 &module_ctx,
10919 &mut module_rx,
10920 hello_frame("aft", PROTOCOL_VERSION, 7),
10921 )
10922 .await;
10923 let project_root = unique_project_root("role-versions");
10924 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
10925 let route_handler = handler.clone();
10926 let route_task = tokio::spawn(async move {
10927 route_handler
10928 .handle_control_frame(
10929 &client_ctx,
10930 route_open_frame_with_role_versions(403, "aft", project_root, sent),
10931 )
10932 .await
10933 .unwrap()
10934 });
10935 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10936 .await
10937 .expect("a well-formed route.open reaches the module as a bind")
10938 .unwrap();
10939 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10940 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
10941 panic!("expected route.bind request, got {bind:?}");
10942 };
10943 handler
10944 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10945 .await
10946 .unwrap();
10947 assert!(route_task.await.unwrap().is_empty());
10948 let published = client_rx.recv().await.unwrap();
10949 assert!(matches!(
10950 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10951 ClientControlResponse::RouteOpen { .. }
10952 ));
10953 role_versions
10954 }
10955
10956 #[tokio::test]
10957 async fn route_open_relays_role_versions_verbatim() {
10958 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
10959 assert_eq!(
10960 bind_role_versions_for(Some(sent.clone()), 141).await,
10961 Some(sent)
10962 );
10963 }
10964
10965 #[tokio::test]
10968 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
10969 assert_eq!(bind_role_versions_for(None, 143).await, None);
10970 assert_eq!(
10971 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
10972 None
10973 );
10974 }
10975
10976 #[tokio::test]
10977 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
10978 let registry = Arc::new(Registry::default());
10979 let forwarding = Arc::new(ForwardingTable::default());
10980 let handler =
10981 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10982 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
10983 hello_via_sink(
10984 &handler,
10985 &module_ctx,
10986 &mut module_rx,
10987 hello_frame("aft", PROTOCOL_VERSION, 7),
10988 )
10989 .await;
10990
10991 let nine: BTreeMap<String, String> = (0..9)
10992 .map(|index| (format!("role-{index}"), "v1".to_string()))
10993 .collect();
10994 for (label, malformed) in [
10995 (
10996 "invalid role name",
10997 role_versions(&[("Tool_Provider", "v1")]),
10998 ),
10999 ("invalid version", role_versions(&[("tool-provider", "v0")])),
11000 ("nine entries", nine),
11001 ] {
11002 let responses = tokio::time::timeout(
11005 Duration::from_secs(1),
11006 handler.handle_control_frame(
11007 &route_ctx(ConnectionId::new(148)).0,
11008 route_open_frame_with_role_versions(
11009 404,
11010 "aft",
11011 unique_project_root("role-versions-malformed"),
11012 Some(malformed),
11013 ),
11014 ),
11015 )
11016 .await
11017 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
11018 .unwrap();
11019 assert_eq!(responses.len(), 1, "{label}");
11020 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
11021 let error = parse_error(&responses[0]);
11022 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
11023 assert_eq!(
11024 error["detail"]["field"], "role_versions",
11025 "{label}: {error}"
11026 );
11027 assert!(
11028 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
11029 "{label}: a malformed declaration is terminal"
11030 );
11031 assert!(
11032 module_rx.try_recv().is_err(),
11033 "{label}: the module must never see a bind"
11034 );
11035 }
11036 }
11037
11038 #[tokio::test]
11042 async fn route_role_versions_capability_is_advertised() {
11043 let handler = ControlHandler::new(Arc::new(Registry::default()));
11044 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
11045 let ack = hello_via_sink(
11046 &handler,
11047 &ctx,
11048 &mut rx,
11049 hello_frame("m", PROTOCOL_VERSION, 1),
11050 )
11051 .await;
11052 let ack = parse_ack(&ack);
11053 assert!(
11054 ack.subc_capabilities
11055 .iter()
11056 .any(|c| c == "route-role-versions/v1"),
11057 "{:?}",
11058 ack.subc_capabilities
11059 );
11060
11061 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11062 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11063 let reply = handler
11064 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
11065 .await
11066 .unwrap()
11067 .pop()
11068 .unwrap();
11069 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11070 serde_json::from_slice(&reply.body).unwrap()
11071 else {
11072 panic!("not a server.describe reply");
11073 };
11074 assert!(
11075 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
11076 "{capabilities:?}"
11077 );
11078 }
11079
11080 #[tokio::test]
11084 async fn catalog_events_capability_is_advertised() {
11085 let handler = ControlHandler::new(Arc::new(Registry::default()));
11086 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11087 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11088 let reply = handler
11089 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
11090 .await
11091 .unwrap()
11092 .pop()
11093 .unwrap();
11094 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11095 serde_json::from_slice(&reply.body).unwrap()
11096 else {
11097 panic!("not a server.describe reply");
11098 };
11099 assert!(
11100 capabilities
11101 .iter()
11102 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
11103 "{capabilities:?}"
11104 );
11105 }
11106
11107 #[tokio::test]
11108 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
11109 let registry = Arc::new(Registry::default());
11110 let forwarding = Arc::new(ForwardingTable::default());
11111 let handler =
11112 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11113 .with_health_probe_timeout(Duration::from_secs(5));
11114 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
11115 hello_via_sink(
11116 &handler,
11117 &module_ctx,
11118 &mut module_rx,
11119 non_routable_hello_frame_with_control_ops(
11120 "mcp",
11121 300,
11122 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11123 ),
11124 )
11125 .await;
11126 assert!(registry
11127 .get_module("mcp")
11128 .unwrap()
11129 .unwrap()
11130 .manifest
11131 .provides
11132 .is_empty());
11133
11134 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
11135 let route_response = handler
11136 .handle_control_frame(
11137 &route_client_ctx,
11138 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
11139 )
11140 .await
11141 .unwrap();
11142 assert_eq!(route_response[0].header.ty, FrameType::Error);
11143 assert_eq!(
11144 parse_error(&route_response[0])["code"],
11145 "target_unavailable"
11146 );
11147 assert!(parse_error(&route_response[0])["message"]
11148 .as_str()
11149 .unwrap()
11150 .contains("does not provide the requested target"));
11151 assert!(module_rx.try_recv().is_err());
11152
11153 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
11154 let health_handler = handler.clone();
11155 let health_task = tokio::spawn(async move {
11156 health_handler
11157 .handle_control_frame(
11158 &health_client_ctx,
11159 supervisor_health_probe_frame(302, "mcp"),
11160 )
11161 .await
11162 .unwrap()
11163 });
11164 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11165 .await
11166 .unwrap()
11167 .unwrap();
11168 assert_eq!(
11169 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11170 ModuleControlRequest::HealthCheck {}
11171 );
11172 handler
11173 .handle_control_frame(
11174 &module_ctx,
11175 health_response(health_frame.header.corr, HealthStatus::Ok),
11176 )
11177 .await
11178 .unwrap();
11179 let health_response = health_task.await.unwrap();
11180 assert_eq!(health_response[0].header.ty, FrameType::Response);
11181 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11182 ClientControlResponse::SupervisorHealthProbe {
11183 module_id, status, ..
11184 } => {
11185 assert_eq!(module_id, "mcp");
11186 assert_eq!(status, HealthStatus::Ok);
11187 }
11188 other => panic!("unexpected health response: {other:?}"),
11189 }
11190
11191 forwarding
11196 .cleanup_connection(module_ctx.connection_id)
11197 .unwrap();
11198 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
11199 let cleanup_response = tokio::time::timeout(
11200 Duration::from_millis(200),
11201 handler.handle_control_frame(
11202 &cleanup_probe_ctx,
11203 supervisor_health_probe_frame(303, "mcp"),
11204 ),
11205 )
11206 .await
11207 .expect("probe should fail immediately when the control lane is gone")
11208 .unwrap();
11209 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
11210 assert_eq!(
11211 parse_error(&cleanup_response[0])["code"],
11212 "target_unavailable"
11213 );
11214 assert!(parse_error(&cleanup_response[0])["message"]
11215 .as_str()
11216 .unwrap()
11217 .contains("no module connection"));
11218
11219 handler
11220 .cleanup_connection(module_ctx.connection_id)
11221 .unwrap();
11222 }
11223
11224 #[tokio::test]
11225 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
11226 let registry = Arc::new(Registry::default());
11227 let supervisor_handle = SupervisorHandle::new();
11228 let supervisor =
11229 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11230 .with_handle(supervisor_handle.clone())
11231 .with_connection_file_path(
11232 std::env::temp_dir()
11233 .join(format!("subc-route-open-warming-{}", std::process::id())),
11234 );
11235 let module = supervisor
11236 .supervise_configured(
11237 ModuleSpec {
11238 module_id: "warming".to_string(),
11239 program: fake_aft_stub_path(),
11240 args: Vec::new(),
11241 env: Vec::new(),
11242 reserved: false,
11243 reserved_prefixes: Vec::new(),
11244 protocol: ModuleProtocol::Subc,
11245 overlap: Default::default(),
11246 },
11247 true,
11248 )
11249 .unwrap();
11250 assert_eq!(module.state().unwrap(), ModuleState::Running);
11251
11252 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11253 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
11254 let response = handler
11255 .handle_control_frame(
11256 &ctx,
11257 route_open_frame(304, "warming", unique_project_root("warming")),
11258 )
11259 .await
11260 .unwrap();
11261 module.stop().await.unwrap();
11262
11263 assert_eq!(response[0].header.ty, FrameType::Error);
11264 let error = parse_error(&response[0]);
11265 assert_eq!(error["code"], "module_warming");
11266 assert!(error["message"]
11267 .as_str()
11268 .unwrap()
11269 .contains("state=running, enabled=true, live=false"));
11270 }
11271
11272 #[test]
11273 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
11274 let handler = ControlHandler::new(Arc::new(Registry::default()));
11275 let capture = EventCapture::default();
11276 let _subscriber =
11277 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11278 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
11279 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
11280 let pending = (0..limit).collect::<Vec<_>>();
11281 let response = handler
11282 .route_open_capacity_refusal(
11283 &ctx,
11284 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
11285 "busy",
11286 pending.len(),
11287 limit,
11288 )
11289 .unwrap();
11290 assert_eq!(parse_error(&response)["code"], "target_unavailable");
11291 let event = capture
11292 .events()
11293 .into_iter()
11294 .find(|event| {
11295 event.target == "control"
11296 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
11297 })
11298 .expect("connection admission refusal event");
11299 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11300 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11301 }
11302
11303 #[test]
11304 fn route_open_target_cap_logs_admission_reason_and_capacity() {
11305 let handler = ControlHandler::new(Arc::new(Registry::default()));
11306 let capture = EventCapture::default();
11307 let _subscriber =
11308 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11309 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
11310 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
11311 let guards = (0..limit)
11312 .map(|_| {
11313 handler
11314 .route_bind_concurrency
11315 .try_admit("busy", limit)
11316 .unwrap()
11317 })
11318 .collect::<Vec<_>>();
11319 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
11320 Err(in_flight) => in_flight,
11321 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
11322 };
11323 let response = handler
11324 .route_open_target_capacity_refusal(
11325 &ctx,
11326 &route_open_frame(397, "busy", unique_project_root("target-cap")),
11327 "busy",
11328 in_flight,
11329 )
11330 .unwrap();
11331 assert_eq!(parse_error(&response)["code"], "target_unavailable");
11332 let event = capture
11333 .events()
11334 .into_iter()
11335 .find(|event| {
11336 event.target == "control"
11337 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
11338 })
11339 .expect("target admission refusal event");
11340 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11341 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11342 drop(guards);
11343 }
11344
11345 #[tokio::test(flavor = "current_thread")]
11346 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
11347 let handler = ControlHandler::new(Arc::new(Registry::default()));
11348 let capture = EventCapture::default();
11349 let _subscriber =
11350 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11351 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
11352 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11353
11354 let _ = handler
11355 .handle_client_control_request(
11356 &ctx,
11357 frame,
11358 ClientControlRequest::SupervisorSetEnabled {
11359 module_id: "broca".to_string(),
11360 enabled: false,
11361 },
11362 )
11363 .await
11364 .unwrap();
11365
11366 let events = capture
11367 .events()
11368 .into_iter()
11369 .filter(|event| {
11370 event.target == "control"
11371 && event.fields.get("message").map(String::as_str)
11372 == Some("supervisor request received")
11373 })
11374 .collect::<Vec<_>>();
11375 assert_eq!(events.len(), 1, "one supervisor request log line");
11376 let event = &events[0];
11377 assert_eq!(event.level, tracing::Level::INFO);
11378 assert_eq!(
11379 event.fields.get("op"),
11380 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
11381 );
11382 assert_eq!(
11383 event.fields.get("module_id"),
11384 Some(&"\"broca\"".to_string())
11385 );
11386 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
11387 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
11388 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
11389 }
11390
11391 #[tokio::test(flavor = "current_thread")]
11392 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
11393 let registry = Arc::new(Registry::default());
11394 let module_connection = ConnectionId::new(303);
11395 registry
11396 .register_with_control_ops(
11397 manifest("aft", PROTOCOL_VERSION),
11398 PROTOCOL_VERSION,
11399 module_connection,
11400 module_baseline_control_ops(),
11401 )
11402 .unwrap();
11403 let handler = ControlHandler::new(registry);
11404 let capture = EventCapture::default();
11405 let _subscriber =
11406 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11407 let (ctx, _rx) = route_ctx(module_connection);
11408 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11409
11410 let _ = handler
11411 .handle_client_control_request(
11412 &ctx,
11413 frame,
11414 ClientControlRequest::SupervisorSetEnabled {
11415 module_id: "broca".to_string(),
11416 enabled: false,
11417 },
11418 )
11419 .await
11420 .unwrap();
11421
11422 let callers = capture
11423 .events()
11424 .into_iter()
11425 .filter(|event| {
11426 event.fields.get("message").map(String::as_str)
11427 == Some("supervisor request received")
11428 })
11429 .map(|event| event.fields.get("caller").cloned())
11430 .collect::<Vec<_>>();
11431 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
11432 }
11433
11434 #[tokio::test(flavor = "current_thread")]
11435 async fn supervisor_list_does_not_log_request_received() {
11436 let handler = ControlHandler::new(Arc::new(Registry::default()));
11437 let capture = EventCapture::default();
11438 let _subscriber =
11439 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11440 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
11441 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
11442
11443 handler
11444 .handle_client_control_request(&ctx, frame, ClientControlRequest::SupervisorList {})
11445 .await
11446 .unwrap();
11447
11448 assert!(capture.events().into_iter().all(|event| {
11449 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11450 }));
11451 }
11452
11453 #[tokio::test(flavor = "current_thread")]
11454 async fn supervisor_rescan_preview_does_not_log_request_received() {
11455 let handler = ControlHandler::new(Arc::new(Registry::default()));
11456 let capture = EventCapture::default();
11457 let _subscriber =
11458 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11459 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
11460 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
11461
11462 handler
11463 .handle_client_control_request(
11464 &ctx,
11465 frame,
11466 ClientControlRequest::SupervisorRescan { preview: true },
11467 )
11468 .await
11469 .unwrap();
11470
11471 assert!(capture.events().into_iter().all(|event| {
11472 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11473 }));
11474 }
11475
11476 struct ApproveOperator;
11477 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
11478 fn prompt(
11479 &self,
11480 _: &str,
11481 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
11482 ) -> crate::operator_confirm::ProviderResult {
11483 crate::operator_confirm::ProviderResult::Approved
11484 }
11485 }
11486
11487 #[tokio::test]
11488 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
11489 assert_eq!(
11490 ControlHandler::default().privileged_operator_prompt,
11491 PrivilegedOperatorPrompt::Log
11492 );
11493 for (initial, changed, running) in [
11494 ("log", "enforce", PrivilegedOperatorPrompt::Log),
11495 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
11496 ] {
11497 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
11498 let path = dir.join("subc.jsonc");
11499 std::fs::write(
11500 &path,
11501 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
11502 )
11503 .unwrap();
11504 let config = crate::daemon_config::load(&path).unwrap().unwrap();
11505 let registry = Arc::new(Registry::default());
11506 let handle = SupervisorHandle::new();
11507 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
11508 .with_handle(handle.clone());
11509 let handler = ControlHandler::new(registry)
11510 .with_supervisor(handle)
11511 .with_privileged_operator_prompt(config.privileged_operator_prompt)
11512 .with_supervisor_rescan(supervisor, &path, config.port);
11513 handler.forwarding.operator_confirms().configure(
11514 Arc::new(ApproveOperator),
11515 crate::operator_confirm::OperatorLimits::default(),
11516 );
11517 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
11518
11519 for (disk_mode, needs_restart) in [
11520 (Some(initial), false),
11521 (Some(changed), true),
11522 (Some(changed), true),
11523 (None, running == PrivilegedOperatorPrompt::Enforce),
11524 (Some(initial), false),
11525 ] {
11526 let mut doc = json!({"version":1});
11527 if let Some(mode) = disk_mode {
11528 doc["privileged_operator_prompt"] = json!(mode);
11529 }
11530 std::fs::write(&path, doc.to_string()).unwrap();
11531 for preview in [true, false] {
11532 let request = ClientControlRequest::SupervisorRescan { preview };
11533 let frame = Frame::build(
11534 FrameType::Request,
11535 control_flags(),
11536 0,
11537 0,
11538 1,
11539 serde_json::to_vec(&request).unwrap(),
11540 )
11541 .unwrap();
11542 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
11543 assert_eq!(frames.len(), 1);
11544 let ClientControlResponse::SupervisorRescan { result } =
11545 serde_json::from_slice(&frames[0].body).unwrap()
11546 else {
11547 panic!("expected a rescan response");
11548 };
11549 let expected = if needs_restart {
11550 vec!["privileged_operator_prompt"]
11551 } else {
11552 Vec::new()
11553 };
11554 assert_eq!(result.restart_required, expected);
11555 assert_eq!(handler.privileged_operator_prompt, running);
11556 assert_eq!(
11557 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
11558 running
11559 );
11560 }
11561 }
11562 }
11563 }
11564
11565 #[tokio::test]
11566 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
11567 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
11568 let path = dir.join("subc.jsonc");
11569 let registry = Arc::new(Registry::default());
11570 let handle = SupervisorHandle::new();
11571 let supervisor =
11572 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
11573 let handler = ControlHandler::new(registry)
11574 .with_supervisor(handle)
11575 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
11576 .with_supervisor_rescan(supervisor, &path, None);
11577 handler.forwarding.operator_confirms().configure(
11578 Arc::new(ApproveOperator),
11579 crate::operator_confirm::OperatorLimits::default(),
11580 );
11581 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
11582 for preview in [true, false] {
11583 std::fs::write(
11584 &path,
11585 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
11586 )
11587 .unwrap();
11588 let request = ClientControlRequest::SupervisorRescan { preview };
11589 let frame = Frame::build(
11590 FrameType::Request,
11591 control_flags(),
11592 0,
11593 0,
11594 1,
11595 serde_json::to_vec(&request).unwrap(),
11596 )
11597 .unwrap();
11598 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
11599 assert_eq!(frames.len(), 1);
11600 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
11601 assert_eq!(
11602 handler.privileged_operator_prompt,
11603 PrivilegedOperatorPrompt::Enforce
11604 );
11605 }
11606 }
11607
11608 #[tokio::test]
11613 async fn route_open_refusal_names_the_check_that_refused() {
11614 let handler = ControlHandler::new(Arc::new(Registry::default()));
11615 let capture = EventCapture::default();
11616 let _subscriber =
11617 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11618 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
11619 let response = handler
11620 .handle_control_frame(
11621 &ctx,
11622 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
11623 )
11624 .await
11625 .unwrap();
11626
11627 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11628 let event = capture
11629 .events()
11630 .into_iter()
11631 .find(|event| {
11632 event.target == "control"
11633 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
11634 })
11635 .expect("route.open refusal event");
11636 assert_eq!(
11637 event.fields.get("reason"),
11638 Some(&"\"not_registered\"".to_string())
11639 );
11640 }
11641
11642 #[tokio::test]
11643 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
11644 let registry = Arc::new(Registry::default());
11645 let supervisor_handle = SupervisorHandle::new();
11646 let supervisor =
11647 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11648 .with_handle(supervisor_handle.clone())
11649 .with_connection_file_path(std::env::temp_dir().join(format!(
11650 "subc-route-open-refusal-info-{}",
11651 std::process::id()
11652 )));
11653 let module = supervisor
11654 .supervise_configured(
11655 ModuleSpec {
11656 module_id: "warming".to_string(),
11657 program: fake_aft_stub_path(),
11658 args: Vec::new(),
11659 env: Vec::new(),
11660 reserved: false,
11661 reserved_prefixes: Vec::new(),
11662 protocol: ModuleProtocol::Subc,
11663 overlap: Default::default(),
11664 },
11665 true,
11666 )
11667 .unwrap();
11668 assert_eq!(module.state().unwrap(), ModuleState::Running);
11669
11670 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11671 assert!(handler
11672 .counters()
11673 .snapshot()
11674 .get("route_open_refused_by_code")
11675 .is_none());
11676 let capture = EventCapture::default();
11677 let _subscriber =
11678 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11679 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
11680 let response = handler
11681 .handle_control_frame(
11682 &ctx,
11683 route_open_frame(394, "warming", unique_project_root("refusal-info")),
11684 )
11685 .await
11686 .unwrap();
11687 module.stop().await.unwrap();
11688
11689 assert_eq!(parse_error(&response[0])["code"], "module_warming");
11690 let event = capture
11691 .events()
11692 .into_iter()
11693 .find(|event| {
11694 event.target == "control"
11695 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
11696 })
11697 .expect("route.open refusal event");
11698 assert_eq!(
11699 event.fields.get("module_id"),
11700 Some(&"\"warming\"".to_string())
11701 );
11702 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
11703 assert_eq!(
11704 event.fields.get("reason"),
11705 Some(&"\"supervised_not_registered\"".to_string())
11706 );
11707 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
11708 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
11709 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
11710 assert_eq!(
11711 handler.counters().snapshot()["route_open_refused_by_code"],
11712 json!({ "module_warming": 1 })
11713 );
11714 }
11715
11716 const OUTAGE_START: &str = "route.open refusing module: not serving";
11717 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
11718
11719 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
11720 capture
11721 .events()
11722 .into_iter()
11723 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
11724 .collect()
11725 }
11726
11727 fn supervise_stub(
11728 registry: &Arc<Registry>,
11729 module_id: &str,
11730 enabled: bool,
11731 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
11732 let supervisor_handle = SupervisorHandle::new();
11733 let supervisor =
11734 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
11735 .with_handle(supervisor_handle.clone())
11736 .with_connection_file_path(std::env::temp_dir().join(format!(
11737 "subc-route-outage-{module_id}-{}",
11738 std::process::id()
11739 )));
11740 let module = supervisor
11741 .supervise_configured(
11742 ModuleSpec {
11743 module_id: module_id.to_string(),
11744 program: fake_aft_stub_path(),
11745 args: Vec::new(),
11746 env: Vec::new(),
11747 reserved: false,
11748 reserved_prefixes: Vec::new(),
11749 protocol: ModuleProtocol::Subc,
11750 overlap: Default::default(),
11751 },
11752 enabled,
11753 )
11754 .unwrap();
11755 (supervisor_handle, module)
11756 }
11757
11758 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
11759 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
11760 module_id: module_id.to_string(),
11761 drain_timeout_ms: Some(50),
11762 })
11763 .unwrap();
11764 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11765 }
11766
11767 #[test]
11771 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
11772 let registry = Arc::new(Registry::default());
11773 let forwarding = Arc::new(ForwardingTable::default());
11774 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11775 let second = ControlHandler::with_forwarding(registry, forwarding);
11776 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
11777 }
11778
11779 #[tokio::test(flavor = "current_thread")]
11783 async fn route_open_unknown_module_refusals_add_no_outage_state() {
11784 let handler = ControlHandler::new(Arc::new(Registry::default()));
11785 let capture = EventCapture::default();
11786 let _subscriber =
11787 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11788 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
11789 for corr in 0..8 {
11790 let response = handler
11791 .handle_control_frame(
11792 &ctx,
11793 route_open_frame(
11794 380 + corr,
11795 &format!("nobody-{corr}"),
11796 unique_project_root("outage-unknown"),
11797 ),
11798 )
11799 .await
11800 .unwrap();
11801 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11802 }
11803
11804 assert_eq!(handler.route_outages.tracked_module_count(), 0);
11805 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
11806 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
11807 }
11808
11809 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11814 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
11815 let registry = Arc::new(Registry::default());
11816 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
11817 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11818 let capture = EventCapture::default();
11819 let _subscriber =
11820 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11821 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
11822 handler.route_outages.record_accepted("outage-restart");
11825
11826 let response = handler
11827 .handle_control_frame(
11828 &ctx,
11829 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
11830 )
11831 .await
11832 .unwrap();
11833 assert_eq!(response[0].header.ty, FrameType::Error);
11834 let starts = outage_lines(&capture, OUTAGE_START);
11835 assert_eq!(starts.len(), 1, "{starts:?}");
11836 assert_eq!(starts[0].level, tracing::Level::WARN);
11837 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
11838 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
11839 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
11840 handler.route_outages.record_accepted("outage-restart");
11841 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
11842
11843 let restart = handler
11844 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
11845 .await
11846 .unwrap();
11847 assert_eq!(
11848 restart[0].header.ty,
11849 FrameType::Response,
11850 "{:?}",
11851 parse_error(&restart[0])
11852 );
11853 handler
11854 .handle_control_frame(
11855 &ctx,
11856 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
11857 )
11858 .await
11859 .unwrap();
11860 module.stop().await.unwrap();
11861
11862 let starts = outage_lines(&capture, OUTAGE_START);
11863 assert_eq!(starts.len(), 2, "{starts:?}");
11864 assert_eq!(starts[1].level, tracing::Level::INFO);
11865 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
11866 }
11867
11868 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11872 async fn failed_operator_restart_leaves_no_operator_mark() {
11873 let registry = Arc::new(Registry::default());
11874 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
11875 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11876 let capture = EventCapture::default();
11877 let _subscriber =
11878 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11879 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
11880 handler.route_outages.record_accepted("outage-disabled");
11881
11882 let restart = handler
11883 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
11884 .await
11885 .unwrap();
11886 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
11887 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
11888
11889 handler
11890 .handle_control_frame(
11891 &ctx,
11892 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
11893 )
11894 .await
11895 .unwrap();
11896 let starts = outage_lines(&capture, OUTAGE_START);
11897 assert_eq!(starts.len(), 1, "{starts:?}");
11898 assert_eq!(starts[0].level, tracing::Level::WARN);
11899 }
11900
11901 #[tokio::test(flavor = "current_thread")]
11902 async fn route_open_unknown_module_escapes_target_module_id() {
11903 let handler = ControlHandler::new(Arc::new(Registry::default()));
11904 let capture = EventCapture::default();
11905 let _subscriber =
11906 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11907 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
11908 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
11909 let response = handler
11910 .handle_control_frame(
11911 &ctx,
11912 route_open_frame(
11913 395,
11914 hostile_module_id,
11915 unique_project_root("hostile-target-module-id"),
11916 ),
11917 )
11918 .await
11919 .unwrap();
11920
11921 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11922 let event = capture
11923 .events()
11924 .into_iter()
11925 .find(|event| {
11926 event.target == "control"
11927 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
11928 })
11929 .expect("route.open unknown-module refusal event");
11930 let logged = event.fields.get("module_id").expect("module_id field");
11931 assert!(!logged.bytes().any(|byte| byte < 0x20));
11932 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
11933 }
11934
11935 #[tokio::test(flavor = "current_thread")]
11936 async fn route_open_module_rejection_uses_daemon_counter_key() {
11937 let registry = Arc::new(Registry::default());
11938 let forwarding = Arc::new(ForwardingTable::default());
11939 let handler =
11940 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11941 let module_connection = ConnectionId::new(95);
11942 let (module_ctx, mut module_rx) = route_ctx(module_connection);
11943 hello_via_sink(
11944 &handler,
11945 &module_ctx,
11946 &mut module_rx,
11947 hello_frame("aft", PROTOCOL_VERSION, 395),
11948 )
11949 .await;
11950
11951 let client_connection = ConnectionId::new(96);
11952 let (client_ctx, _client_rx) = route_ctx(client_connection);
11953 let capture = EventCapture::default();
11954 let _subscriber =
11955 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11956 let (route_task, bind) = relay_route_open(
11957 &handler,
11958 client_connection,
11959 &client_ctx.egress,
11960 &mut module_rx,
11961 396,
11962 "aft",
11963 "hostile-module-code",
11964 )
11965 .await;
11966 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
11967 let rejection = Frame::build(
11968 FrameType::Error,
11969 control_flags(),
11970 0,
11971 0,
11972 bind.header.corr,
11973 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
11974 )
11975 .unwrap();
11976 handler
11977 .handle_control_frame(&module_ctx, rejection)
11978 .await
11979 .unwrap();
11980
11981 let response = route_task.await.unwrap();
11982 assert_eq!(parse_error(&response[0])["code"], hostile_code);
11983 let counters = handler.counters().snapshot();
11984 assert_eq!(
11985 counters["route_open_refused_by_code"],
11986 json!({ "module_rejected": 1 })
11987 );
11988 assert!(counters["route_open_refused_by_code"]
11989 .get(hostile_code)
11990 .is_none());
11991
11992 let event = capture
11993 .events()
11994 .into_iter()
11995 .find(|event| {
11996 event.target == "control"
11997 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
11998 })
11999 .expect("route.open module-rejection refusal event");
12000 let logged = event.fields.get("module_code").expect("module_code field");
12001 assert!(!logged.bytes().any(|byte| byte < 0x20));
12002 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12003 }
12004
12005 #[tokio::test]
12006 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
12007 let registry = Arc::new(Registry::default());
12008 let supervisor_handle = SupervisorHandle::new();
12009 let missing_program = std::env::temp_dir().join(format!(
12010 "subc-route-open-missing-program-{}",
12011 std::process::id()
12012 ));
12013 let supervisor =
12014 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12015 .with_handle(supervisor_handle.clone());
12016 let module = supervisor
12017 .supervise_configured(
12018 ModuleSpec {
12019 module_id: "failed".to_string(),
12020 program: missing_program,
12021 args: Vec::new(),
12022 env: Vec::new(),
12023 reserved: false,
12024 reserved_prefixes: Vec::new(),
12025 protocol: ModuleProtocol::Subc,
12026 overlap: Default::default(),
12027 },
12028 true,
12029 )
12030 .unwrap();
12031 assert_eq!(module.state().unwrap(), ModuleState::Failed);
12032
12033 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12034 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
12035 let response = handler
12036 .handle_control_frame(
12037 &ctx,
12038 route_open_frame(305, "failed", unique_project_root("failed")),
12039 )
12040 .await
12041 .unwrap();
12042
12043 assert_eq!(response[0].header.ty, FrameType::Error);
12044 let error = parse_error(&response[0]);
12045 assert_eq!(error["code"], "target_unavailable");
12046 assert!(error["message"]
12047 .as_str()
12048 .unwrap()
12049 .contains("state=failed, enabled=true, live=false"));
12050 }
12051
12052 #[tokio::test]
12053 async fn route_open_role_mismatch_remains_target_unavailable() {
12054 let registry = Arc::new(Registry::default());
12055 let handler = ControlHandler::new(Arc::clone(®istry));
12056 handler
12057 .handle_control(
12058 ConnectionId::new(41),
12059 non_routable_hello_frame_with_control_ops("health-only", 306, None),
12060 )
12061 .unwrap();
12062
12063 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
12064 let response = handler
12065 .handle_control_frame(
12066 &ctx,
12067 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
12068 )
12069 .await
12070 .unwrap();
12071
12072 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12073 assert!(parse_error(&response[0])["message"]
12074 .as_str()
12075 .unwrap()
12076 .contains("does not provide the requested target"));
12077 }
12078
12079 #[tokio::test]
12080 async fn route_open_inactive_registration_remains_target_unavailable() {
12081 let registry = Arc::new(Registry::default());
12082 let handler = ControlHandler::new(Arc::clone(®istry));
12083 handler
12084 .handle_control(
12085 ConnectionId::new(43),
12086 hello_frame("inactive", PROTOCOL_VERSION, 308),
12087 )
12088 .unwrap();
12089 assert!(registry
12090 .set_module_state_for_test("inactive", ChannelState::Closed)
12091 .unwrap());
12092
12093 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
12094 let response = handler
12095 .handle_control_frame(
12096 &ctx,
12097 route_open_frame(309, "inactive", unique_project_root("inactive")),
12098 )
12099 .await
12100 .unwrap();
12101
12102 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12103 assert!(parse_error(&response[0])["message"]
12104 .as_str()
12105 .unwrap()
12106 .contains("is not active"));
12107 }
12108
12109 #[tokio::test]
12110 async fn late_health_reply_is_recorded_through_the_module_response_path() {
12111 let registry = Arc::new(Registry::default());
12112 let forwarding = Arc::new(ForwardingTable::default());
12113 let supervisor_handle = SupervisorHandle::new();
12114 let supervisor =
12115 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
12116 .with_forwarding(Arc::clone(&forwarding))
12117 .with_handle(supervisor_handle.clone());
12118 let module = supervisor
12119 .supervise_configured(
12120 crate::ModuleSpec {
12121 module_id: "late-health-response".to_string(),
12122 program: PathBuf::from("disabled-module"),
12123 args: Vec::new(),
12124 env: Vec::new(),
12125 reserved: false,
12126 reserved_prefixes: Vec::new(),
12127 protocol: ModuleProtocol::Subc,
12128 overlap: Default::default(),
12129 },
12130 false,
12131 )
12132 .unwrap();
12133 let handler =
12134 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12135 .with_supervisor(supervisor_handle);
12136 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
12137 handler
12138 .handle_control_frame(
12139 &module_ctx,
12140 hello_frame_with_control_ops(
12141 "late-health-response",
12142 PROTOCOL_VERSION,
12143 7,
12144 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12145 ),
12146 )
12147 .await
12148 .unwrap();
12149 let probe_started_at = Instant::now() - Duration::from_millis(80);
12150 let pending = forwarding
12151 .begin_health_probe_rpc_for(
12152 "late-health-response",
12153 MODULE_CONTROL_OP_HEALTH_CHECK,
12154 probe_started_at,
12155 Instant::now() - Duration::from_millis(1),
12156 )
12157 .unwrap();
12158 assert!(forwarding
12159 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
12160 .unwrap());
12161
12162 let responses = handler
12163 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
12164 .await
12165 .unwrap();
12166
12167 assert!(responses.is_empty());
12168 let health = module.status().unwrap().health;
12169 assert_eq!(health.late_answer_count, 1);
12170 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
12171 }
12172
12173 #[tokio::test]
12174 async fn health_probe_timeout_and_module_death_are_typed() {
12175 let registry = Arc::new(Registry::default());
12176 let forwarding = Arc::new(ForwardingTable::default());
12177 let handler =
12178 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12179 .with_health_probe_timeout(Duration::from_millis(50));
12180 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
12181 hello_via_sink(
12182 &handler,
12183 &module_ctx,
12184 &mut module_rx,
12185 hello_frame_with_control_ops(
12186 "aft",
12187 PROTOCOL_VERSION,
12188 7,
12189 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12190 ),
12191 )
12192 .await;
12193
12194 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12195 let responses = handler
12196 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
12197 .await
12198 .unwrap();
12199 assert_eq!(responses[0].header.ty, FrameType::Error);
12200 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
12201 let _ = module_rx.try_recv();
12202
12203 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
12204 let health_handler = handler.clone();
12205 let death_task = tokio::spawn(async move {
12206 health_handler
12207 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
12208 .await
12209 .unwrap()
12210 });
12211 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12212 .await
12213 .unwrap()
12214 .unwrap();
12215 handler
12216 .cleanup_connection(module_ctx.connection_id)
12217 .unwrap();
12218 let responses = death_task.await.unwrap();
12219 assert_eq!(responses[0].header.ty, FrameType::Error);
12220 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
12221 }
12222
12223 #[test]
12224 fn hello_requires_exact_protocol_version() {
12225 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
12226 let registry = Arc::new(Registry::default());
12227 let handler = ControlHandler::new(Arc::clone(®istry));
12228 let responses = handler
12229 .handle_control(
12230 ConnectionId::new(connection),
12231 hello_frame("aft", offered, 9),
12232 )
12233 .unwrap();
12234
12235 assert_eq!(responses.len(), 1);
12236 assert_eq!(responses[0].header.ty, FrameType::Error);
12237 let error = parse_error(&responses[0]);
12238 assert_eq!(error["code"], "version_unsupported");
12239 assert!(registry.get_module("aft").unwrap().is_none());
12240 assert_eq!(registry.active_registration_count().unwrap(), 0);
12241 }
12242 }
12243
12244 #[test]
12245 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
12246 let registry = Arc::new(Registry::default());
12247 let forwarding = Arc::new(ForwardingTable::default());
12248 let handler =
12249 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12250 let module_connection = ConnectionId::new(301);
12251 let registration = registry
12252 .register_with_control_ops(
12253 manifest("aft-push", PROTOCOL_VERSION),
12254 PROTOCOL_VERSION,
12255 module_connection,
12256 module_baseline_control_ops(),
12257 )
12258 .unwrap();
12259 let (module_tx, _module_rx) = mpsc::channel(8);
12260 let endpoint = forwarding
12261 .register_module_connection(
12262 module_connection,
12263 "aft-push".to_string(),
12264 PROTOCOL_VERSION,
12265 manifest_concurrency(®istration.manifest),
12266 FrameSink::new(module_tx),
12267 )
12268 .unwrap();
12269
12270 let unknown = Frame::build(
12272 FrameType::Push,
12273 control_flags(),
12274 0,
12275 0,
12276 5,
12277 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
12278 )
12279 .unwrap();
12280 let out = handler.handle_status_update(endpoint, unknown).unwrap();
12281 assert!(
12282 out.is_empty(),
12283 "unknown push op must be ignored, got {out:?}"
12284 );
12285
12286 let malformed = Frame::build(
12288 FrameType::Push,
12289 control_flags(),
12290 0,
12291 0,
12292 6,
12293 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
12294 )
12295 .unwrap();
12296 let out = handler.handle_status_update(endpoint, malformed).unwrap();
12297 assert_eq!(out.len(), 1);
12298 assert_eq!(out[0].header.ty, FrameType::Error);
12299 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
12300 }
12301
12302 #[test]
12303 fn hello_rejected_when_connection_already_owns_client_routes() {
12304 let registry = Arc::new(Registry::default());
12305 let forwarding = Arc::new(ForwardingTable::default());
12306 let handler =
12307 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12308 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
12310 let client_connection = ConnectionId::new(202);
12311
12312 let responses = handler
12315 .handle_control(
12316 client_connection,
12317 hello_frame("aft-second", PROTOCOL_VERSION, 9),
12318 )
12319 .unwrap();
12320 assert_eq!(responses[0].header.ty, FrameType::Error);
12321 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
12322 assert!(registry.get_module("aft-second").unwrap().is_none());
12323 }
12324
12325 #[tokio::test]
12326 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
12327 let registry = Arc::new(Registry::default());
12328 let forwarding = Arc::new(ForwardingTable::default());
12329 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
12330 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
12331 hello_via_sink(
12332 &handler,
12333 &module_ctx,
12334 &mut module_rx,
12335 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
12336 )
12337 .await;
12338 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
12339 let pending = forwarding
12340 .begin_route_bind_relay_for_test(
12341 client_ctx.connection_id,
12342 client_ctx.egress.clone(),
12343 2,
12344 "alpha",
12345 )
12346 .unwrap();
12347 forwarding
12348 .complete_pending_relay(
12349 module_ctx.connection_id,
12350 pending.corr,
12351 RouteBindRelayOutcome::Accepted,
12352 )
12353 .unwrap();
12354 client_rx.try_recv().unwrap();
12355 for module_id in ["beta", "alpha"] {
12356 let replies = handler
12357 .handle_control_frame(
12358 &module_ctx,
12359 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
12360 )
12361 .await
12362 .unwrap();
12363 assert_eq!(replies.len(), 1, "second HELLO must be refused");
12364 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
12365 }
12366 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
12367 assert!(registry.get_module("beta").unwrap().is_none());
12368 assert!(matches!(
12369 forwarding
12370 .lookup_data_route(
12371 client_ctx.connection_id,
12372 pending.client_channel,
12373 pending.client_epoch,
12374 )
12375 .unwrap(),
12376 DataRoute::Client(DataRouteState::Bound(_))
12377 ));
12378 assert!(handler
12379 .hello_launch_nonces
12380 .lock()
12381 .unwrap()
12382 .presented(module_ctx.connection_id, Some("alpha-nonce")));
12383 assert!(module_rx.try_recv().is_err());
12384 }
12385
12386 #[test]
12387 fn reserved_module_hello_requires_matching_launch_nonce() {
12388 let registry = Arc::new(Registry::default());
12389 let supervisor = SupervisorHandle::new();
12390 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
12393 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12394
12395 let no_nonce = handler
12397 .handle_control(
12398 ConnectionId::new(1),
12399 hello_frame("vault", PROTOCOL_VERSION, 1),
12400 )
12401 .unwrap();
12402 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
12403 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
12404 assert!(registry.get_module("vault").unwrap().is_none());
12405
12406 let wrong = handler
12408 .handle_control(
12409 ConnectionId::new(2),
12410 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
12411 )
12412 .unwrap();
12413 assert_eq!(wrong[0].header.ty, FrameType::Error);
12414 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
12415 assert!(registry.get_module("vault").unwrap().is_none());
12416
12417 let ok = handler
12419 .handle_control(
12420 ConnectionId::new(3),
12421 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
12422 )
12423 .unwrap();
12424 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
12425 assert!(registry.get_module("vault").unwrap().is_some());
12426 }
12427
12428 #[test]
12429 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
12430 let registry = Arc::new(Registry::default());
12431 let supervisor = SupervisorHandle::new();
12432 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
12433 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
12434 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12435
12436 let squat = handler
12437 .handle_control(
12438 ConnectionId::new(1),
12439 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
12440 )
12441 .unwrap();
12442 assert_eq!(squat[0].header.ty, FrameType::Error);
12443 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
12444 assert!(parse_error(&squat[0])["message"]
12445 .as_str()
12446 .unwrap()
12447 .contains("fed:"));
12448
12449 let accepted_peer = handler
12450 .handle_control(
12451 ConnectionId::new(2),
12452 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
12453 )
12454 .unwrap();
12455 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
12456
12457 let accepted_short = handler
12458 .handle_control(
12459 ConnectionId::new(3),
12460 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
12461 )
12462 .unwrap();
12463 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
12464
12465 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
12466 let response = handler
12467 .handle_control(
12468 ConnectionId::new(conn),
12469 hello_frame(module_id, PROTOCOL_VERSION, conn),
12470 )
12471 .unwrap();
12472 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
12473 }
12474 }
12475
12476 #[test]
12477 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
12478 let registry = Arc::new(Registry::default());
12479 let supervisor = SupervisorHandle::new();
12480 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
12481 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
12482 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
12483 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12484
12485 let owner_nonce = handler
12486 .handle_control(
12487 ConnectionId::new(1),
12488 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
12489 )
12490 .unwrap();
12491 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
12492 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
12493 assert!(registry.get_module("fed:special").unwrap().is_none());
12494
12495 let exact_nonce = handler
12496 .handle_control(
12497 ConnectionId::new(2),
12498 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
12499 )
12500 .unwrap();
12501 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
12502 assert!(registry.get_module("fed:special").unwrap().is_some());
12503 }
12504
12505 #[test]
12506 fn non_reserved_module_ignores_launch_nonce() {
12507 let registry = Arc::new(Registry::default());
12508 let handler = ControlHandler::new(Arc::clone(®istry));
12511 let no_nonce = handler
12512 .handle_control(
12513 ConnectionId::new(1),
12514 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
12515 )
12516 .unwrap();
12517 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
12518 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
12519
12520 let echoed_nonce = handler
12521 .handle_control(
12522 ConnectionId::new(2),
12523 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
12524 )
12525 .unwrap();
12526 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
12527 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
12528 }
12529
12530 #[test]
12531 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
12532 let handler = ControlHandler::default();
12533 let conn = ConnectionId::new(1);
12534 let malformed = Frame::build(
12535 FrameType::Hello,
12536 control_flags(),
12537 0,
12538 0,
12539 3,
12540 b"{not json".to_vec(),
12541 )
12542 .unwrap();
12543
12544 let error = handler.handle_control(conn, malformed).unwrap();
12545 assert_eq!(error[0].header.ty, FrameType::Error);
12546 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
12547
12548 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
12549 let pong = handler.handle_control(conn, ping).unwrap();
12550 assert_eq!(pong[0].header.ty, FrameType::Pong);
12551 assert_eq!(pong[0].header.corr, 4);
12552 }
12553
12554 #[test]
12555 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
12556 let registry = Arc::new(Registry::default());
12557 let handler = ControlHandler::new(Arc::clone(®istry));
12558
12559 handler
12560 .handle_control(
12561 ConnectionId::new(1),
12562 hello_frame("aft", PROTOCOL_VERSION, 1),
12563 )
12564 .unwrap();
12565 let duplicate = handler
12566 .handle_control(
12567 ConnectionId::new(2),
12568 hello_frame("aft", PROTOCOL_VERSION, 2),
12569 )
12570 .unwrap();
12571
12572 assert_eq!(duplicate[0].header.ty, FrameType::Error);
12573 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
12574 let registration = registry.get_module("aft").unwrap().unwrap();
12575 assert_eq!(registration.connection_id, ConnectionId::new(1));
12576 }
12577
12578 #[test]
12579 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
12580 let registry = Arc::new(Registry::default());
12581 let forwarding = Arc::new(ForwardingTable::default());
12582 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
12583 let handler =
12584 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12585 .with_process_liveness(process_liveness);
12586 let (ctx, route_channel, route_epoch) =
12587 bind_liveness_route(®istry, &forwarding, "aft-dead");
12588 let responses = handler
12589 .handle_route_poll(
12590 &ctx,
12591 route_poll_frame(41, PollKind::Liveness, route_channel),
12592 route_channel,
12593 route_epoch,
12594 PollKind::Liveness,
12595 )
12596 .unwrap();
12597
12598 assert_eq!(responses.len(), 1);
12599 assert_eq!(responses[0].header.ty, FrameType::Response);
12600 assert_route_poll_liveness(&responses[0], false);
12601 }
12602
12603 #[test]
12604 fn liveness_poll_without_process_source_uses_bound_route() {
12605 let registry = Arc::new(Registry::default());
12606 let forwarding = Arc::new(ForwardingTable::default());
12607 let handler =
12608 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12609 let (ctx, route_channel, route_epoch) =
12610 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
12611 let responses = handler
12612 .handle_route_poll(
12613 &ctx,
12614 route_poll_frame(42, PollKind::Liveness, route_channel),
12615 route_channel,
12616 route_epoch,
12617 PollKind::Liveness,
12618 )
12619 .unwrap();
12620
12621 assert_route_poll_liveness(&responses[0], true);
12622 }
12623
12624 #[test]
12625 fn liveness_poll_untracked_process_source_uses_bound_route() {
12626 let registry = Arc::new(Registry::default());
12627 let forwarding = Arc::new(ForwardingTable::default());
12628 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
12629 let handler =
12630 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12631 .with_process_liveness(process_liveness);
12632 let (ctx, route_channel, route_epoch) =
12633 bind_liveness_route(®istry, &forwarding, "aft-untracked");
12634 let responses = handler
12635 .handle_route_poll(
12636 &ctx,
12637 route_poll_frame(43, PollKind::Liveness, route_channel),
12638 route_channel,
12639 route_epoch,
12640 PollKind::Liveness,
12641 )
12642 .unwrap();
12643
12644 assert_route_poll_liveness(&responses[0], true);
12645 }
12646
12647 #[tokio::test]
12648 async fn unknown_op_returns_unknown_control_op() {
12649 let handler = ControlHandler::default();
12650 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
12651 let request = Frame::build(
12652 FrameType::Request,
12653 control_flags(),
12654 0,
12655 0,
12656 55,
12657 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
12658 )
12659 .unwrap();
12660
12661 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12662
12663 assert_eq!(response.len(), 1);
12664 assert_eq!(response[0].header.ty, FrameType::Error);
12665 assert_eq!(response[0].header.corr, 55);
12666 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
12667 }
12668
12669 #[tokio::test]
12670 async fn supervisor_provenance_rejects_unknown_exact_module() {
12671 let handler = ControlHandler::default();
12672 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
12673 let request = Frame::build(
12674 FrameType::Request,
12675 control_flags(),
12676 0,
12677 0,
12678 57,
12679 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
12680 )
12681 .unwrap();
12682
12683 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12684
12685 assert_eq!(response.len(), 1);
12686 assert_eq!(response[0].header.ty, FrameType::Error);
12687 assert_eq!(response[0].header.corr, 57);
12688 let error = parse_error(&response[0]);
12689 assert_eq!(error["code"], "unknown_module");
12690 assert_eq!(error["message"], "module_id 'missing' is not supervised");
12691 }
12692
12693 #[test]
12694 fn provenance_probe_override_keeps_handler_tests_deterministic() {
12695 let expected = subc_control::RunningImageAgreement::Unavailable {
12696 reason: subc_control::RunningImageUnavailableReason::HashFailed,
12697 };
12698 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
12699 assert_eq!(handler.provenance_probe_override, Some(expected));
12700 }
12701
12702 #[test]
12703 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
12704 let verdict = reload_verdict(
12705 std::path::Path::new("/bin/new"),
12706 Some(std::path::Path::new("/bin/old")),
12707 subc_control::RunningImageAgreement::Unavailable {
12708 reason: subc_control::RunningImageUnavailableReason::HashFailed,
12709 },
12710 );
12711 assert!(matches!(
12712 verdict.path,
12713 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
12714 if configured == std::path::Path::new("/bin/new")
12715 && spawned_from == std::path::Path::new("/bin/old")
12716 ));
12717 }
12718
12719 #[cfg(windows)]
12720 #[test]
12721 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
12722 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
12723 let configured = dir.join("module.exe");
12724 std::fs::write(&configured, b"executable path fixture").unwrap();
12725 let spawned = std::fs::canonicalize(&configured).unwrap();
12726 assert_ne!(
12727 configured, spawned,
12728 "fixture must exercise different path spellings"
12729 );
12730 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
12731 let lower_case = configured.to_string_lossy().to_lowercase();
12732 for spelling in [
12733 &configured,
12734 Path::new(&forward_slashes),
12735 Path::new(&lower_case),
12736 ] {
12737 let verdict = reload_verdict(
12738 spelling,
12739 Some(&spawned),
12740 subc_control::RunningImageAgreement::Unavailable {
12741 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12742 },
12743 );
12744 assert_eq!(
12745 verdict.path,
12746 ReloadPathAgreement::Match,
12747 "configured {spelling:?}, spawned {spawned:?}"
12748 );
12749 }
12750 }
12751
12752 #[cfg(windows)]
12753 #[test]
12754 fn windows_reload_path_keeps_different_existing_program_pending() {
12755 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
12756 let old = dir.join("old.exe");
12757 let new = dir.join("new.exe");
12758 std::fs::write(&old, b"same executable content").unwrap();
12759 std::fs::copy(&old, &new).unwrap();
12760 let spawned = std::fs::canonicalize(&old).unwrap();
12761 let image = subc_control::RunningImageAgreement::Unavailable {
12762 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12763 };
12764 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
12765 assert!(
12766 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
12767 if configured == new && spawned_from == spawned)
12768 );
12769 let missing = dir.join("missing.exe");
12770 assert!(matches!(
12771 reload_verdict(&missing, Some(&spawned), image).path,
12772 ReloadPathAgreement::Mismatch { .. }
12773 ));
12774 }
12775
12776 #[test]
12777 fn reload_verdict_detects_replaced_image_at_same_path() {
12778 let image = subc_control::RunningImageAgreement::Mismatch {
12779 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
12780 digest: "old".into(),
12781 },
12782 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
12783 digest: "new".into(),
12784 },
12785 };
12786 let verdict = reload_verdict(
12787 std::path::Path::new("/bin/same"),
12788 Some(std::path::Path::new("/bin/same")),
12789 image.clone(),
12790 );
12791 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
12792 assert_eq!(verdict.image, image);
12793 }
12794
12795 #[test]
12796 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
12797 let image = subc_control::RunningImageAgreement::Unavailable {
12798 reason: subc_control::RunningImageUnavailableReason::NotRunning,
12799 };
12800 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
12801 assert_eq!(
12802 verdict.path,
12803 subc_control::ReloadPathAgreement::Unavailable {
12804 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
12805 }
12806 );
12807 assert_eq!(verdict.image, image);
12808
12809 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
12810 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12811 };
12812 let verdict = reload_verdict(
12813 std::path::Path::new("/bin/same"),
12814 Some(std::path::Path::new("/bin/same")),
12815 unconfirmed.clone(),
12816 );
12817 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
12818 assert_eq!(verdict.image, unconfirmed);
12819 }
12820
12821 #[test]
12822 fn reload_verdict_preserves_each_image_unavailability_reason() {
12823 use subc_control::RunningImageUnavailableReason as Reason;
12824
12825 for reason in [
12826 Reason::NotRunning,
12827 Reason::UnsupportedPlatform,
12828 Reason::RunningExecutableUnreadable,
12829 Reason::SpawnedPathUnreadable,
12830 Reason::HashFailed,
12831 Reason::ProcessIdentityUnconfirmed,
12832 Reason::Unknown("future_probe_reason".to_string()),
12833 ] {
12834 let image = subc_control::RunningImageAgreement::Unavailable {
12835 reason: reason.clone(),
12836 };
12837 let verdict = reload_verdict(
12838 std::path::Path::new("/bin/same"),
12839 Some(std::path::Path::new("/bin/same")),
12840 image.clone(),
12841 );
12842 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
12843 assert_eq!(verdict.image, image, "{reason:?}");
12844 }
12845 }
12846
12847 #[tokio::test]
12848 async fn malformed_control_bodies_return_invalid_control_body() {
12849 let handler = ControlHandler::default();
12850 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
12851
12852 for (corr, body) in [
12853 (56, br#"{"route_channel":1}"#.as_slice()),
12854 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
12855 (
12856 58,
12857 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
12858 ),
12859 ] {
12860 let request = Frame::build(
12861 FrameType::Request,
12862 control_flags(),
12863 0,
12864 0,
12865 corr,
12866 body.to_vec(),
12867 )
12868 .unwrap();
12869 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12870
12871 assert_eq!(response.len(), 1);
12872 assert_eq!(response[0].header.ty, FrameType::Error);
12873 assert_eq!(response[0].header.corr, corr);
12874 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
12875 }
12876 }
12877
12878 #[tokio::test]
12879 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
12880 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12881 let registry = Arc::new(Registry::default());
12882 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
12883 let router = Router::with_control_handler(Arc::clone(&control));
12884 let connection = router.begin_connection();
12885 let (ctx, mut rx) = route_ctx(connection.id());
12886
12887 router
12888 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
12889 .await
12890 .unwrap();
12891 let response = rx.recv().await.unwrap();
12892 let ack = parse_ack(&response);
12893 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
12894 let channel = 1;
12895
12896 let goodbye =
12897 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
12898 router.route_for_connection(&ctx, goodbye).await.unwrap();
12899 assert!(rx.try_recv().is_err());
12900 assert!(registry.get_module("aft").unwrap().is_none());
12901
12902 router
12903 .route_for_connection(&ctx, channel_request(channel, 13))
12904 .await
12905 .unwrap();
12906 let error_frame = rx.recv().await.unwrap();
12907 assert_eq!(error_frame.header.ty, FrameType::Error);
12908 assert_eq!(error_frame.header.channel, channel);
12909 let captured = crate::router::test_log::captured_logs(&logs);
12910 assert_eq!(
12911 captured
12912 .lines()
12913 .filter(|line| {
12914 line.contains("module_id=aft")
12915 && line.contains("reason=explicit_goodbye")
12916 && line.contains("module registration ended")
12917 })
12918 .count(),
12919 1,
12920 "unexpected GOODBYE registry log: {captured}"
12921 );
12922 }
12923
12924 #[tokio::test]
12925 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
12926 let registry = Arc::new(Registry::default());
12927 let handler = ControlHandler::new(registry).with_capability_config(
12928 [("prov".to_string(), true), ("cons".to_string(), true)],
12929 BTreeMap::new(),
12930 );
12931 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
12932 register_capability_manifest(
12933 &handler,
12934 &provider_ctx,
12935 &mut provider_rx,
12936 capability_manifest("prov", &["thing/v1"], &[]),
12937 1,
12938 )
12939 .await;
12940 let mut consumer = capability_manifest("cons", &[], &[]);
12941 consumer.capabilities.as_mut().unwrap().requires.push(
12942 subc_protocol::manifest::CapabilityRequirement {
12943 capability: "thing/v1".to_string(),
12944 need: subc_protocol::manifest::CapabilityNeed::Required,
12945 },
12946 );
12947 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
12948 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
12949 assert_eq!(
12950 handler.capability_evaluator.verdict("cons", "thing/v1"),
12951 Some(CapabilityVerdict::Provided)
12952 );
12953 let (mut client_rx, _) = open_route_for_capability_test(
12954 &handler,
12955 &provider_ctx,
12956 &mut provider_rx,
12957 703,
12958 3,
12959 "prov",
12960 None,
12961 )
12962 .await;
12963 let goodbye =
12964 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
12965 handler
12966 .handle_control_frame(&provider_ctx, goodbye)
12967 .await
12968 .unwrap();
12969 assert_eq!(
12970 handler.capability_evaluator.verdict("cons", "thing/v1"),
12971 Some(CapabilityVerdict::NeverProvided)
12972 );
12973 let closed = client_rx
12974 .try_recv()
12975 .expect("GOODBYE pushes route.closed before route GOODBYE");
12976 assert!(
12977 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
12978 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
12979 );
12980 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
12981 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
12982 }
12983
12984 struct RouteClosedFixture {
12985 handler: ControlHandler,
12986 #[cfg(unix)]
12987 supervisor: Supervisor,
12988 module_ctx: RouteCtx,
12989 #[cfg(unix)]
12990 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12991 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12992 route: Arc<crate::forwarding::RouteBinding>,
12993 }
12994
12995 async fn route_closed_fixture() -> RouteClosedFixture {
12996 let registry = Arc::new(Registry::default());
12997 let handler = ControlHandler::new(Arc::clone(®istry));
12998 #[cfg(unix)]
12999 let supervisor = Supervisor::new(registry, RestartPolicy::default())
13000 .with_forwarding(Arc::clone(&handler.forwarding));
13001 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
13002 register_capability_manifest(
13003 &handler,
13004 &module_ctx,
13005 &mut module_rx,
13006 capability_manifest("prov", &[], &[]),
13007 1,
13008 )
13009 .await;
13010 let (client_rx, bind) = open_route_for_capability_test(
13011 &handler,
13012 &module_ctx,
13013 &mut module_rx,
13014 702,
13015 2,
13016 "prov",
13017 None,
13018 )
13019 .await;
13020 let ModuleControlRequest::RouteBind {
13021 route_channel: channel,
13022 epoch: route_epoch,
13023 ..
13024 } = bind
13025 else {
13026 panic!("expected route.bind");
13027 };
13028 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
13029 handler
13030 .forwarding
13031 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
13032 .unwrap()
13033 else {
13034 panic!("the acknowledged route must be bound");
13035 };
13036 RouteClosedFixture {
13037 handler,
13038 #[cfg(unix)]
13039 supervisor,
13040 module_ctx,
13041 #[cfg(unix)]
13042 module_rx,
13043 client_rx,
13044 route,
13045 }
13046 }
13047
13048 impl RouteClosedFixture {
13049 #[cfg(unix)]
13050 async fn cut(&mut self) {
13051 let notices = async {
13052 let mut draining = self.module_rx.recv().await.unwrap();
13053 assert!(matches!(
13054 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
13055 &draining.body
13056 )
13057 .unwrap(),
13058 subc_protocol::session::ModuleControlCommand::Draining {
13059 reason: RouteCloseReason::Restart,
13060 ..
13061 }
13062 ));
13063 draining.flushed.take().unwrap().send(()).unwrap();
13064 let mut closing = self.client_rx.recv().await.unwrap();
13065 assert!(matches!(
13066 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
13067 ClientControlPush::RouteClosing {
13068 reason: RouteCloseReason::Restart,
13069 ..
13070 }
13071 ));
13072 closing.flushed.take().unwrap().send(()).unwrap();
13073 };
13074 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
13075 result.unwrap();
13076 }
13077
13078 fn disconnect(&mut self) -> ClientControlPush {
13079 self.handler
13080 .cleanup_connection(self.module_ctx.connection_id)
13081 .unwrap();
13082 let closed = self.client_rx.try_recv().unwrap();
13083 assert_eq!(closed.header.ty, FrameType::Push);
13084 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
13085 assert!(matches!(
13086 &push,
13087 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
13088 if channels == &[self.route.client_channel]
13089 ));
13090 assert_eq!(
13091 self.client_rx.try_recv().unwrap().header.ty,
13092 FrameType::Goodbye
13093 );
13094 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
13095 push
13096 }
13097 }
13098
13099 #[cfg(unix)]
13100 #[tokio::test(start_paused = true)]
13101 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
13102 let mut fixture = route_closed_fixture().await;
13103 fixture.cut().await;
13104 assert!(matches!(
13105 fixture.disconnect(),
13106 ClientControlPush::RouteClosed {
13107 reason: RouteCloseReason::Restart,
13108 drained: true,
13109 ..
13110 }
13111 ));
13112 }
13113
13114 #[cfg(unix)]
13115 #[tokio::test(start_paused = true)]
13116 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
13117 let mut fixture = route_closed_fixture().await;
13118 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
13119 let started = tokio::time::Instant::now();
13120 fixture.cut().await;
13121 assert!(started.elapsed() >= Duration::from_secs(2));
13122 assert_eq!(fixture.route.flow.in_flight(), 1);
13123 assert!(matches!(
13124 fixture.disconnect(),
13125 ClientControlPush::RouteClosed {
13126 reason: RouteCloseReason::Restart,
13127 drained: false,
13128 ..
13129 }
13130 ));
13131 }
13132
13133 #[tokio::test]
13134 async fn module_connection_crash_reports_crash_and_not_drained() {
13135 let mut fixture = route_closed_fixture().await;
13136 assert!(matches!(
13140 fixture.disconnect(),
13141 ClientControlPush::RouteClosed {
13142 reason: RouteCloseReason::Crash,
13143 drained: false,
13144 ..
13145 }
13146 ));
13147 }
13148
13149 #[tokio::test]
13150 async fn dropping_router_connection_releases_registration() {
13151 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13152 let registry = Arc::new(Registry::default());
13153 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
13154 let router = Router::with_control_handler(Arc::clone(&control));
13155 let connection = router.begin_connection();
13156 let connection_id = connection.id();
13157 let (ctx, mut rx) = route_ctx(connection_id);
13158
13159 router
13160 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
13161 .await
13162 .unwrap();
13163 let response = rx.recv().await.unwrap();
13164 let ack = parse_ack(&response);
13165 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13166 assert!(registry.get_module("aft").unwrap().is_some());
13167
13168 drop(connection);
13169
13170 assert!(registry.get_module("aft").unwrap().is_none());
13171 assert_eq!(registry.active_registration_count().unwrap(), 0);
13172
13173 control
13174 .cleanup_connection(ConnectionId::new(u64::MAX))
13175 .unwrap();
13176 let captured = crate::router::test_log::captured_logs(&logs);
13177 println!("captured registry lifecycle logs:\n{captured}");
13178 let events: Vec<_> = captured
13179 .lines()
13180 .filter(|line| {
13181 line.contains("module registered module_id=aft ")
13182 || line.contains("module registration ended")
13183 })
13184 .collect();
13185 assert_eq!(
13186 events.len(),
13187 2,
13188 "unexpected registry lifecycle logs: {captured}"
13189 );
13190 assert!(events[0].contains("module registered module_id=aft "));
13191 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
13192 assert!(events[1].contains(&format!(
13193 "module_id=aft connection_id={}",
13194 connection_id.get()
13195 )));
13196 assert!(events[1].contains("reason=connection_closed"));
13197 assert!(events[1].contains("module registration ended"));
13198 assert_eq!(
13199 captured
13200 .lines()
13201 .filter(|line| line.contains("module registered module_id=aft "))
13202 .count(),
13203 1,
13204 "legacy registration admission line must appear once: {captured}"
13205 );
13206 }
13207
13208 #[tokio::test]
13209 async fn hello_registration_keeps_legacy_module_registered_line_once() {
13210 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13211 let registry = Arc::new(Registry::default());
13212 let control = ControlHandler::new(Arc::clone(®istry));
13213 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
13214 hello_via_sink(
13215 &control,
13216 &ctx,
13217 &mut rx,
13218 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
13219 )
13220 .await;
13221
13222 let captured = crate::router::test_log::captured_logs(&logs);
13223 let admissions: Vec<_> = captured
13224 .lines()
13225 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
13226 .collect();
13227 assert_eq!(
13228 admissions.len(),
13229 1,
13230 "legacy admission line must remain exactly once: {captured}"
13231 );
13232 assert!(admissions[0].contains("routable_provider=true"));
13233 assert!(admissions[0].contains("connection_id=777"));
13234 }
13235
13236 fn capability_manifest(
13237 module_id: &str,
13238 provides: &[&str],
13239 must_never_reach: &[&str],
13240 ) -> ModuleManifest {
13241 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
13242 manifest.capabilities = Some(CapabilityDeclarations {
13243 provides: provides
13244 .iter()
13245 .map(|capability| (*capability).to_string())
13246 .collect(),
13247 requires: Vec::new(),
13248 must_never_reach: must_never_reach
13249 .iter()
13250 .map(|capability| (*capability).to_string())
13251 .collect(),
13252 });
13253 manifest
13254 }
13255
13256 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
13257 Frame::build(
13258 FrameType::Hello,
13259 control_flags(),
13260 0,
13261 0,
13262 corr,
13263 serde_json::to_vec(&ModuleHelloBody {
13264 protocol_ver: manifest.protocol_ver,
13265 manifest,
13266 control_ops: None,
13267 launch_nonce: None,
13268 })
13269 .expect("capability test HELLO serializes"),
13270 )
13271 .expect("capability test HELLO frame builds")
13272 }
13273
13274 fn catalog_update_with_capabilities_frame(
13275 corr: u64,
13276 capabilities: CapabilityDeclarations,
13277 ) -> Frame {
13278 Frame::build(
13279 FrameType::Request,
13280 control_flags(),
13281 0,
13282 0,
13283 corr,
13284 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
13285 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
13286 capabilities: Some(capabilities),
13287 ready: None,
13288 })
13289 .expect("capability catalog.update serializes"),
13290 )
13291 .expect("capability catalog.update frame builds")
13292 }
13293
13294 async fn register_capability_manifest(
13295 handler: &ControlHandler,
13296 ctx: &RouteCtx,
13297 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13298 manifest: ModuleManifest,
13299 corr: u64,
13300 ) {
13301 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
13302 }
13303
13304 async fn open_route_for_capability_test(
13305 handler: &ControlHandler,
13306 target_ctx: &RouteCtx,
13307 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13308 client_connection_id: u64,
13309 corr: u64,
13310 target_module_id: &str,
13311 consumer_identity: Option<ConsumerIdentity>,
13312 ) -> (
13313 mpsc::Receiver<crate::router::OutboundFrame>,
13314 ModuleControlRequest,
13315 ) {
13316 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
13317 let route_handler = handler.clone();
13318 let target_module_id = target_module_id.to_string();
13319 let route_task = tokio::spawn(async move {
13320 route_handler
13321 .handle_control_frame(
13322 &client_ctx,
13323 route_open_frame_with_admission_facts(
13324 corr,
13325 &target_module_id,
13326 unique_project_root("admission-facts"),
13327 consumer_identity,
13328 None,
13329 ),
13330 )
13331 .await
13332 .expect("capability test route.open succeeds")
13333 });
13334 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
13335 .await
13336 .expect("capability test route.open must reach route.bind")
13337 .expect("target control receiver stays open");
13338 let bind_request: ModuleControlRequest =
13339 serde_json::from_slice(&bind.body).expect("route.bind decodes");
13340 handler
13341 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
13342 .await
13343 .expect("capability test route.bind ACK succeeds");
13344 assert!(route_task.await.expect("route.open task joins").is_empty());
13345 let opened = client_rx
13346 .recv()
13347 .await
13348 .expect("successful route.open publishes a response");
13349 assert!(matches!(
13350 serde_json::from_slice::<ClientControlResponse>(&opened.body),
13351 Ok(ClientControlResponse::RouteOpen { .. })
13352 ));
13353 (client_rx, bind_request)
13354 }
13355
13356 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
13357 assert_eq!(frame.header.ty, FrameType::Push);
13358 assert_eq!(frame.header.channel, 0);
13359 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
13360 .expect("route.closed control push decodes");
13361 let ClientControlPush::RouteClosed { channels, .. } = &push else {
13362 panic!("expected route.closed");
13363 };
13364 assert_eq!(channels.len(), 1, "exactly one violating route closed");
13365 let channels = channels.clone();
13366 assert_eq!(
13367 push,
13368 ClientControlPush::RouteClosed {
13369 module_id: target_module_id.to_string(),
13370 channels,
13371 reason: RouteCloseReason::CapabilityDenied,
13372 drained: false,
13373 abandoned: 0,
13374 excluded_subscriptions: 0,
13375 terminal: Some(false),
13376 }
13377 );
13378 }
13379
13380 #[tokio::test]
13381 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
13382 let registry = Arc::new(Registry::default());
13383 let forwarding = Arc::new(ForwardingTable::default());
13384 let supervisor = SupervisorHandle::new();
13385 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13386 let handler =
13387 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13388 .with_supervisor(supervisor);
13389 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
13390 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
13391 register_capability_manifest(
13392 &handler,
13393 &target_ctx,
13394 &mut target_rx,
13395 capability_manifest("target", &["credentials-provider/v1"], &[]),
13396 1,
13397 )
13398 .await;
13399 register_capability_manifest(
13400 &handler,
13401 &opener_ctx,
13402 &mut opener_rx,
13403 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13404 2,
13405 )
13406 .await;
13407
13408 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
13409 let replies = handler
13410 .handle_control_frame(
13411 &client_ctx,
13412 route_open_frame_with_admission_facts(
13413 3,
13414 "target",
13415 unique_project_root("admission-facts"),
13416 Some(ConsumerIdentity {
13417 module_id: "opener".to_string(),
13418 launch_nonce: "opener-nonce".to_string(),
13419 }),
13420 None,
13421 ),
13422 )
13423 .await
13424 .expect("denied route.open returns a typed frame");
13425 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
13426 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13427 assert!(
13428 target_rx.try_recv().is_err(),
13429 "forbidden route.open must not relay route.bind"
13430 );
13431 }
13432
13433 #[tokio::test]
13434 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
13435 let registry = Arc::new(Registry::default());
13436 let forwarding = Arc::new(ForwardingTable::default());
13437 let supervisor = SupervisorHandle::new();
13438 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13439 let handler =
13440 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13441 .with_supervisor(supervisor);
13442 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
13443 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
13444 register_capability_manifest(
13445 &handler,
13446 &target_ctx,
13447 &mut target_rx,
13448 capability_manifest("target", &["credentials-provider/v1"], &[]),
13449 1,
13450 )
13451 .await;
13452 register_capability_manifest(
13453 &handler,
13454 &old_opener_ctx,
13455 &mut old_opener_rx,
13456 capability_manifest("opener", &[], &[]),
13457 2,
13458 )
13459 .await;
13460 let (mut client_rx, _) = open_route_for_capability_test(
13461 &handler,
13462 &target_ctx,
13463 &mut target_rx,
13464 712,
13465 3,
13466 "target",
13467 Some(ConsumerIdentity {
13468 module_id: "opener".to_string(),
13469 launch_nonce: "opener-nonce".to_string(),
13470 }),
13471 )
13472 .await;
13473 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13474
13475 handler
13476 .cleanup_connection(old_opener_ctx.connection_id)
13477 .expect("old opener registration cleans up");
13478 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
13479 register_capability_manifest(
13480 &handler,
13481 &new_opener_ctx,
13482 &mut new_opener_rx,
13483 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13484 4,
13485 )
13486 .await;
13487
13488 assert_capability_denied_push(
13489 client_rx
13490 .try_recv()
13491 .expect("HELLO deny addition must emit route.closed")
13492 .frame,
13493 "target",
13494 );
13495 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13496 assert!(matches!(
13497 target_rx.try_recv(),
13498 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
13499 ));
13500 }
13501
13502 #[tokio::test]
13503 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
13504 let registry = Arc::new(Registry::default());
13505 let forwarding = Arc::new(ForwardingTable::default());
13506 let supervisor = SupervisorHandle::new();
13507 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13508 let handler =
13509 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13510 .with_supervisor(supervisor);
13511 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
13512 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
13513 register_capability_manifest(
13514 &handler,
13515 &target_ctx,
13516 &mut target_rx,
13517 capability_manifest("target", &[], &[]),
13518 1,
13519 )
13520 .await;
13521 register_capability_manifest(
13522 &handler,
13523 &opener_ctx,
13524 &mut opener_rx,
13525 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13526 2,
13527 )
13528 .await;
13529 let (mut client_rx, _) = open_route_for_capability_test(
13530 &handler,
13531 &target_ctx,
13532 &mut target_rx,
13533 722,
13534 3,
13535 "target",
13536 Some(ConsumerIdentity {
13537 module_id: "opener".to_string(),
13538 launch_nonce: "opener-nonce".to_string(),
13539 }),
13540 )
13541 .await;
13542 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13543
13544 let replies = handler
13545 .handle_control_frame(
13546 &target_ctx,
13547 catalog_update_with_capabilities_frame(
13548 4,
13549 CapabilityDeclarations {
13550 provides: vec!["credentials-provider/v1".to_string()],
13551 requires: Vec::new(),
13552 must_never_reach: Vec::new(),
13553 },
13554 ),
13555 )
13556 .await
13557 .expect("claim catalog.update succeeds");
13558 assert!(matches!(
13559 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
13560 Ok(ModuleControlResponseToModule::CatalogUpdate {})
13561 ));
13562 assert_capability_denied_push(
13563 client_rx
13564 .try_recv()
13565 .expect("claim addition must emit route.closed")
13566 .frame,
13567 "target",
13568 );
13569 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13570 assert!(matches!(
13571 target_rx.try_recv(),
13572 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
13573 ));
13574 }
13575
13576 #[tokio::test]
13577 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
13578 let registry = Arc::new(Registry::default());
13579 let forwarding = Arc::new(ForwardingTable::default());
13580 let supervisor = SupervisorHandle::new();
13581 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13582 let handler =
13583 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13584 .with_supervisor(supervisor);
13585 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
13586 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
13587 register_capability_manifest(
13588 &handler,
13589 &target_ctx,
13590 &mut target_rx,
13591 capability_manifest("target", &["credentials-provider/v1"], &[]),
13592 1,
13593 )
13594 .await;
13595 register_capability_manifest(
13596 &handler,
13597 &opener_ctx,
13598 &mut opener_rx,
13599 capability_manifest("opener", &[], &[]),
13600 2,
13601 )
13602 .await;
13603 let (mut client_rx, _) = open_route_for_capability_test(
13604 &handler,
13605 &target_ctx,
13606 &mut target_rx,
13607 732,
13608 3,
13609 "target",
13610 Some(ConsumerIdentity {
13611 module_id: "opener".to_string(),
13612 launch_nonce: "opener-nonce".to_string(),
13613 }),
13614 )
13615 .await;
13616 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13617
13618 handler
13619 .handle_control_frame(
13620 &target_ctx,
13621 catalog_update_with_capabilities_frame(
13622 4,
13623 CapabilityDeclarations {
13624 provides: Vec::new(),
13625 requires: Vec::new(),
13626 must_never_reach: Vec::new(),
13627 },
13628 ),
13629 )
13630 .await
13631 .expect("claim removal catalog.update succeeds");
13632 assert_eq!(
13633 forwarding.active_binding_count().unwrap(),
13634 1,
13635 "removing an attested target claim must leave the route census unchanged"
13636 );
13637 assert!(
13638 client_rx.try_recv().is_err(),
13639 "claim removal must not emit route.closed capability_denied"
13640 );
13641 assert!(
13642 target_rx.try_recv().is_err(),
13643 "claim removal must not send the target a route GOODBYE"
13644 );
13645 }
13646
13647 #[tokio::test]
13650 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
13651 let registry = Arc::new(Registry::default());
13652 let forwarding = Arc::new(ForwardingTable::default());
13653 let supervisor = SupervisorHandle::new();
13654 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13655 let handler =
13656 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13657 .with_supervisor(supervisor);
13658 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
13659 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
13660 register_capability_manifest(
13661 &handler,
13662 &target_ctx,
13663 &mut target_rx,
13664 capability_manifest("target", &["credentials-provider/v1"], &[]),
13665 1,
13666 )
13667 .await;
13668 register_capability_manifest(
13669 &handler,
13670 &opener_ctx,
13671 &mut opener_rx,
13672 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13673 2,
13674 )
13675 .await;
13676
13677 let (_client_rx, bind) = open_route_for_capability_test(
13678 &handler,
13679 &target_ctx,
13680 &mut target_rx,
13681 742,
13682 3,
13683 "target",
13684 None,
13685 )
13686 .await;
13687 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
13688 panic!("direct scope-honesty route must bind");
13689 };
13690 assert_eq!(principal, Some(Principal::Direct));
13691 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13692 }
13693
13694 #[tokio::test]
13697 async fn must_never_reach_self_route_is_capability_forbidden() {
13698 let registry = Arc::new(Registry::default());
13699 let forwarding = Arc::new(ForwardingTable::default());
13700 let supervisor = SupervisorHandle::new();
13701 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
13702 let handler =
13703 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13704 .with_supervisor(supervisor);
13705 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
13706 register_capability_manifest(
13707 &handler,
13708 &self_ctx,
13709 &mut self_rx,
13710 capability_manifest(
13711 "self-provider",
13712 &["credentials-provider/v1"],
13713 &["credentials-provider/v1"],
13714 ),
13715 1,
13716 )
13717 .await;
13718
13719 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
13720 let replies = handler
13721 .handle_control_frame(
13722 &client_ctx,
13723 route_open_frame_with_admission_facts(
13724 2,
13725 "self-provider",
13726 unique_project_root("admission-facts"),
13727 Some(ConsumerIdentity {
13728 module_id: "self-provider".to_string(),
13729 launch_nonce: "self-nonce".to_string(),
13730 }),
13731 None,
13732 ),
13733 )
13734 .await
13735 .expect("self-route refusal returns a typed frame");
13736 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
13737 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13738 assert!(
13739 self_rx.try_recv().is_err(),
13740 "self denial must not relay route.bind"
13741 );
13742 }
13743
13744 #[test]
13745 fn unsupported_channel_zero_frame_returns_error() {
13746 let handler = ControlHandler::default();
13747 let request = Frame::build(
13748 FrameType::Request,
13749 control_flags(),
13750 0,
13751 0,
13752 21,
13753 b"opaque".to_vec(),
13754 )
13755 .unwrap();
13756
13757 let response = handler
13758 .handle_control(ConnectionId::new(1), request)
13759 .unwrap();
13760
13761 assert_eq!(response[0].header.ty, FrameType::Error);
13762 assert_eq!(
13763 parse_error(&response[0])["code"],
13764 "unsupported_control_frame"
13765 );
13766 }
13767
13768 mod swap {
13773 use super::*;
13774
13775 const INCUMBENT: ConnectionId = ConnectionId::new(30);
13776 const CANDIDATE: ConnectionId = ConnectionId::new(40);
13777
13778 struct Swap {
13779 registry: Arc<Registry>,
13780 forwarding: Arc<ForwardingTable>,
13781 handler: ControlHandler,
13782 incumbent_ctx: RouteCtx,
13783 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13784 candidate_ctx: RouteCtx,
13785 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13786 }
13787
13788 async fn swap_with_incumbent() -> Swap {
13789 let registry = Arc::new(Registry::default());
13790 let forwarding = Arc::new(ForwardingTable::default());
13791 let handler =
13792 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13793 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
13794 hello_via_sink(
13795 &handler,
13796 &incumbent_ctx,
13797 &mut incumbent_rx,
13798 hello_frame("aft", PROTOCOL_VERSION, 7),
13799 )
13800 .await;
13801 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
13802 Swap {
13803 registry,
13804 forwarding,
13805 handler,
13806 incumbent_ctx,
13807 incumbent_rx,
13808 candidate_ctx,
13809 candidate_rx,
13810 }
13811 }
13812
13813 fn register_candidate(swap: &Swap, ready: Option<bool>) {
13814 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
13815 candidate_manifest.ready = ready;
13816 let registration = swap
13817 .registry
13818 .register_candidate_with_control_ops(
13819 candidate_manifest,
13820 PROTOCOL_VERSION,
13821 CANDIDATE,
13822 module_baseline_control_ops(),
13823 )
13824 .unwrap();
13825 swap.forwarding
13826 .register_candidate_module_connection(
13827 CANDIDATE,
13828 "aft".to_string(),
13829 PROTOCOL_VERSION,
13830 manifest_concurrency(®istration.manifest),
13831 swap.candidate_ctx.egress.clone(),
13832 )
13833 .unwrap();
13834 }
13835
13836 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
13837 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
13838 swap.registry.promote_candidate("aft").unwrap().unwrap();
13839 cutover.incumbent.unwrap()
13840 }
13841
13842 #[tokio::test]
13851 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
13852 let mut swap = swap_with_incumbent().await;
13853 let hold = Arc::new(std::sync::Barrier::new(2));
13854 let handler = swap
13855 .handler
13856 .clone()
13857 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
13858 handler
13859 .supervisor
13860 .open_swap("aft", "candidate-nonce".to_string());
13861 let hello = {
13862 let handler = handler.clone();
13863 let candidate_ctx = swap.candidate_ctx.clone();
13864 std::thread::spawn(move || {
13865 handler
13866 .handle_hello(
13867 candidate_ctx.connection_id,
13868 Some(candidate_ctx.egress.clone()),
13869 hello_frame_with_nonce(
13870 "aft",
13871 PROTOCOL_VERSION,
13872 8,
13873 Some("candidate-nonce"),
13874 ),
13875 )
13876 .unwrap()
13877 })
13878 };
13879
13880 hold.wait();
13881 assert!(swap
13884 .registry
13885 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
13886 .unwrap()
13887 .is_some());
13888 assert!(swap
13889 .registry
13890 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
13891 .unwrap()
13892 .is_none());
13893 assert!(swap
13894 .forwarding
13895 .module_endpoint_for_connection(CANDIDATE)
13896 .unwrap()
13897 .is_none());
13898
13899 hold.wait();
13900 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
13901 assert_eq!(
13902 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
13903 FrameType::HelloAck
13904 );
13905 let published = swap
13906 .registry
13907 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
13908 .unwrap()
13909 .expect("candidate is published once its HELLO completes");
13910 assert_eq!(published.connection_id, CANDIDATE);
13911 assert!(swap
13912 .forwarding
13913 .module_endpoint_for_connection(CANDIDATE)
13914 .unwrap()
13915 .is_some());
13916 }
13917
13918 fn keyed_total(counters: &Value, key: &str) -> u64 {
13919 counters[key]
13920 .as_object()
13921 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
13922 .unwrap_or(0)
13923 }
13924
13925 #[tokio::test]
13926 async fn replacement_logs_old_end_and_new_admission_once() {
13927 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13928 let mut swap = swap_with_incumbent().await;
13929 swap.handler
13930 .supervisor
13931 .open_swap("aft", "candidate-nonce".to_string());
13932 hello_via_sink(
13933 &swap.handler,
13934 &swap.candidate_ctx,
13935 &mut swap.candidate_rx,
13936 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
13937 )
13938 .await;
13939 cutover(&swap);
13940 swap.handler.cleanup_connection(INCUMBENT).unwrap();
13941
13942 let captured = crate::router::test_log::captured_logs(&logs);
13943 println!("captured replacement registry logs:\n{captured}");
13944 let new_admissions: Vec<_> = captured
13945 .lines()
13946 .filter(|line| {
13947 line.contains("connection_id=40")
13948 && line.contains("swap candidate registered; not routable until cutover")
13949 })
13950 .collect();
13951 assert_eq!(
13952 new_admissions.len(),
13953 1,
13954 "unexpected admission logs: {captured}"
13955 );
13956 assert!(new_admissions[0].contains("module_id=aft"));
13957 assert!(new_admissions[0].contains("connection_id=40"));
13958 assert!(
13959 new_admissions[0].contains("swap candidate registered; not routable until cutover")
13960 );
13961 assert!(new_admissions[0].contains("ready=true"));
13962
13963 let old_admissions: Vec<_> = captured
13964 .lines()
13965 .filter(|line| {
13966 line.contains("module registered module_id=aft ")
13967 && line.contains("connection_id=30")
13968 })
13969 .collect();
13970 assert_eq!(
13971 old_admissions.len(),
13972 1,
13973 "unexpected incumbent admission: {captured}"
13974 );
13975
13976 let old_ends: Vec<_> = captured
13977 .lines()
13978 .filter(|line| {
13979 line.contains("connection_id=30") && line.contains("module registration ended")
13980 })
13981 .collect();
13982 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
13983 assert!(old_ends[0].contains("module_id=aft"));
13984 assert!(old_ends[0].contains("connection_id=30"));
13985 assert!(old_ends[0].contains("reason=replaced"));
13986 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
13987 assert_eq!(
13988 captured
13989 .lines()
13990 .filter(|line| line.contains("module registration promoted"))
13991 .count(),
13992 1,
13993 "unexpected promotion logs: {captured}"
13994 );
13995 }
13996
13997 #[tokio::test]
14003 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
14004 let mut swap = swap_with_incumbent().await;
14005 let handler = swap.handler.clone();
14006
14007 let cotenant = ConnectionId::new(31);
14009 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
14010 let (cotenant_task, cotenant_bind) = relay_route_open(
14011 &handler,
14012 cotenant,
14013 &cotenant_ctx.egress,
14014 &mut swap.incumbent_rx,
14015 100,
14016 "aft",
14017 "swap-cotenant",
14018 )
14019 .await;
14020 handler
14021 .handle_control_frame(
14022 &swap.incumbent_ctx,
14023 route_bind_ack(cotenant_bind.header.corr),
14024 )
14025 .await
14026 .unwrap();
14027 assert!(cotenant_task.await.unwrap().is_empty());
14028 let (cotenant_channel, cotenant_epoch) =
14029 published_route(&cotenant_rx.recv().await.unwrap());
14030
14031 let caller = ConnectionId::new(32);
14033 let (caller_ctx, mut caller_rx) = route_ctx(caller);
14034 let (caller_task, caller_bind) = relay_route_open(
14035 &handler,
14036 caller,
14037 &caller_ctx.egress,
14038 &mut swap.incumbent_rx,
14039 101,
14040 "aft",
14041 "swap-caller",
14042 )
14043 .await;
14044 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
14045
14046 register_candidate(&swap, None);
14047 cutover(&swap);
14048
14049 let ack = handler
14051 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
14052 .await;
14053 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
14054 if module_loop_error.is_some() {
14055 handler.cleanup_connection(INCUMBENT).unwrap();
14058 }
14059
14060 assert!(
14062 cotenant_rx.try_recv().is_err(),
14063 "the co-tenant route on the incumbent was torn down by one late ack: \
14064 {module_loop_error:?}"
14065 );
14066 assert!(matches!(
14067 swap.forwarding
14068 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
14069 .unwrap(),
14070 DataRoute::Client(DataRouteState::Bound(_))
14071 ));
14072 assert_eq!(module_loop_error, None);
14073 assert!(swap
14074 .registry
14075 .get_module_by_connection(INCUMBENT)
14076 .unwrap()
14077 .is_some());
14078
14079 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
14081 .await
14082 .expect("the incumbent is told to drop the abandoned binding")
14083 .unwrap()
14084 .frame;
14085 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14086 assert_eq!(goodbye.header.channel, abandoned_channel);
14087 assert_eq!(goodbye.header.epoch, abandoned_epoch);
14088 assert!(swap.incumbent_rx.try_recv().is_err());
14089
14090 let response = caller_task.await.unwrap();
14092 assert_eq!(response.len(), 1);
14093 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14094 assert!(caller_rx.try_recv().is_err());
14095
14096 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
14100 let counters = handler.counters().snapshot();
14101 assert_eq!(
14102 keyed_total(&counters, "route_open_accepted_by_principal"),
14103 1
14104 );
14105 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
14106 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
14107 }
14108
14109 #[tokio::test]
14113 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
14114 let mut swap = swap_with_incumbent().await;
14115 register_candidate(&swap, None);
14116 let incumbent = cutover(&swap);
14117 swap.forwarding
14118 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
14119 .unwrap()
14120 .expect("the incumbent is still registered");
14121
14122 let client = ConnectionId::new(33);
14123 let (client_ctx, mut client_rx) = route_ctx(client);
14124 let route_handler = swap.handler.clone();
14125 let open_ctx = RouteCtx {
14126 connection_id: client,
14127 egress: client_ctx.egress.clone(),
14128 };
14129 let mut route_task = tokio::spawn(async move {
14130 route_handler
14131 .handle_control_frame(
14132 &open_ctx,
14133 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
14134 )
14135 .await
14136 .unwrap()
14137 });
14138 let bind = tokio::select! {
14139 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
14140 response = &mut route_task => {
14141 let response = response.unwrap();
14142 panic!(
14143 "post-cutover route.open was refused instead of relayed to the candidate: {}",
14144 parse_error(&response[0])["code"]
14145 );
14146 }
14147 };
14148 swap.handler
14149 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
14150 .await
14151 .unwrap();
14152 assert!(route_task.await.unwrap().is_empty());
14153 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
14154 match swap
14155 .forwarding
14156 .lookup_data_route(client, channel, epoch)
14157 .unwrap()
14158 {
14159 DataRoute::Client(DataRouteState::Bound(route)) => {
14160 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
14161 }
14162 other => panic!("expected a bound route on the candidate, got {other:?}"),
14163 }
14164 assert!(swap.incumbent_rx.try_recv().is_err());
14165 }
14166
14167 #[tokio::test]
14172 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
14173 let swap = swap_with_incumbent().await;
14174 register_candidate(&swap, Some(false));
14175 let update = Frame::build(
14176 FrameType::Request,
14177 control_flags(),
14178 0,
14179 0,
14180 55,
14181 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14182 provides: manifest("aft", PROTOCOL_VERSION).provides,
14183 capabilities: None,
14184 ready: Some(true),
14185 })
14186 .unwrap(),
14187 )
14188 .unwrap();
14189
14190 let replies = swap
14191 .handler
14192 .handle_control_frame(&swap.candidate_ctx, update)
14193 .await
14194 .unwrap();
14195
14196 assert_eq!(replies.len(), 1);
14197 assert_eq!(
14198 replies[0].header.ty,
14199 FrameType::Response,
14200 "candidate catalog.update was refused: {:?}",
14201 serde_json::from_slice::<Value>(&replies[0].body).ok()
14202 );
14203 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
14204 assert_eq!(
14205 swap.registry
14206 .get_module("aft")
14207 .unwrap()
14208 .unwrap()
14209 .connection_id,
14210 INCUMBENT
14211 );
14212 }
14213 }
14214
14215 mod swap_admission {
14219 use super::*;
14220
14221 const INCUMBENT_NONCE: &str = "incumbent-nonce";
14222 const CANDIDATE_NONCE: &str = "candidate-nonce";
14223
14224 fn handler_with_incumbent(
14225 module_id: &str,
14226 reserved: bool,
14227 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
14228 let registry = Arc::new(Registry::default());
14229 let supervisor = SupervisorHandle::new();
14230 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
14231 if reserved {
14232 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
14233 }
14234 let handler =
14235 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
14236 let incumbent = handler
14237 .handle_control(
14238 ConnectionId::new(1),
14239 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
14240 )
14241 .unwrap();
14242 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
14243 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
14244 (registry, supervisor, handler)
14245 }
14246
14247 #[test]
14253 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
14254 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
14255
14256 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
14257 let replies = handler
14258 .handle_control(
14259 ConnectionId::new(connection),
14260 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
14261 )
14262 .unwrap();
14263 assert_eq!(replies[0].header.ty, FrameType::Error);
14264 assert_eq!(
14265 parse_error(&replies[0])["code"],
14266 "swap_token_invalid",
14267 "nonce {nonce:?}"
14268 );
14269 }
14270 assert!(registry.get_candidate("aft").unwrap().is_none());
14271 assert_eq!(
14272 registry.get_module("aft").unwrap().unwrap().connection_id,
14273 ConnectionId::new(1)
14274 );
14275
14276 let admitted = handler
14279 .handle_control(
14280 ConnectionId::new(4),
14281 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
14282 )
14283 .unwrap();
14284 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
14285 assert_eq!(
14286 registry
14287 .get_candidate("aft")
14288 .unwrap()
14289 .unwrap()
14290 .connection_id,
14291 ConnectionId::new(4)
14292 );
14293 assert_eq!(
14294 registry.get_module("aft").unwrap().unwrap().connection_id,
14295 ConnectionId::new(1),
14296 "the candidate must not take the active slot"
14297 );
14298 let replayed = handler
14299 .handle_control(
14300 ConnectionId::new(5),
14301 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
14302 )
14303 .unwrap();
14304 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
14305
14306 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
14310 let squatter = handler
14311 .handle_control(
14312 ConnectionId::new(6),
14313 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
14314 )
14315 .unwrap();
14316 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
14317 assert!(
14318 registry.get_module("aft").unwrap().is_none(),
14319 "a squatter took the active slot of an id being swapped"
14320 );
14321 }
14322
14323 #[test]
14328 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
14329 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
14330
14331 let replies = handler
14332 .handle_control(
14333 ConnectionId::new(2),
14334 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14335 )
14336 .unwrap();
14337
14338 assert_eq!(
14339 replies[0].header.ty,
14340 FrameType::HelloAck,
14341 "reserved candidate refused: {:?}",
14342 serde_json::from_slice::<Value>(&replies[0].body).ok()
14343 );
14344 assert_eq!(
14345 registry
14346 .get_candidate("vault")
14347 .unwrap()
14348 .unwrap()
14349 .connection_id,
14350 ConnectionId::new(2)
14351 );
14352 }
14353
14354 #[test]
14357 fn without_an_open_swap_the_ordinary_gates_decide() {
14358 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
14359 supervisor.close_swap("vault");
14360
14361 let candidate = handler
14362 .handle_control(
14363 ConnectionId::new(2),
14364 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14365 )
14366 .unwrap();
14367 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
14368 let duplicate = handler
14369 .handle_control(
14370 ConnectionId::new(3),
14371 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
14372 )
14373 .unwrap();
14374 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14375 assert!(registry.get_candidate("vault").unwrap().is_none());
14376 }
14377 }
14378
14379 mod scopes {
14383 mod activity {
14384 include!("activity_scope_tests.rs");
14385 }
14386 mod delta {
14387 include!("scope_handler_delta_tests.rs");
14388 }
14389 mod call_modes {
14390 include!("scope_call_mode_tests.rs");
14391 }
14392
14393 use subc_protocol::scope::{
14394 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
14395 ScopeStatus,
14396 };
14397
14398 use super::*;
14399
14400 const OWNER: &str = "prefrontal-core";
14401
14402 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
14403 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
14404 }
14405
14406 async fn call(
14407 handler: &ControlHandler,
14408 ctx: &RouteCtx,
14409 request: &ModuleControlRequestFromModule,
14410 ) -> Frame {
14411 let body = serde_json::to_vec(request).unwrap();
14412 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
14413 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
14414 assert_eq!(replies.len(), 1, "{replies:?}");
14415 replies.pop().unwrap()
14416 }
14417
14418 async fn sync(
14419 handler: &ControlHandler,
14420 ctx: &RouteCtx,
14421 generation: u64,
14422 scopes: Vec<ScopeRecord>,
14423 ) -> Result<ModuleControlResponseToModule, String> {
14424 let reply = call(
14425 handler,
14426 ctx,
14427 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
14428 )
14429 .await;
14430 match reply.header.ty {
14431 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
14432 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
14433 }
14434 }
14435
14436 async fn describe(
14437 handler: &ControlHandler,
14438 ctx: &RouteCtx,
14439 owner: &str,
14440 scope_ref: &str,
14441 ) -> ModuleControlResponseToModule {
14442 let reply = call(
14443 handler,
14444 ctx,
14445 &ModuleControlRequestFromModule::ScopeDescribe {
14446 owner: Principal::Reserved {
14447 module_id: owner.to_string(),
14448 },
14449 scope_ref: scope_ref.to_string(),
14450 },
14451 )
14452 .await;
14453 assert_eq!(
14454 reply.header.ty,
14455 FrameType::Response,
14456 "{:?}",
14457 parse_error(&reply)
14458 );
14459 serde_json::from_slice(&reply.body).unwrap()
14460 }
14461
14462 #[tokio::test]
14463 async fn scope_describe_never_echoes_bind_proof_or_hash() {
14464 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
14465 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
14466 let mut rig = rig().await;
14467 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
14468 .await;
14469 let _bound = rig
14470 .bound(
14471 Some(AFT),
14472 PLEXUS,
14473 Some(
14474 rig.selector("s", Some(1))
14475 .with_bind_proof(Some(PROOF.into())),
14476 ),
14477 )
14478 .await;
14479 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
14480 let output = serde_json::to_string(&reply).unwrap();
14481 assert!(output.contains("\"status\":\"live\""));
14482 assert!(!output.contains(PROOF));
14483 assert!(!output.contains(HASH));
14484 }
14485
14486 async fn module(
14488 handler: &ControlHandler,
14489 connection: u64,
14490 module_id: &str,
14491 nonce: Option<&str>,
14492 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
14493 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
14494 hello_via_sink(
14495 handler,
14496 &ctx,
14497 &mut rx,
14498 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
14499 )
14500 .await;
14501 (ctx, rx)
14502 }
14503
14504 #[tokio::test]
14507 async fn a_client_connection_cannot_sync_or_describe() {
14508 let handler = ControlHandler::new(Arc::new(Registry::default()));
14509 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
14510 for request in [
14511 ModuleControlRequestFromModule::ScopeSync {
14512 generation: 1,
14513 scopes: vec![head("s", 1)],
14514 },
14515 ModuleControlRequestFromModule::ScopeDescribe {
14516 owner: Principal::Direct,
14517 scope_ref: "s".to_string(),
14518 },
14519 ] {
14520 let reply = call(&handler, &ctx, &request).await;
14521 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
14522 }
14523 assert!(
14524 !handler
14525 .scopes
14526 .read()
14527 .unwrap()
14528 .describe(
14529 &Principal::Reserved {
14530 module_id: OWNER.to_string()
14531 },
14532 "s"
14533 )
14534 .owner_synced
14535 );
14536 }
14537
14538 #[tokio::test]
14541 async fn a_module_without_a_supervised_launch_cannot_sync() {
14542 let handler = ControlHandler::new(Arc::new(Registry::default()));
14543 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
14544 assert_eq!(
14545 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
14546 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14547 );
14548 }
14549
14550 #[tokio::test]
14551 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
14552 let supervisor = SupervisorHandle::new();
14553 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14554 let handler = ControlHandler::new(Arc::new(Registry::default()))
14555 .with_supervisor(supervisor.clone());
14556 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
14557 sync(&handler, &incumbent, 1, vec![head("s", 1)])
14558 .await
14559 .expect("the current launch syncs");
14560
14561 supervisor.open_swap(OWNER, "n2".to_string());
14564 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
14565 assert_eq!(
14566 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
14567 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14568 );
14569 sync(&handler, &incumbent, 2, vec![head("s", 1)])
14570 .await
14571 .expect("the serving owner syncs during the swap");
14572
14573 supervisor.close_swap(OWNER);
14576 assert_eq!(
14577 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
14578 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14579 );
14580 sync(&handler, &incumbent, 3, vec![head("s", 1)])
14581 .await
14582 .expect("the serving owner syncs after the rollback");
14583 handler.cleanup_connection(candidate.connection_id).unwrap();
14584
14585 supervisor.open_swap(OWNER, "n3".to_string());
14590 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
14591 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
14592 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
14593 .await
14594 .expect("the promoted launch takes authority");
14595 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
14596 panic!("unexpected reply {reply:?}");
14597 };
14598 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
14599 assert_eq!(
14600 sync(&handler, &incumbent, 4, Vec::new()).await,
14601 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14602 );
14603 }
14604
14605 #[tokio::test]
14608 async fn closing_the_authority_connection_frees_sync_authority() {
14609 let supervisor = SupervisorHandle::new();
14610 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14611 let handler = ControlHandler::new(Arc::new(Registry::default()))
14612 .with_supervisor(supervisor.clone());
14613 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
14614 sync(&handler, &first, 10, vec![head("s", 1)])
14615 .await
14616 .unwrap();
14617 handler.cleanup_connection(first.connection_id).unwrap();
14618
14619 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
14620 sync(&handler, &second, 1, vec![head("s", 1)])
14621 .await
14622 .expect("the next connection takes the released authority");
14623 }
14624
14625 #[tokio::test]
14626 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
14627 let supervisor = SupervisorHandle::new();
14628 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14629 let handler =
14630 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
14631 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
14632 sync(&handler, &first, 10, vec![head("s", 1)])
14633 .await
14634 .unwrap();
14635 handler
14636 .handle_control_frame(
14637 &first,
14638 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
14639 )
14640 .await
14641 .unwrap();
14642 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
14643 sync(&handler, &second, 1, vec![head("s", 1)])
14644 .await
14645 .expect("GOODBYE releases authority even if the old socket remains open");
14646 }
14647
14648 #[tokio::test]
14649 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
14650 let registry = Arc::new(Registry::default());
14651 let supervisor_handle = SupervisorHandle::new();
14652 let supervisor =
14653 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
14654 .with_handle(supervisor_handle.clone())
14655 .with_daemon_incarnation("incarnation-7".to_string());
14656 supervisor
14659 .supervise_configured(
14660 ModuleSpec {
14661 module_id: OWNER.to_string(),
14662 program: PathBuf::from("/nonexistent/prefrontal-core"),
14663 args: Vec::new(),
14664 env: Vec::new(),
14665 reserved: false,
14666 reserved_prefixes: Vec::new(),
14667 protocol: ModuleProtocol::Subc,
14668 overlap: Default::default(),
14669 },
14670 false,
14671 )
14672 .unwrap();
14673 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
14674 let handler =
14675 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14676 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
14677
14678 let ModuleControlResponseToModule::ScopeDescribe {
14680 status,
14681 daemon_incarnation,
14682 owner_synced,
14683 owner_configured,
14684 scope,
14685 ..
14686 } = describe(&handler, &reader, OWNER, "s").await
14687 else {
14688 panic!("not a describe reply");
14689 };
14690 assert_eq!(status, ScopeStatus::NotLive);
14691 assert_eq!(daemon_incarnation, "incarnation-7");
14692 assert!(!owner_synced);
14693 assert!(owner_configured);
14694 assert!(scope.is_none());
14695
14696 let ModuleControlResponseToModule::ScopeDescribe {
14699 status,
14700 owner_configured,
14701 ..
14702 } = describe(&handler, &reader, "ghost", "s").await
14703 else {
14704 panic!("not a describe reply");
14705 };
14706 assert_eq!(status, ScopeStatus::NotLive);
14707 assert!(!owner_configured);
14708
14709 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
14711 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
14712 let ModuleControlResponseToModule::ScopeDescribe {
14713 status,
14714 scope_epoch,
14715 owner_synced,
14716 scope,
14717 ..
14718 } = describe(&handler, &reader, OWNER, "s").await
14719 else {
14720 panic!("not a describe reply");
14721 };
14722 assert_eq!(status, ScopeStatus::Live);
14723 assert_eq!(scope_epoch, Some(4));
14724 assert!(owner_synced);
14725 let stamp = scope.expect("a live scope carries its stamp");
14726 assert!(
14727 stamp.owner_authorized,
14728 "prefrontal-core is the default authority"
14729 );
14730 assert_eq!(stamp.kind, ScopeKind::Head);
14731 }
14732
14733 #[tokio::test]
14734 async fn scope_authority_owners_decides_owner_authorized() {
14735 let supervisor = SupervisorHandle::new();
14736 supervisor.set_spawn_nonce("broca", "b1".to_string());
14737 let handler = ControlHandler::new(Arc::new(Registry::default()))
14738 .with_supervisor(supervisor)
14739 .with_scope_authority_owners(vec!["broca".to_string()]);
14740 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
14741 let mut gated = head("s", 1);
14742 gated.attributes.agent_id = Some("agent".to_string());
14743 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
14744 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
14745 describe(&handler, &broca, "broca", "s").await
14746 else {
14747 panic!("not a describe reply");
14748 };
14749 assert!(scope.unwrap().owner_authorized);
14750 }
14751
14752 #[tokio::test]
14756 async fn scope_ops_and_the_scopes_capability_are_advertised() {
14757 let handler = ControlHandler::new(Arc::new(Registry::default()));
14758 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
14759 let ack = hello_via_sink(
14760 &handler,
14761 &ctx,
14762 &mut rx,
14763 hello_frame("m", PROTOCOL_VERSION, 1),
14764 )
14765 .await;
14766 let ack = parse_ack(&ack);
14767 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
14768 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
14769 }
14770 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
14771
14772 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
14773 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
14774 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
14775 let reply = handler
14776 .handle_control_frame(&client, frame)
14777 .await
14778 .unwrap()
14779 .pop()
14780 .unwrap();
14781 let ClientControlResponse::ServerDescribe { capabilities, .. } =
14782 serde_json::from_slice(&reply.body).unwrap()
14783 else {
14784 panic!("not a server.describe reply");
14785 };
14786 assert!(
14787 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
14788 "{capabilities:?}"
14789 );
14790 }
14791
14792 const PLEXUS: &str = "plexus";
14795 const OTHER: &str = "other";
14796 const AFT: &str = "aft";
14797 const BROCA: &str = "broca";
14798 const MAGIC: &str = "magic-context";
14799
14800 fn nonce(module_id: &str) -> String {
14801 format!("nonce-{module_id}")
14802 }
14803
14804 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
14805 let (tx, rx) = mpsc::channel(64);
14806 (
14807 RouteCtx {
14808 connection_id: ConnectionId::new(connection),
14809 egress: FrameSink::new(tx),
14810 },
14811 rx,
14812 )
14813 }
14814
14815 struct Rig {
14819 handler: ControlHandler,
14820 forwarding: Arc<ForwardingTable>,
14821 owner: RouteCtx,
14822 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14823 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
14824 generation: u64,
14825 next_connection: u64,
14826 _supervisor: Supervisor,
14827 }
14828
14829 async fn rig() -> Rig {
14830 rig_with_flow_support(true).await
14831 }
14832
14833 async fn rig_with_flow_support(flow_support: bool) -> Rig {
14834 rig_with_scope_capabilities(flow_support, false).await
14835 }
14836
14837 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
14838 let mut provides = Vec::new();
14839 if flow_support {
14840 provides.push("flow-scopes/v1");
14841 }
14842 if run_support {
14843 provides.push("agent-run-scopes/v1");
14844 }
14845 rig_with_target_capabilities(&provides).await
14846 }
14847
14848 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
14849 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
14850 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
14851 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
14854 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
14855 Frame::build(
14856 FrameType::Hello,
14857 control_flags(),
14858 0,
14859 0,
14860 connection,
14861 serde_json::to_vec(&body).unwrap(),
14862 )
14863 .unwrap()
14864 }
14865
14866 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
14867 let registry = Arc::new(Registry::default());
14868 let forwarding = Arc::new(ForwardingTable::default());
14869 let supervisor_handle = SupervisorHandle::new();
14870 let supervisor =
14871 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
14872 .with_handle(supervisor_handle.clone());
14873 supervisor
14874 .supervise_configured(
14875 ModuleSpec {
14876 module_id: OWNER.to_string(),
14877 program: PathBuf::from("/nonexistent/prefrontal-core"),
14878 args: Vec::new(),
14879 env: Vec::new(),
14880 reserved: false,
14881 reserved_prefixes: Vec::new(),
14882 protocol: ModuleProtocol::Subc,
14883 overlap: Default::default(),
14884 },
14885 false,
14886 )
14887 .unwrap();
14888 for module_id in [OWNER, AFT, BROCA, MAGIC] {
14889 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
14890 }
14891 let handler =
14892 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14893 .with_supervisor(supervisor_handle);
14894 let (owner, mut owner_rx) = wide_ctx(1);
14895 hello_via_sink(
14896 &handler,
14897 &owner,
14898 &mut owner_rx,
14899 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
14900 )
14901 .await;
14902 let mut modules = BTreeMap::new();
14903 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
14904 let (ctx, mut rx) = wide_ctx(connection);
14905 let hello = hello_with_capabilities(module_id, connection, provides);
14906 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
14907 modules.insert(module_id.to_string(), (ctx, rx));
14908 }
14909 Rig {
14910 handler,
14911 forwarding,
14912 owner,
14913 _owner_rx: owner_rx,
14914 modules,
14915 generation: 0,
14916 next_connection: 100,
14917 _supervisor: supervisor,
14918 }
14919 }
14920
14921 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
14922 ScopeCarrier::new(Principal::Reserved {
14923 module_id: module_id.to_string(),
14924 })
14925 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
14926 }
14927
14928 fn session(scope_epoch: u64) -> ScopeRecord {
14931 let mut record = head("s", scope_epoch);
14932 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
14933 record.attributes.agent_id = Some("agent-1".to_string());
14934 record.attributes.delegates = true;
14935 record
14936 }
14937
14938 impl Rig {
14939 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
14940 self.generation += 1;
14941 sync(&self.handler, &self.owner, self.generation, scopes)
14942 .await
14943 .expect("the owner's sync is accepted");
14944 }
14945
14946 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
14947 ScopeSelector::new(
14948 Principal::Reserved {
14949 module_id: OWNER.to_string(),
14950 },
14951 scope_ref.to_string(),
14952 scope_epoch,
14953 )
14954 }
14955
14956 fn open_frame(
14957 &mut self,
14958 opener: Option<&str>,
14959 target: &str,
14960 scope: Option<ScopeSelector>,
14961 ) -> (
14962 RouteCtx,
14963 mpsc::Receiver<crate::router::OutboundFrame>,
14964 Frame,
14965 ) {
14966 self.next_connection += 1;
14967 let (ctx, rx) = wide_ctx(self.next_connection);
14968 let root = unique_project_root("scoped-open");
14969 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
14970 target: RouteTarget::ToolProvider {
14971 module_id: target.to_string(),
14972 },
14973 identity: BindIdentity::new(
14974 root.path().to_path_buf(),
14975 "unit".to_string(),
14976 "session".to_string(),
14977 ),
14978 consumer_identity: opener.map(|module_id| ConsumerIdentity {
14979 module_id: module_id.to_string(),
14980 launch_nonce: nonce(module_id),
14981 }),
14982 consumer_capabilities: None,
14983 role_versions: None,
14984 admission_facts: None,
14985 scope,
14986 })
14987 .unwrap();
14988 let frame = Frame::build(
14989 FrameType::Request,
14990 control_flags(),
14991 0,
14992 0,
14993 self.next_connection,
14994 body,
14995 )
14996 .unwrap();
14997 (ctx, rx, frame)
14998 }
14999
15000 async fn refused(
15002 &mut self,
15003 opener: Option<&str>,
15004 target: &str,
15005 scope: Option<ScopeSelector>,
15006 ) -> String {
15007 self.refusal_body(opener, target, scope).await["code"]
15008 .as_str()
15009 .unwrap()
15010 .to_string()
15011 }
15012
15013 async fn refusal_body(
15014 &mut self,
15015 opener: Option<&str>,
15016 target: &str,
15017 scope: Option<ScopeSelector>,
15018 ) -> Value {
15019 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
15020 let replies = tokio::time::timeout(
15021 Duration::from_secs(2),
15022 self.handler.handle_control_frame(&ctx, frame),
15023 )
15024 .await
15025 .expect("the open must be refused before waiting for a bind ack")
15026 .unwrap();
15027 assert_eq!(replies.len(), 1, "{replies:?}");
15028 assert_eq!(replies[0].header.ty, FrameType::Error);
15029 let (_, module_rx) = self.modules.get_mut(target).unwrap();
15030 assert!(
15031 module_rx.try_recv().is_err(),
15032 "a refused open relays nothing"
15033 );
15034 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
15035 parse_error(&replies[0])
15036 }
15037
15038 async fn relayed(
15040 &mut self,
15041 opener: Option<&str>,
15042 target: &str,
15043 scope: Option<ScopeSelector>,
15044 ) -> Relayed {
15045 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
15046 let handler = self.handler.clone();
15047 let task_ctx = ctx.clone();
15048 let task = tokio::spawn(async move {
15049 handler
15050 .handle_control_frame(&task_ctx, frame)
15051 .await
15052 .unwrap()
15053 });
15054 let (_, module_rx) = self.modules.get_mut(target).unwrap();
15055 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
15056 .await
15057 .expect("the target receives the relayed route.bind")
15058 .unwrap()
15059 .frame;
15060 Relayed {
15061 target: target.to_string(),
15062 client: ctx,
15063 client_rx: rx,
15064 task,
15065 bind,
15066 }
15067 }
15068
15069 async fn ack(&self, relayed: &Relayed) {
15070 let (module, _) = &self.modules[&relayed.target];
15071 self.handler
15072 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
15073 .await
15074 .unwrap();
15075 }
15076
15077 async fn bound(
15079 &mut self,
15080 opener: Option<&str>,
15081 target: &str,
15082 scope: Option<ScopeSelector>,
15083 ) -> Bound {
15084 let relayed = self.relayed(opener, target, scope).await;
15085 self.ack(&relayed).await;
15086 let Relayed {
15087 target,
15088 client,
15089 mut client_rx,
15090 task,
15091 bind,
15092 } = relayed;
15093 assert!(
15094 task.await.unwrap().is_empty(),
15095 "the open is answered by commit"
15096 );
15097 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
15098 Bound {
15099 target,
15100 client,
15101 client_rx,
15102 channel,
15103 epoch,
15104 bind,
15105 }
15106 }
15107
15108 fn live(&self, route: &Bound) -> bool {
15109 matches!(
15110 self.forwarding
15111 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
15112 .unwrap(),
15113 DataRoute::Client(DataRouteState::Bound(_))
15114 )
15115 }
15116 }
15117
15118 struct Relayed {
15119 target: String,
15120 client: RouteCtx,
15121 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15122 task: tokio::task::JoinHandle<Vec<Frame>>,
15123 bind: Frame,
15124 }
15125
15126 struct Bound {
15127 target: String,
15128 client: RouteCtx,
15129 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15130 channel: u16,
15131 epoch: u32,
15132 bind: Frame,
15133 }
15134
15135 impl Bound {
15136 fn closed_reason(&mut self) -> RouteCloseReason {
15139 let mut reason = None;
15140 let mut goodbye = false;
15141 while let Ok(outbound) = self.client_rx.try_recv() {
15142 let frame = outbound.frame;
15143 match frame.header.ty {
15144 FrameType::Goodbye => {
15145 assert_eq!(
15146 (frame.header.channel, frame.header.epoch),
15147 (self.channel, self.epoch)
15148 );
15149 goodbye = true;
15150 }
15151 FrameType::Push => {
15152 let ClientControlPush::RouteClosed {
15153 reason: r,
15154 module_id,
15155 ..
15156 } = serde_json::from_slice(&frame.body).unwrap()
15157 else {
15158 panic!("unexpected push");
15159 };
15160 assert_eq!(module_id, self.target);
15161 reason = Some(r);
15162 }
15163 other => panic!("unexpected frame {other:?}"),
15164 }
15165 }
15166 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
15167 reason.expect("the client is told why the route closed")
15168 }
15169
15170 fn untouched(&mut self) -> bool {
15171 self.client_rx.try_recv().is_err()
15172 }
15173
15174 fn stamp(&self) -> Option<ScopeStamp> {
15175 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
15176 ModuleControlRequest::RouteBind { scope, .. } => scope,
15177 other => panic!("expected a route.bind, got {other:?}"),
15178 }
15179 }
15180 }
15181
15182 #[tokio::test]
15183 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
15184 ) {
15185 let mut rig = rig().await;
15186 let mut record = session(1);
15187 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
15188 record.child_owners = vec![Principal::Reserved {
15189 module_id: MAGIC.to_string(),
15190 }];
15191 rig.sync(vec![record]).await;
15192 let scope = || Some(rig_selector("s", Some(1)));
15193
15194 rig.bound(Some(OWNER), PLEXUS, scope()).await;
15197 rig.bound(Some(AFT), OTHER, scope()).await;
15198 rig.bound(Some(BROCA), PLEXUS, scope()).await;
15199
15200 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
15204 assert_eq!(
15205 rig.refused(opener, target, scope()).await,
15206 error_codes::SCOPE_NOT_CARRIER,
15207 "{opener:?} -> {target}"
15208 );
15209 }
15210 }
15211
15212 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15213 ScopeSelector::new(
15214 Principal::Reserved {
15215 module_id: OWNER.to_string(),
15216 },
15217 scope_ref.to_string(),
15218 scope_epoch,
15219 )
15220 }
15221
15222 #[tokio::test]
15223 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
15224 let mut rig = rig().await;
15225 rig.sync(vec![session(1)]).await;
15226 for opener in [OWNER, AFT] {
15227 assert_eq!(
15228 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
15229 .await,
15230 error_codes::SCOPE_EPOCH_REQUIRED,
15231 "{opener}"
15232 );
15233 }
15234 }
15235
15236 #[tokio::test]
15237 async fn admission_separates_not_synced_not_live_and_ended() {
15238 let mut rig = rig().await;
15239 let code = rig
15241 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15242 .await;
15243 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
15244 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
15245
15246 let ghost = ScopeSelector::new(
15248 Principal::Reserved {
15249 module_id: "ghost".to_string(),
15250 },
15251 "s".to_string(),
15252 Some(1),
15253 );
15254 assert_eq!(
15255 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
15256 error_codes::SCOPE_NOT_LIVE
15257 );
15258
15259 rig.sync(vec![session(2)]).await;
15260 assert_eq!(
15261 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
15262 .await,
15263 error_codes::SCOPE_NOT_LIVE
15264 );
15265 for epoch in [1, 3] {
15266 assert_eq!(
15267 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
15268 .await,
15269 error_codes::SCOPE_ENDED,
15270 "epoch {epoch}"
15271 );
15272 }
15273 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
15275 .await;
15276 }
15277
15278 #[tokio::test]
15279 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
15280 let mut rig = rig().await;
15281 rig.sync(vec![session(1)]).await;
15282 let route = rig
15283 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15284 .await;
15285 let stamp = route.stamp().expect("a scoped bind carries the stamp");
15286 assert_eq!(stamp.scope_ref, "s");
15287 assert_eq!(stamp.scope_epoch, 1);
15288 assert_eq!(stamp.kind, ScopeKind::Head);
15289 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
15290 assert!(stamp.attributes.delegates);
15291 assert!(stamp.owner_authorized);
15292 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15293 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
15294
15295 let (broca, mut broca_rx) = wide_ctx(50);
15298 hello_via_sink(
15299 &rig.handler,
15300 &broca,
15301 &mut broca_rx,
15302 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
15303 )
15304 .await;
15305 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
15306 .await
15307 .unwrap();
15308 let own = ScopeSelector::new(
15309 Principal::Reserved {
15310 module_id: BROCA.to_string(),
15311 },
15312 "b".to_string(),
15313 Some(1),
15314 );
15315 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
15316 assert!(!route.stamp().unwrap().owner_authorized);
15317 }
15318
15319 #[tokio::test]
15320 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
15321 let mut rig = rig().await;
15322 let mut record = head("s", 1);
15323 record.carriers = vec![carrier(AFT, None)];
15324 let flow_id = "Flow:run-7/step_2!~";
15325 record.attributes.flow_id = Some(flow_id.to_string());
15326 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
15327 .await
15328 .unwrap();
15329 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
15330 panic!("not a sync reply");
15331 };
15332 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
15333 let route = rig
15334 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15335 .await;
15336 assert!(rig.live(&route), "the stamped bind committed");
15337 let stamp = route.stamp().expect("a flow scope carries a stamp");
15338 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
15339 assert_eq!(stamp.attributes.agent_id, None);
15340 assert!(!stamp.attributes.delegates);
15341 assert!(stamp.owner_authorized);
15342 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15343 assert_eq!(unscoped.stamp(), None);
15344 }
15345
15346 #[tokio::test]
15347 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
15348 let mut rig = rig_with_flow_support(false).await;
15349 let mut record = session(1);
15350 record.attributes.flow_id = Some("flow:7".to_string());
15351 rig.sync(vec![record]).await;
15352 for opener in [OWNER, AFT] {
15353 let body = rig
15354 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
15355 .await;
15356 assert_eq!(body["code"], "target_flow_unsupported");
15357 let message = body["message"].as_str().unwrap();
15358 for required in [PLEXUS, "flow-scopes/v1"] {
15359 assert!(message.contains(required), "{message}");
15360 }
15361 }
15362 }
15363
15364 #[tokio::test]
15365 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
15366 let mut rig = rig_with_flow_support(true).await;
15367 let mut record = session(1);
15368 record.attributes.flow_id = Some("flow:7".to_string());
15369 rig.sync(vec![record]).await;
15370 for opener in [OWNER, AFT] {
15371 let route = rig
15372 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
15373 .await;
15374 assert!(rig.live(&route));
15375 assert_eq!(
15376 route.stamp().unwrap().attributes.flow_id.as_deref(),
15377 Some("flow:7")
15378 );
15379 }
15380 }
15381
15382 #[tokio::test]
15383 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
15384 use std::future::Future;
15385
15386 let mut rig = rig_with_flow_support(true).await;
15387 let mut record = session(1);
15388 record.attributes.flow_id = Some("flow:7".to_string());
15389 rig.sync(vec![record]).await;
15390 let (client, mut client_rx, frame) =
15391 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
15392 for _ in 0..64 {
15395 client.egress.try_send(route_bind_ack(1)).unwrap();
15396 }
15397 let handler = rig.handler.clone();
15398 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15399 std::future::poll_fn(|cx| {
15400 assert!(open.as_mut().poll(cx).is_pending());
15401 std::task::Poll::Ready(())
15402 })
15403 .await;
15404 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15405
15406 let old_connection = rig.modules[PLEXUS].0.connection_id;
15407 rig.handler.cleanup_connection(old_connection).unwrap();
15408 let (replacement, mut replacement_rx) = wide_ctx(200);
15409 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
15410 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
15411 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
15412 let hello = Frame::build(
15413 FrameType::Hello,
15414 control_flags(),
15415 0,
15416 0,
15417 200,
15418 serde_json::to_vec(&body).unwrap(),
15419 )
15420 .unwrap();
15421 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
15422
15423 client_rx.try_recv().unwrap();
15424 let replies = tokio::time::timeout(Duration::from_secs(2), open)
15425 .await
15426 .expect("the replacement is refused without waiting for a bind ack")
15427 .unwrap();
15428 assert_eq!(replies.len(), 1);
15429 let body = parse_error(&replies[0]);
15430 assert_eq!(body["code"], "target_flow_unsupported");
15431 for required in [PLEXUS, "flow-scopes/v1"] {
15432 assert!(body["message"].as_str().unwrap().contains(required));
15433 }
15434 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
15435 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
15436 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15437 assert!(rig
15438 .handler
15439 .registry
15440 .get_module_by_connection(replacement.connection_id)
15441 .unwrap()
15442 .is_some());
15443 }
15444
15445 #[tokio::test]
15446 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
15447 use std::future::Future;
15448 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
15449 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15450 const NEXT_HASH: &str =
15451 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
15452 let mut rig = rig().await;
15453 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15454 .await;
15455 let (client, mut rx, frame) = rig.open_frame(
15456 Some(AFT),
15457 PLEXUS,
15458 Some(
15459 rig.selector("s", Some(1))
15460 .with_bind_proof(Some(PROOF.into())),
15461 ),
15462 );
15463 for _ in 0..64 {
15466 client.egress.try_send(route_bind_ack(1)).unwrap();
15467 }
15468 let handler = rig.handler.clone();
15469 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15470 std::future::poll_fn(|cx| {
15471 assert!(open.as_mut().poll(cx).is_pending());
15472 std::task::Poll::Ready(())
15473 })
15474 .await;
15475 rig.sync(vec![
15476 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
15477 ])
15478 .await;
15479 rx.try_recv().unwrap();
15480 let replies = tokio::time::timeout(Duration::from_secs(2), open)
15481 .await
15482 .unwrap()
15483 .unwrap();
15484 assert_eq!(
15485 parse_error(&replies[0])["code"],
15486 error_codes::SCOPE_BIND_PROOF_MISMATCH
15487 );
15488 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
15489 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
15490 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15491 }
15492
15493 #[tokio::test]
15494 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
15495 {
15496 let mut rig = rig_with_flow_support(false).await;
15497 rig.sync(vec![session(1)]).await;
15498 let route = rig
15499 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15500 .await;
15501 assert!(rig.live(&route));
15502 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
15503 let mut record = session(1);
15504 record.attributes.flow_id = Some("flow:7".to_string());
15505 rig.sync(vec![record]).await;
15506 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
15507 assert!(rig.live(&unscoped));
15508 assert_eq!(unscoped.stamp(), None);
15509 }
15510
15511 #[tokio::test]
15512 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
15513 let mut rig = rig().await;
15514 let mut record = session(1);
15515 record.attributes.flow_id = Some("flow:7".to_string());
15516 rig.sync(vec![record.clone()]).await;
15517 let before = rig.forwarding.published_scope_tag(OWNER, "s");
15518 let mut owner_route = rig
15519 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
15520 .await;
15521 let mut carrier_route = rig
15522 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
15523 .await;
15524 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15525 rig.sync(vec![record.clone()]).await;
15526 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
15527 assert!(rig.live(&owner_route) && owner_route.untouched());
15528 assert!(rig.live(&carrier_route) && carrier_route.untouched());
15529
15530 record.attributes.flow_id = Some("flow:8".to_string());
15531 rig.sync(vec![record]).await;
15532 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
15533 let before = before.unwrap();
15534 assert_eq!(after.scope_epoch, before.scope_epoch);
15535 assert!(after.version > before.version);
15536 for route in [&mut owner_route, &mut carrier_route] {
15537 assert!(!rig.live(route));
15538 assert_eq!(
15539 route.closed_reason(),
15540 RouteCloseReason::ScopeDelegationChanged
15541 );
15542 }
15543 assert!(rig.live(&unscoped) && unscoped.untouched());
15544 for target in [PLEXUS, OTHER] {
15547 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
15548 let goodbye = module_rx
15549 .try_recv()
15550 .expect("the provider sees the drain")
15551 .frame;
15552 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15553 assert!(module_rx.try_recv().is_err());
15554 }
15555 let rebound = rig
15556 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15557 .await;
15558 assert_eq!(
15559 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
15560 Some("flow:8")
15561 );
15562 }
15563
15564 #[tokio::test]
15569 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
15570 let mut rig = rig().await;
15571 rig.sync(vec![session(1)]).await;
15572 let mut cotenant = rig
15573 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
15574 .await;
15575
15576 let mut changed = session(1);
15577 changed.child_owners.push(Principal::Reserved {
15578 module_id: MAGIC.to_string(),
15579 });
15580 let mut ended = None;
15581 for (code, next) in [
15582 (error_codes::SCOPE_CHANGED, vec![changed]),
15583 (error_codes::SCOPE_ENDED, Vec::new()),
15584 ] {
15585 let relayed = rig
15586 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15587 .await;
15588 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
15589 ended = Some(next.is_empty());
15590 rig.sync(next).await;
15591 rig.ack(&relayed).await;
15592 let replies = relayed.task.await.unwrap();
15593 assert_eq!(replies.len(), 1, "{replies:?}");
15594 assert_eq!(parse_error(&replies[0])["code"], code);
15595 assert_eq!(
15596 subc_protocol::error_codes::is_retryable_route_open(code),
15597 code == error_codes::SCOPE_CHANGED
15598 );
15599 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15600 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15602 let mut goodbyes = Vec::new();
15605 while let Ok(outbound) = plexus_rx.try_recv() {
15606 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
15607 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
15608 }
15609 assert!(
15610 goodbyes.contains(&(bind_channel, bind_epoch)),
15611 "{goodbyes:?}"
15612 );
15613 assert!(rig
15614 .handler
15615 .registry
15616 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
15617 .unwrap()
15618 .is_some());
15619 }
15620 assert_eq!(ended, Some(true));
15621 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
15624 }
15625
15626 #[tokio::test]
15629 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
15630 struct Case {
15631 name: &'static str,
15632 change: fn(&mut ScopeRecord),
15633 closed: [Option<RouteCloseReason>; 4],
15636 }
15637 use RouteCloseReason::*;
15638 let cases = [
15639 Case {
15640 name: "a carrier entry removed",
15641 change: |r| {
15642 r.carriers.retain(|c| {
15643 c.principal
15644 != Principal::Reserved {
15645 module_id: AFT.to_string(),
15646 }
15647 })
15648 },
15649 closed: [None, Some(ScopeCarrierRemoved), None, None],
15650 },
15651 Case {
15652 name: "a target removed from a carrier",
15653 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
15654 closed: [None, None, None, Some(ScopeCarrierRemoved)],
15655 },
15656 Case {
15657 name: "a bare carrier narrowed to targets",
15658 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
15659 closed: [None, Some(ScopeCarrierRemoved), None, None],
15660 },
15661 Case {
15662 name: "delegates turned off",
15663 change: |r| r.attributes.delegates = false,
15664 closed: [Some(ScopeDelegationChanged); 4],
15665 },
15666 Case {
15667 name: "agent_id changed",
15668 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
15669 closed: [Some(ScopeDelegationChanged); 4],
15670 },
15671 Case {
15672 name: "a carrier added, child owners changed, the record re-sent",
15673 change: |r| {
15674 r.carriers.push(carrier(MAGIC, None));
15675 r.child_owners.push(Principal::Reserved {
15676 module_id: MAGIC.to_string(),
15677 });
15678 },
15679 closed: [None; 4],
15680 },
15681 Case {
15682 name: "a target added",
15683 change: |r| {
15684 r.carriers[1]
15685 .targets
15686 .as_mut()
15687 .unwrap()
15688 .push("third".to_string())
15689 },
15690 closed: [None; 4],
15691 },
15692 Case {
15693 name: "delegates turned on",
15694 change: |r| r.attributes.delegates = true,
15695 closed: [None; 4],
15696 },
15697 ];
15698 for case in cases {
15699 let mut rig = rig().await;
15700 rig.sync(vec![session(1)]).await;
15701 let scope = || Some(rig_selector("s", Some(1)));
15702 let mut routes = [
15703 rig.bound(Some(OWNER), PLEXUS, scope()).await,
15704 rig.bound(Some(AFT), PLEXUS, scope()).await,
15705 rig.bound(Some(BROCA), PLEXUS, scope()).await,
15706 rig.bound(Some(BROCA), OTHER, scope()).await,
15707 ];
15708 let mut record = session(1);
15709 (case.change)(&mut record);
15710 rig.sync(vec![record]).await;
15711 for (index, expected) in case.closed.iter().enumerate() {
15712 let route = &mut routes[index];
15713 match expected {
15714 Some(reason) => {
15715 assert!(!rig.live(route), "{}: route {index} still live", case.name);
15716 assert_eq!(
15717 route.closed_reason(),
15718 *reason,
15719 "{}: route {index}",
15720 case.name
15721 );
15722 }
15723 None => {
15724 assert!(rig.live(route), "{}: route {index} closed", case.name);
15725 assert!(
15726 route.untouched(),
15727 "{}: route {index} was told something",
15728 case.name
15729 );
15730 }
15731 }
15732 }
15733 }
15734 }
15735
15736 #[tokio::test]
15737 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
15738 for next in [Vec::new(), vec![session(2)]] {
15740 let mut rig = rig().await;
15741 rig.sync(vec![session(1)]).await;
15742 let mut route = rig
15743 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15744 .await;
15745 rig.sync(next).await;
15746 assert!(!rig.live(&route));
15747 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
15748 }
15749
15750 let mut rig = rig().await;
15753 let mut child = session(1);
15754 child.scope_ref = "child".to_string();
15755 child.kind = ScopeKind::Worker;
15756 child.parent = Some(ScopeParent::new(
15757 Principal::Reserved {
15758 module_id: OWNER.to_string(),
15759 },
15760 "s".to_string(),
15761 1,
15762 ));
15763 rig.sync(vec![session(1), child.clone()]).await;
15764 let mut child_route = rig
15765 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
15766 .await;
15767 assert_eq!(
15768 child_route.stamp().unwrap().parent_state,
15769 Some(ParentState::Linked)
15770 );
15771 rig.sync(vec![child]).await;
15772 assert!(!rig.live(&child_route));
15773 assert_eq!(
15774 child_route.closed_reason(),
15775 RouteCloseReason::ScopeParentEnded
15776 );
15777 }
15778
15779 #[tokio::test]
15780 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
15781 ) {
15782 let mut rig = rig().await;
15783 rig.sync(vec![session(1)]).await;
15784 let mut route = rig
15785 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15786 .await;
15787 let before = rig.forwarding.published_scope_tag(OWNER, "s");
15788 rig.sync(vec![session(1)]).await;
15789 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
15790 assert!(rig.live(&route) && route.untouched());
15791
15792 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
15796 .forwarding
15797 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
15798 .unwrap()
15799 else {
15800 panic!("the route is bound");
15801 };
15802 binding.flow.acquire_tagged(9, false).await.unwrap();
15803 let mut widened = session(1);
15804 widened.carriers.push(carrier(MAGIC, None));
15805 rig.sync(vec![widened]).await;
15806 assert!(rig.live(&route) && route.untouched());
15807 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15808 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
15809 assert_eq!(binding.flow.in_flight(), 1);
15812 binding
15813 .flow
15814 .acquire_tagged(10, false)
15815 .await
15816 .expect("the flow is still open");
15817 }
15818
15819 #[tokio::test]
15822 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
15823 let mut rig = rig().await;
15824 rig.sync(vec![session(1)]).await;
15825 let mut on_incumbent = rig
15826 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15827 .await;
15828
15829 let (candidate, _candidate_rx) = wide_ctx(9);
15832 let registration = rig
15833 .handler
15834 .registry
15835 .register_candidate_with_control_ops(
15836 manifest(PLEXUS, PROTOCOL_VERSION),
15837 PROTOCOL_VERSION,
15838 candidate.connection_id,
15839 module_baseline_control_ops(),
15840 )
15841 .unwrap();
15842 rig.forwarding
15843 .register_candidate_module_connection(
15844 candidate.connection_id,
15845 PLEXUS.to_string(),
15846 PROTOCOL_VERSION,
15847 manifest_concurrency(®istration.manifest),
15848 candidate.egress.clone(),
15849 )
15850 .unwrap();
15851 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
15852 rig.handler
15853 .registry
15854 .promote_candidate(PLEXUS)
15855 .unwrap()
15856 .unwrap();
15857 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
15858
15859 rig.sync(Vec::new()).await;
15860 assert!(!rig.live(&on_incumbent));
15861 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
15862 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15863 let goodbye = incumbent_rx
15864 .try_recv()
15865 .expect("the superseded endpoint is told")
15866 .frame;
15867 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15868 }
15869 }
15870}
15871
15872#[cfg(test)]
15873mod concurrency_default_exposure_tests {
15874 use super::*;
15875
15876 fn hello_body(role_json: &str) -> Vec<u8> {
15877 format!(
15878 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
15879 )
15880 .into_bytes()
15881 }
15882
15883 fn manifest_from(body: &[u8]) -> ModuleManifest {
15884 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
15885 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
15886 .expect("manifest parses")
15887 }
15888
15889 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
15890
15891 #[test]
15892 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
15893 let body = hello_body(&format!(
15894 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
15895 ));
15896 let manifest = manifest_from(&body);
15897 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
15900 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
15901 }
15902
15903 #[test]
15904 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
15905 let body = hello_body(&format!(
15906 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
15907 ));
15908 let manifest = manifest_from(&body);
15909 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
15910 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
15911 }
15912
15913 #[test]
15914 fn non_management_roles_are_never_reported() {
15915 let body = hello_body(
15916 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
15917 );
15918 let manifest = manifest_from(&body);
15919 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
15920 }
15921}