1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_event_declarations,
25 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
26 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
27 },
28 scope::{
29 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
30 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
31 SCOPE_SYNC_OP,
32 },
33 session::{
34 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
35 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
36 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
37 ROLE_VERSIONS_FIELD,
38 },
39 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
40 Priority, RouteTarget, PROTOCOL_VERSION,
41};
42use tokio::time::{timeout_at, Instant};
43use tracing::{debug, info, warn};
44
45use crate::{
46 caller_identity::CallerIdentity,
47 capability_requirements::{
48 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
49 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
50 RuntimeModule,
51 },
52 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
53 forwarding::{
54 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
55 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
56 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
57 },
58 observability::{
59 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
60 },
61 privileged_prompt::{prompt_text, PrivilegedOperation},
62 provenance::{
63 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
64 },
65 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
66 router::{RouteCtx, RouterError},
67 scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
68 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
69 stderr_tail::{CaptureState, TailEntry},
70 supervise::{
71 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
72 SupervisorHandle, SwapHelloAdmission,
73 },
74 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
75};
76
77pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
83
84const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
85const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
86const CAP_PING_PONG: &str = "ping_pong_v1";
87const CAP_SESSION_ATTACH: &str = "session_attach_v1";
88const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
89
90const SUBC_CONTROL_OPS: &[&str] = &[
91 ops::ACTIVITY_RECORD,
92 ops::ACTIVITY_QUERY,
93 ops::ACTIVITY_SUBSCRIBE,
94 ops::SERVER_DESCRIBE,
95 ops::CATALOG_LIST,
96 ops::ROUTE_OPEN,
97 ops::ROUTE_POLL,
98 ops::ROUTE_CLOSING,
99 ops::ROUTE_CLOSED,
100 ops::SUPERVISOR_LIST,
101 ops::SUPERVISOR_RESTART,
102 ops::SUPERVISOR_SWAP,
103 ops::SUPERVISOR_RELOAD,
104 ops::SUPERVISOR_RESCAN,
105 ops::SUPERVISOR_RELEASE_RESERVED,
106 ops::SUPERVISOR_SET_ENABLED,
107 ops::SUPERVISOR_HEALTH_PROBE,
108 ops::SUPERVISOR_HEALTH,
109 ops::SUPERVISOR_STDERR_TAIL,
110 ops::SUPERVISOR_TERMINALS,
111 ops::SUPERVISOR_ROUTES,
112 ops::SUPERVISOR_PROVENANCE,
113 ops::SUPERVISOR_SPAWN_SNAPSHOT,
114 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
115];
116
117const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
118 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
119 "supervisor.live_roots",
120 SCOPE_SYNC_OP,
121 SCOPE_APPLY_OP,
122 SCOPE_DESCRIBE_OP,
123 "operator.confirm",
124];
125
126const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
130
131const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
132
133pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
141
142pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
157
158pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
176
177const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
178const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
179
180fn reload_verdict(
181 configured: &Path,
182 spawned_from: Option<&Path>,
183 image: subc_control::RunningImageAgreement,
184) -> PendingReloadVerdict {
185 let path = match spawned_from {
186 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
187 ReloadPathAgreement::Match
188 }
189 Some(spawned_from) => ReloadPathAgreement::Mismatch {
190 configured: configured.to_path_buf(),
191 spawned_from: spawned_from.to_path_buf(),
192 },
193 None => ReloadPathAgreement::Unavailable {
194 reason: if matches!(
195 image,
196 subc_control::RunningImageAgreement::Unavailable {
197 reason: subc_control::RunningImageUnavailableReason::NotRunning
198 }
199 ) {
200 ReloadPathUnavailableReason::NotRunning
201 } else {
202 ReloadPathUnavailableReason::SpawnedPathUnavailable
203 },
204 },
205 };
206 PendingReloadVerdict { path, image }
207}
208
209fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
210 if configured == spawned_from {
211 return true;
212 }
213 #[cfg(windows)]
214 {
215 match (
221 std::fs::canonicalize(configured),
222 std::fs::canonicalize(spawned_from),
223 ) {
224 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
225 _ => false,
226 }
227 }
228 #[cfg(not(windows))]
229 {
230 false
231 }
232}
233
234#[derive(Clone)]
235struct DaemonProvenanceFacts {
236 build: DaemonBuildProvenance,
237 pid: Option<u32>,
238 started_at_ms: Option<u64>,
239 start_clock: Option<crate::clock::StartClock>,
240 executable_path: Option<PathBuf>,
241 executable_identity: Option<SpawnedFileIdentity>,
242 process_start_time: Option<u64>,
243 probe: ExecutableIdentityProbe,
244}
245
246impl Default for DaemonProvenanceFacts {
247 fn default() -> Self {
248 Self {
249 build: DaemonBuildProvenance {
250 build_git_sha: None,
251 build_lock_digest: None,
252 },
253 pid: None,
254 started_at_ms: None,
255 start_clock: None,
256 executable_path: None,
257 executable_identity: None,
258 process_start_time: None,
259 probe: ExecutableIdentityProbe::default(),
260 }
261 }
262}
263
264#[derive(Debug, Clone)]
265struct SupervisorRescanContext {
266 supervisor: Supervisor,
267 config_path: PathBuf,
268 configured_port: Option<u16>,
269 storage_config: Option<crate::daemon_config::StorageConfig>,
270 admission_facts_carrier_module_id: Option<String>,
271 admission_facts_targets: Option<Vec<String>>,
272 scope_authority_owners: Vec<String>,
273 privileged_operator_prompt: PrivilegedOperatorPrompt,
274}
275
276const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
297 "reloading",
298 "supervisor_not_live",
299 "registration_not_active",
300 "no_forwarding_connection",
301 "relay_send_failed",
302];
303
304#[derive(Clone)]
306pub struct ControlHandler {
307 registry: Arc<Registry>,
308 activity: Option<crate::activity::ActivityLedger>,
309 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
310 forwarding: Arc<ForwardingTable>,
311 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
312 supervisor: SupervisorHandle,
313 subc_capabilities: Arc<[String]>,
314 route_bind_relay_timeout: Duration,
318 route_bind_relay_timeouts: BTreeMap<String, Duration>,
322 route_bind_breakers: RouteBindBreakers,
325 route_bind_concurrency: RouteBindConcurrency,
328 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
333 route_bind_breaker_threshold: u32,
335 route_bind_breaker_cooldown: Duration,
337 health_probe_timeout: Duration,
338 storage_config: Option<crate::daemon_config::StorageConfig>,
341 machine_id: Option<crate::machine_id::MachineId>,
345 admission_facts_carrier_module_id: Option<String>,
346 admission_facts_targets: Option<Vec<String>>,
347 scopes: Arc<RwLock<ScopeTable>>,
351 scope_deadline_added: Arc<tokio::sync::Notify>,
355 scope_authority_owners: Vec<String>,
358 privileged_operator_prompt: PrivilegedOperatorPrompt,
360 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
363 #[cfg(test)]
364 caller_lookup: Option<TestCallerLookup>,
365 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
368 rescan: Option<SupervisorRescanContext>,
369 connected_clients: ConnectedClients,
370 counters: DaemonCounters,
371 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
372 daemon_provenance: DaemonProvenanceFacts,
373 #[cfg(test)]
374 control_dispatch_delay: Option<Duration>,
375 #[cfg(test)]
376 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
377 #[cfg(test)]
381 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
382 #[cfg(test)]
388 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
389}
390
391impl fmt::Debug for ControlHandler {
392 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
393 f.debug_struct("ControlHandler")
394 .field("registry", &self.registry)
395 .field("forwarding", &self.forwarding)
396 .field("process_liveness", &self.process_liveness.is_some())
397 .field("supervisor", &self.supervisor)
398 .field("subc_capabilities", &self.subc_capabilities)
399 .finish()
400 }
401}
402
403struct RouteOpenRequest {
404 target: RouteTarget,
405 identity: BindIdentity,
406 consumer_identity: Option<ConsumerIdentity>,
407 consumer_capabilities: Option<Vec<String>>,
408 role_versions: Option<BTreeMap<String, String>>,
409 admission_facts: Option<serde_json::Value>,
410 scope: Option<ScopeSelector>,
411}
412
413struct RouteBindReservationGuard {
414 forwarding: Arc<ForwardingTable>,
415 endpoint: ModuleEndpointId,
416 relay_corr: u64,
417 armed: bool,
418}
419
420struct ModuleControlRpcGuard {
421 forwarding: Arc<ForwardingTable>,
422 endpoint: ModuleEndpointId,
423 corr: u64,
424 armed: bool,
425}
426
427impl ModuleControlRpcGuard {
428 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
429 Self {
430 forwarding,
431 endpoint,
432 corr,
433 armed: true,
434 }
435 }
436
437 fn disarm(&mut self) {
438 self.armed = false;
439 }
440}
441
442impl Drop for ModuleControlRpcGuard {
443 fn drop(&mut self) {
444 if self.armed {
445 let _ = self
446 .forwarding
447 .cancel_module_control_rpc(self.endpoint, self.corr);
448 }
449 }
450}
451
452impl RouteBindReservationGuard {
453 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
454 Self {
455 forwarding,
456 endpoint,
457 relay_corr,
458 armed: true,
459 }
460 }
461
462 fn release_and_disarm(&mut self) {
463 if !self.armed {
464 return;
465 }
466 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
467 self.endpoint,
468 self.relay_corr,
469 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
470 ) {
471 send_goodbye_target_best_effort(
472 &self.forwarding.counters(),
473 &target,
474 "canceled route.bind",
475 );
476 }
477 self.armed = false;
478 }
479
480 fn disarm(&mut self) {
481 self.armed = false;
482 }
483}
484
485impl Drop for RouteBindReservationGuard {
486 fn drop(&mut self) {
487 self.release_and_disarm();
488 }
489}
490
491#[derive(Debug, Clone, Default)]
521pub(crate) struct RouteBindBreakers {
522 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
523}
524
525#[derive(Debug, Clone, Default)]
526pub(crate) struct RouteBindConcurrency {
527 modules: Arc<Mutex<HashMap<String, usize>>>,
528}
529
530struct RouteBindConcurrencyGuard {
531 concurrency: RouteBindConcurrency,
532 module_id: String,
533}
534
535impl RouteBindConcurrency {
536 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
539 let mut modules = self
540 .modules
541 .lock()
542 .expect("route.bind concurrency mutex poisoned");
543 let in_flight = modules.entry(module_id.to_string()).or_default();
544 if *in_flight >= limit {
545 return Err(*in_flight);
546 }
547 *in_flight += 1;
548 Ok(RouteBindConcurrencyGuard {
549 concurrency: self.clone(),
550 module_id: module_id.to_string(),
551 })
552 }
553}
554
555impl Drop for RouteBindConcurrencyGuard {
556 fn drop(&mut self) {
557 let mut modules = self
558 .concurrency
559 .modules
560 .lock()
561 .expect("route.bind concurrency mutex poisoned");
562 let remove = {
563 let in_flight = modules
564 .get_mut(&self.module_id)
565 .expect("admitted route.bind has a concurrency entry");
566 *in_flight -= 1;
567 *in_flight == 0
568 };
569 if remove {
570 modules.remove(&self.module_id);
571 }
572 }
573}
574
575#[derive(Debug, Default)]
576struct ModuleBreakerState {
577 consecutive_timeouts: u32,
579 cooldown_until: Option<Instant>,
582 probe_in_flight: Option<Arc<()>>,
587}
588
589enum RouteBindAdmission<'a> {
591 Admitted {
592 guard: RouteBindBreakerGuard<'a>,
593 probe: bool,
596 },
597 Refused {
598 consecutive_timeouts: u32,
599 retry_in: Duration,
603 probe_in_flight: bool,
604 },
605}
606
607struct RouteBindBreakerGuard<'a> {
614 breakers: RouteBindBreakers,
615 module_id: &'a str,
616 probe_token: Option<Arc<()>>,
617 settled: bool,
618}
619
620impl RouteBindBreakerGuard<'_> {
621 fn record_accepted(&mut self) -> bool {
625 self.settled = true;
626 self.breakers.record_accepted(self.module_id)
627 }
628
629 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
632 self.settled = true;
633 self.breakers.record_timeout(
634 self.module_id,
635 self.probe_token.as_ref(),
636 threshold,
637 cooldown,
638 )
639 }
640
641 fn record_inconclusive(&mut self) {
650 self.settled = true;
651 self.breakers
652 .record_inconclusive(self.module_id, self.probe_token.as_ref());
653 }
654}
655
656impl Drop for RouteBindBreakerGuard<'_> {
657 fn drop(&mut self) {
658 if !self.settled {
659 self.breakers
660 .record_inconclusive(self.module_id, self.probe_token.as_ref());
661 }
662 }
663}
664
665struct BreakerOpened {
669 consecutive_timeouts: u32,
670 reopened_after_probe: bool,
673}
674
675impl RouteBindBreakers {
676 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
677 self.modules
678 .lock()
679 .expect("route.bind breaker mutex poisoned")
680 }
681
682 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
685 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
686 probe: probe_token.is_some(),
687 guard: RouteBindBreakerGuard {
688 breakers: self.clone(),
689 module_id,
690 probe_token,
691 settled: false,
692 },
693 };
694
695 let mut modules = self.lock();
696 let Some(state) = modules.get_mut(module_id) else {
697 return admitted(None);
698 };
699 let Some(cooldown_until) = state.cooldown_until else {
700 return admitted(None);
701 };
702 if state.probe_in_flight.is_some() {
703 return RouteBindAdmission::Refused {
704 consecutive_timeouts: state.consecutive_timeouts,
705 retry_in: Duration::ZERO,
706 probe_in_flight: true,
707 };
708 }
709 let now = Instant::now();
710 if now < cooldown_until {
711 return RouteBindAdmission::Refused {
712 consecutive_timeouts: state.consecutive_timeouts,
713 retry_in: cooldown_until - now,
714 probe_in_flight: false,
715 };
716 }
717 let token = Arc::new(());
718 state.probe_in_flight = Some(Arc::clone(&token));
719 admitted(Some(token))
720 }
721
722 fn record_accepted(&self, module_id: &str) -> bool {
723 self.lock()
724 .remove(module_id)
725 .is_some_and(|state| state.cooldown_until.is_some())
726 }
727
728 fn record_timeout(
729 &self,
730 module_id: &str,
731 probe_token: Option<&Arc<()>>,
732 threshold: u32,
733 cooldown: Duration,
734 ) -> Option<BreakerOpened> {
735 let mut modules = self.lock();
736 let state = modules.entry(module_id.to_string()).or_default();
737 let was_open = state.cooldown_until.is_some();
738 let was_probe = Self::owns_probe(state, probe_token);
739 if was_probe {
740 state.probe_in_flight = None;
741 }
742 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
743 if state.consecutive_timeouts < threshold {
744 return None;
745 }
746 state.cooldown_until = Some(Instant::now() + cooldown);
747 Some(BreakerOpened {
748 consecutive_timeouts: state.consecutive_timeouts,
749 reopened_after_probe: was_open && was_probe,
750 })
751 }
752
753 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
754 state
758 .probe_in_flight
759 .as_ref()
760 .zip(token)
761 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
762 }
763
764 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
765 if let Some(state) = self.lock().get_mut(module_id) {
766 if Self::owns_probe(state, probe_token) {
767 state.probe_in_flight = None;
768 }
769 }
770 }
771
772 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
787 self.lock()
788 .remove(module_id)
789 .map(|state| state.consecutive_timeouts)
790 .filter(|discarded| *discarded > 0)
791 }
792
793 fn open_snapshot(&self) -> Option<serde_json::Value> {
800 let now = Instant::now();
801 let modules = self.lock();
802 let open = modules
803 .iter()
804 .filter_map(|(module_id, state)| {
805 let cooldown_until = state.cooldown_until?;
806 Some((
807 module_id.clone(),
808 serde_json::json!({
809 "consecutive_timeouts": state.consecutive_timeouts,
810 "cooldown_remaining_ms":
811 cooldown_until.saturating_duration_since(now).as_millis() as u64,
812 "probe_in_flight": state.probe_in_flight.is_some(),
813 }),
814 ))
815 })
816 .collect::<serde_json::Map<String, serde_json::Value>>();
817 (!open.is_empty()).then_some(serde_json::Value::Object(open))
818 }
819}
820
821impl ControlHandler {
822 pub fn new(registry: Arc<Registry>) -> Self {
823 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
824 }
825
826 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
827 let counters = forwarding.counters();
828 let route_bind_breakers = forwarding.route_bind_breakers();
832 let route_bind_concurrency = forwarding.route_bind_concurrency();
833 let route_outages = forwarding.route_outages();
834 Self {
835 registry,
836 activity: None,
837 activity_streams: Arc::new(Mutex::new(HashMap::new())),
838 forwarding,
839 process_liveness: None,
840 supervisor: SupervisorHandle::new(),
841 subc_capabilities: Arc::from([
842 CAP_MANIFEST_REGISTRATION.to_string(),
843 CAP_CHANNEL_LIFECYCLE.to_string(),
844 CAP_PING_PONG.to_string(),
845 CAP_SESSION_ATTACH.to_string(),
846 CAP_ADMISSION_FACTS_RELAY.to_string(),
847 CAP_SCOPES_V1.to_string(),
848 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
849 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
850 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
851 ]),
852 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
853 route_bind_relay_timeouts: BTreeMap::new(),
854 route_bind_breakers,
855 route_bind_concurrency,
856 route_outages,
857 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
858 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
859 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
860 storage_config: None,
861 machine_id: None,
862 admission_facts_carrier_module_id: None,
863 admission_facts_targets: None,
864 scopes: Arc::new(RwLock::new(ScopeTable::new(
865 crate::daemon_config::default_scope_authority_owners(),
866 ))),
867 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
868 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
869 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
870 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
871 #[cfg(test)]
872 caller_lookup: None,
873 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
874 rescan: None,
875 connected_clients: ConnectedClients::new(),
876 counters,
877 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
878 daemon_provenance: DaemonProvenanceFacts::default(),
879 #[cfg(test)]
880 control_dispatch_delay: None,
881 #[cfg(test)]
882 provenance_probe_override: None,
883 #[cfg(test)]
884 route_bind_relay_sent_hold: None,
885 #[cfg(test)]
886 hello_registry_to_forwarding_hold: None,
887 }
888 }
889
890 pub fn with_storage_config(
893 mut self,
894 storage_config: Option<crate::daemon_config::StorageConfig>,
895 ) -> Self {
896 self.storage_config = storage_config;
897 self
898 }
899
900 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
903 self.activity = Some(crate::activity::ActivityLedger::open(
904 run_dir.join("activity.jsonl"),
905 incarnation,
906 ));
907 self.supervisor.set_activity(self.activity.clone());
908 self
909 }
910
911 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
914 self.machine_id = machine_id;
915 self
916 }
917
918 pub fn with_admission_facts_config(
922 mut self,
923 carrier_module_id: Option<String>,
924 targets: Option<Vec<String>>,
925 ) -> Self {
926 self.admission_facts_carrier_module_id = carrier_module_id;
927 self.admission_facts_targets = targets;
928 self
929 }
930
931 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
935 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
936 self.scope_authority_owners = owners;
937 self
938 }
939
940 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
942 self.privileged_operator_prompt = mode;
943 self
944 }
945
946 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
949 self.route_bind_relay_timeout = timeout;
950 self
951 }
952
953 pub fn with_route_bind_relay_timeouts(
959 mut self,
960 timeouts: impl IntoIterator<Item = (String, Duration)>,
961 ) -> Self {
962 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
963 self
964 }
965
966 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
972 self.route_bind_relay_timeouts
973 .get(module_id)
974 .copied()
975 .unwrap_or(self.route_bind_relay_timeout)
976 }
977
978 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
986 self.route_bind_breaker_threshold = threshold.max(1);
987 self.route_bind_breaker_cooldown = cooldown;
988 self
989 }
990
991 #[cfg(test)]
992 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
993 self.health_probe_timeout = timeout;
994 self
995 }
996
997 #[cfg(test)]
998 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
999 self.control_dispatch_delay = Some(delay);
1000 self
1001 }
1002
1003 #[cfg(test)]
1007 pub(crate) fn with_route_bind_relay_sent_hold(
1008 mut self,
1009 hold: Arc<tokio::sync::Barrier>,
1010 ) -> Self {
1011 self.route_bind_relay_sent_hold = Some(hold);
1012 self
1013 }
1014
1015 #[cfg(test)]
1020 pub(crate) fn with_hello_registry_to_forwarding_hold(
1021 mut self,
1022 hold: Arc<std::sync::Barrier>,
1023 ) -> Self {
1024 self.hello_registry_to_forwarding_hold = Some(hold);
1025 self
1026 }
1027
1028 #[cfg(test)]
1029 fn hold_between_registry_and_forwarding(&self) {
1030 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1031 hold.wait();
1032 hold.wait();
1033 }
1034 }
1035
1036 pub fn with_process_liveness(
1037 mut self,
1038 process_liveness: Arc<dyn ModuleProcessLiveness>,
1039 ) -> Self {
1040 self.process_liveness = Some(process_liveness);
1041 self
1042 }
1043
1044 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1045 self.supervisor = supervisor;
1046 self.supervisor.set_activity(self.activity.clone());
1047 self
1048 }
1049
1050 pub fn with_daemon_provenance(
1051 mut self,
1052 pid: u32,
1053 started_at_ms: u64,
1054 executable_path: Option<PathBuf>,
1055 build_git_sha: Option<String>,
1056 build_lock_digest: Option<String>,
1057 ) -> Self {
1058 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1059 let process_start_time = process_start_time(pid);
1060 self.daemon_provenance = DaemonProvenanceFacts {
1061 build: DaemonBuildProvenance {
1062 build_git_sha,
1063 build_lock_digest,
1064 },
1065 pid: Some(pid),
1066 started_at_ms: Some(started_at_ms),
1067 start_clock: None,
1068 executable_path,
1069 executable_identity,
1070 process_start_time,
1071 probe: ExecutableIdentityProbe::default(),
1072 };
1073 self
1074 }
1075
1076 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1077 self.daemon_provenance.start_clock = Some(clock);
1078 self
1079 }
1080
1081 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1083 self.scopes
1084 .write()
1085 .unwrap_or_else(|poisoned| poisoned.into_inner())
1086 .set_wall_clock(Arc::new(clock));
1087 self
1088 }
1089
1090 #[cfg(test)]
1091 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1092 self.provenance_probe_override = Some(result);
1093 self
1094 }
1095
1096 pub fn with_capability_config(
1100 self,
1101 modules: impl IntoIterator<Item = (String, bool)>,
1102 reserved_capabilities: BTreeMap<String, String>,
1103 ) -> Self {
1104 self.capability_evaluator
1105 .configure(modules, reserved_capabilities);
1106 self
1107 }
1108
1109 pub fn with_supervisor_rescan(
1110 mut self,
1111 supervisor: Supervisor,
1112 config_path: impl Into<PathBuf>,
1113 configured_port: Option<u16>,
1114 ) -> Self {
1115 self.rescan = Some(SupervisorRescanContext {
1116 supervisor,
1117 config_path: config_path.into(),
1118 configured_port,
1119 storage_config: self.storage_config.clone(),
1120 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1121 admission_facts_targets: self.admission_facts_targets.clone(),
1122 scope_authority_owners: self.scope_authority_owners.clone(),
1123 privileged_operator_prompt: self.privileged_operator_prompt,
1124 });
1125 self
1126 }
1127
1128 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1129 self.connected_clients = connected_clients;
1130 self
1131 }
1132
1133 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1134 Arc::clone(&self.forwarding)
1135 }
1136
1137 pub(crate) fn counters(&self) -> DaemonCounters {
1138 self.counters.clone()
1139 }
1140
1141 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1144 tokio::spawn(async move {
1145 loop {
1146 self.capability_evaluator
1147 .wait_for_change_or_deadline()
1148 .await;
1149 self.refresh_capability_requirements();
1150 }
1151 });
1152 }
1153
1154 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1160 tokio::spawn(async move {
1161 loop {
1162 let has_deadlines = self
1163 .scopes
1164 .read()
1165 .unwrap_or_else(|poisoned| poisoned.into_inner())
1166 .has_deadlines();
1167 if !has_deadlines {
1168 self.scope_deadline_added.notified().await;
1169 continue;
1170 }
1171 tokio::time::sleep(Duration::from_secs(1)).await;
1172 if let Err(error) = self.sweep_expired_scopes() {
1173 warn!(%error, "scope expiry sweep failed");
1174 }
1175 }
1176 });
1177 }
1178
1179 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1180 let mut table = self
1181 .scopes
1182 .write()
1183 .unwrap_or_else(|poisoned| poisoned.into_inner());
1184 let swept = table.sweep_expired();
1185 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1186 drop(table);
1187 self.close_scope_drained_routes(drained);
1188 Ok(())
1189 }
1190
1191 fn publish_scope_changes(
1192 &self,
1193 changes: &[crate::scopes::ScopeTagChange],
1194 expired: &[crate::scopes::ScopeExpired],
1195 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1196 if changes.is_empty() {
1197 return Ok(Vec::new());
1198 }
1199 let drained = self
1200 .forwarding
1201 .publish_scope_changes(changes)
1202 .map_err(RouterError::Forwarding)?;
1203 let mut counts = HashMap::new();
1204 for route in &drained {
1205 *counts
1206 .entry((
1207 route.scope.owner.as_str(),
1208 route.scope.scope_ref.as_str(),
1209 route.scope.tag.scope_epoch,
1210 ))
1211 .or_insert(0usize) += 1;
1212 }
1213 for scope in expired {
1214 let routes_closed = counts
1215 .get(&(
1216 scope.owner.as_str(),
1217 scope.scope_ref.as_str(),
1218 scope.scope_epoch,
1219 ))
1220 .copied()
1221 .unwrap_or(0);
1222 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1223 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1224 routes_closed, "scope expired");
1225 }
1226 Ok(drained)
1227 }
1228
1229 fn runtime_capability_snapshot(
1230 &self,
1231 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1232 let runtime = self
1233 .supervisor
1234 .list()
1235 .into_iter()
1236 .map(|module| {
1237 let status = module.status().map_err(|err| {
1238 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1239 })?;
1240 Ok(RuntimeModule {
1241 module_id: status.module_id,
1242 state: status.state,
1243 enabled: status.enabled,
1244 })
1245 })
1246 .collect::<Result<Vec<_>, RouterError>>()?;
1247 let registrations = self.registry.list_registered_modules().map_err(|err| {
1253 RouterError::backend(
1254 0,
1255 0,
1256 format!("failed to list capability registrations: {err}"),
1257 )
1258 })?;
1259 let registrations = registrations
1260 .into_iter()
1261 .map(|registration| RegisteredModule {
1262 module_id: registration.manifest.module_id,
1263 module_version: registration.manifest.module_version,
1264 capabilities: registration.manifest.capabilities,
1265 })
1266 .collect();
1267 Ok((runtime, registrations))
1268 }
1269
1270 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1277 let cached_registration = RegisteredModule {
1278 module_id: registration.manifest.module_id.clone(),
1279 module_version: registration.manifest.module_version.clone(),
1280 capabilities: registration.manifest.capabilities.clone(),
1281 };
1282 if self.capability_evaluator.record_hello(&cached_registration) {
1283 warn!(
1284 module_id = %cached_registration.module_id,
1285 "capability claims drifted from the cached manifest"
1286 );
1287 }
1288 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1289 self.enforce_capability_denies();
1290 }
1291 self.refresh_capability_requirements();
1292 }
1293
1294 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1298 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1299 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1300 self.supervisor.set_swap_promotion_observer(observer);
1301 }
1302
1303 pub fn refresh_capability_requirements(&self) {
1304 match self.runtime_capability_snapshot() {
1305 Ok((runtime, registrations)) => {
1306 log_requirement_events(
1307 self.capability_evaluator
1308 .evaluate_now(&runtime, ®istrations),
1309 );
1310 }
1311 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1312 }
1313 }
1314
1315 fn enforce_capability_denies(&self) {
1319 let registrations = match self.registry.list_registered_modules() {
1320 Ok(snapshot) => snapshot,
1321 Err(err) => {
1322 warn!(error = %err, "failed to read registrations for capability deny census");
1323 return;
1324 }
1325 };
1326 let manifests = registrations
1327 .into_iter()
1328 .map(|registration| {
1329 (
1330 registration.manifest.module_id.clone(),
1331 registration.manifest,
1332 )
1333 })
1334 .collect::<BTreeMap<_, _>>();
1335 let census = match self.forwarding.route_census(None) {
1336 Ok(census) => census,
1337 Err(err) => {
1338 warn!(error = %err, "failed to read route census for capability deny enforcement");
1339 return;
1340 }
1341 };
1342
1343 for (target_module_id, routes) in census {
1344 let Some(target_manifest) = manifests.get(&target_module_id) else {
1345 continue;
1346 };
1347 let mut closed_routes = Vec::new();
1348 let mut module_goodbyes = Vec::new();
1349 for route in routes {
1350 let Principal::Reserved {
1351 module_id: opening_module_id,
1352 } = &route.principal
1353 else {
1354 continue;
1355 };
1356 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1357 continue;
1358 };
1359 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1360 continue;
1361 };
1362
1363 match self.forwarding.release_client_route(
1364 route.goodbye_target.connection_id,
1365 route.goodbye_target.channel,
1366 route.goodbye_target.epoch,
1367 ) {
1368 Ok(RouteRelease::Removed(module_goodbye)) => {
1369 warn!(
1370 opening_module_id,
1371 target_module_id,
1372 capability,
1373 "force-closing route because an attested capability deny edge now matches"
1374 );
1375 closed_routes.push(route);
1376 module_goodbyes.push(module_goodbye);
1377 }
1378 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1379 Err(err) => warn!(
1380 opening_module_id,
1381 target_module_id,
1382 capability,
1383 error = %err,
1384 "failed to force-close capability-denied route"
1385 ),
1386 }
1387 }
1388
1389 if closed_routes.is_empty() {
1390 continue;
1391 }
1392 send_route_control_pushes(
1393 &self.forwarding,
1394 closed_routes,
1395 ClientControlPush::RouteClosed {
1396 module_id: target_module_id,
1397 channels: Vec::new(),
1398 reason: RouteCloseReason::CapabilityDenied,
1399 drained: false,
1400 abandoned: 0,
1401 excluded_subscriptions: 0,
1402 terminal: Some(false),
1403 },
1404 );
1405 self.emit_route_goodbyes(module_goodbyes);
1406 }
1407 }
1408
1409 fn not_ready_reason(
1415 &self,
1416 registration: &crate::registry::ModuleRegistration,
1417 ) -> Option<NotReadyReason> {
1418 if !registration.ready {
1419 return Some(NotReadyReason {
1420 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1421 capability: None,
1422 });
1423 }
1424 self.first_unprovided_required_capability(registration)
1425 .map(|capability| NotReadyReason {
1426 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1427 capability: Some(capability),
1428 })
1429 }
1430
1431 fn first_unprovided_required_capability(
1449 &self,
1450 registration: &crate::registry::ModuleRegistration,
1451 ) -> Option<String> {
1452 let required = registration
1453 .manifest
1454 .capabilities
1455 .iter()
1456 .flat_map(|declarations| declarations.requires.iter())
1457 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1458 .map(|requirement| requirement.capability.as_str())
1459 .collect::<BTreeSet<_>>();
1460 if required.is_empty() {
1461 return None;
1462 }
1463 let module_id = registration.manifest.module_id.as_str();
1464 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1465 if required
1466 .iter()
1467 .any(|capability| verdict(capability).is_none())
1468 {
1469 self.refresh_capability_requirements();
1470 }
1471 required
1472 .into_iter()
1473 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1474 .map(str::to_string)
1475 }
1476
1477 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1478 self.capability_evaluator
1479 .statuses()
1480 .into_iter()
1481 .map(capability_requirement_status)
1482 .collect()
1483 }
1484
1485 fn deregister_connection(
1492 &self,
1493 connection_id: ConnectionId,
1494 reason: RegistrationEndReason,
1495 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1496 self.registry
1497 .deregister_connection_with_reason(connection_id, reason)
1498 }
1499
1500 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1501 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1502 return None;
1503 }
1504 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1505 parse_client_control_request(&frame.body)
1506 else {
1507 return None;
1508 };
1509 Some(target_module_id(&target).to_string())
1510 }
1511
1512 pub(crate) fn route_open_capacity_refusal(
1513 &self,
1514 ctx: &RouteCtx,
1515 frame: &Frame,
1516 target_module_id: &str,
1517 in_flight: usize,
1518 limit: usize,
1519 ) -> Result<Frame, RouterError> {
1520 self.route_open_admission_refusal_frame(
1521 ctx,
1522 frame,
1523 target_module_id,
1524 "open_admission_full",
1525 (in_flight, limit),
1526 format!(
1527 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1528 ),
1529 )
1530 }
1531
1532 fn route_open_target_capacity_refusal(
1533 &self,
1534 ctx: &RouteCtx,
1535 frame: &Frame,
1536 target_module_id: &str,
1537 in_flight: usize,
1538 ) -> Result<Frame, RouterError> {
1539 self.route_open_admission_refusal_frame(
1540 ctx,
1541 frame,
1542 target_module_id,
1543 "target_binds_full",
1544 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1545 format!(
1546 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1547 ),
1548 )
1549 }
1550
1551 fn route_open_admission_refusal_frame(
1559 &self,
1560 ctx: &RouteCtx,
1561 frame: &Frame,
1562 target_module_id: &str,
1563 reason: &'static str,
1564 (in_flight, limit): (usize, usize),
1565 message: impl Into<String>,
1566 ) -> Result<Frame, RouterError> {
1567 let code = error_codes::TARGET_UNAVAILABLE;
1568 self.counters.increment_route_open_refused(code);
1569 info!(
1570 target: "control",
1571 code,
1572 reason,
1573 module_id = ?target_module_id,
1574 connection_id = ctx.connection_id.get(),
1575 in_flight,
1576 limit,
1577 "route.open refused"
1578 );
1579 control_error_frame(frame, code, message.into())
1580 }
1581
1582 #[cfg(test)]
1588 pub fn handle_control(
1589 &self,
1590 connection_id: ConnectionId,
1591 frame: Frame,
1592 ) -> Result<Vec<Frame>, RouterError> {
1593 match frame.header.ty {
1594 FrameType::Ping => Ok(vec![pong(&frame)?]),
1595 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1596 FrameType::Goodbye => self.handle_goodbye(connection_id),
1597 ty => Ok(vec![control_error_frame(
1598 &frame,
1599 "unsupported_control_frame",
1600 format!("unsupported channel-0 frame {ty:?}"),
1601 )?]),
1602 }
1603 }
1604
1605 pub async fn handle_control_frame(
1606 &self,
1607 ctx: &RouteCtx,
1608 frame: Frame,
1609 ) -> Result<Vec<Frame>, RouterError> {
1610 self.handle_control_frame_timed(ctx, frame, None).await
1611 }
1612
1613 pub(crate) fn record_caller_endpoints(
1614 &self,
1615 connection: ConnectionId,
1616 endpoints: Option<(SocketAddr, SocketAddr)>,
1617 ) {
1618 let mut callers = self
1619 .caller_endpoints
1620 .lock()
1621 .unwrap_or_else(|p| p.into_inner());
1622 callers.remove(&connection);
1623 if let Some(endpoints) = endpoints {
1624 callers.insert(connection, endpoints);
1625 }
1626 }
1627
1628 #[cfg(test)]
1629 pub(crate) fn with_caller_lookup_for_test(
1630 mut self,
1631 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1632 ) -> Self {
1633 self.caller_lookup = Some(Arc::new(lookup));
1634 self
1635 }
1636
1637 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1638 let endpoints = self
1639 .caller_endpoints
1640 .lock()
1641 .unwrap_or_else(|p| p.into_inner())
1642 .get(&connection)
1643 .copied();
1644 #[cfg(test)]
1645 let lookup = self.caller_lookup.clone();
1646 tokio::task::spawn_blocking(move || {
1647 #[cfg(test)]
1648 if let Some(lookup) = lookup {
1649 return lookup(endpoints);
1650 }
1651 endpoints
1652 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1653 .unwrap_or_default()
1654 })
1655 .await
1656 .unwrap_or_default()
1657 }
1658
1659 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1660 frame.header.channel == 0
1661 && frame.header.ty == FrameType::Request
1662 && self
1663 .forwarding
1664 .module_endpoint_for_connection(ctx.connection_id)
1665 .is_ok_and(|endpoint| endpoint.is_none())
1666 && parse_client_control_request(&frame.body)
1667 .ok()
1668 .is_some_and(|request| privileged_operation(&request).is_some())
1669 }
1670
1671 pub(crate) async fn handle_control_frame_timed(
1672 &self,
1673 ctx: &RouteCtx,
1674 frame: Frame,
1675 dispatch_started_at: Option<StdInstant>,
1676 ) -> Result<Vec<Frame>, RouterError> {
1677 match frame.header.ty {
1678 FrameType::Ping => Ok(vec![pong(&frame)?]),
1679 FrameType::Hello => {
1680 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1681 }
1682 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1683 FrameType::Cancel => {
1684 if self
1687 .forwarding
1688 .operator_confirms()
1689 .cancel(ctx.connection_id, frame.header.corr)
1690 || self
1691 .supervisor
1692 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1693 || self.cancel_activity_subscription(ctx, &frame)?
1694 {
1695 Ok(Vec::new())
1696 } else {
1697 Ok(vec![control_error_frame(
1698 &frame,
1699 "unknown_subscription",
1700 "no subscription has this correlation id",
1701 )?])
1702 }
1703 }
1704 FrameType::Request => {
1705 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1708 if op
1709 .as_ref()
1710 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1711 {
1712 return self.handle_activity_record(ctx, frame).await;
1713 }
1714 if op.as_ref().is_some_and(|probe| {
1715 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1716 }) {
1717 return self.handle_activity_read(ctx, frame).await;
1718 }
1719 if op
1720 .as_ref()
1721 .is_some_and(|probe| probe.op == "operator.confirm")
1722 {
1723 return self.handle_operator_confirm(ctx, frame);
1724 }
1725 if self
1726 .forwarding
1727 .module_endpoint_for_connection(ctx.connection_id)
1728 .map_err(RouterError::Forwarding)?
1729 .is_some()
1730 {
1731 if !is_known_module_request_op(&frame.body) {
1732 return Ok(vec![control_error_frame(
1733 &frame,
1734 "unsupported_control_frame",
1735 "module-originated channel-0 REQUEST is not supported",
1736 )?]);
1737 }
1738 let request = match parse_module_control_request_from_module(&frame.body) {
1739 Ok(request) => request,
1740 Err((err, ControlRequestBodyError::UnknownOp)) => {
1741 return Ok(vec![control_error_frame(
1742 &frame,
1743 "unsupported_control_frame",
1744 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1745 )?])
1746 }
1747 Err((err, ControlRequestBodyError::InvalidBody)) => {
1748 return Ok(vec![control_error_frame(
1749 &frame,
1750 "invalid_control_body",
1751 format!("malformed module control body: {err}"),
1752 )?])
1753 }
1754 };
1755 let op = module_control_request_op(&request);
1756 let corr = frame.header.corr;
1757 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1758 let result =
1759 self.handle_module_control_request(ctx.connection_id, frame, request);
1760 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1761 return result;
1762 }
1763
1764 if is_known_module_request_op(&frame.body) {
1765 return Ok(vec![control_error_frame(
1766 &frame,
1767 "not_registered",
1768 "catalog.update requires an active module registration owned by this connection",
1769 )?]);
1770 }
1771
1772 let request = match parse_client_control_request(&frame.body) {
1773 Ok(request) => request,
1774 Err((err, ControlRequestBodyError::UnknownOp)) => {
1775 return Ok(vec![control_error_frame(
1776 &frame,
1777 "unknown_control_op",
1778 format!("unknown client control op: {err}"),
1779 )?])
1780 }
1781 Err((err, ControlRequestBodyError::InvalidBody)) => {
1782 return Ok(vec![control_error_frame(
1783 &frame,
1784 "invalid_control_body",
1785 format!("malformed client control body: {err}"),
1786 )?])
1787 }
1788 };
1789 let op = client_control_request_op(&request);
1790 let corr = frame.header.corr;
1791 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1792 #[cfg(test)]
1793 if let Some(delay) = self.control_dispatch_delay {
1794 tokio::time::sleep(delay).await;
1795 }
1796 let result = self
1797 .handle_client_control_request(ctx, frame, request)
1798 .await;
1799 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1800 result
1801 }
1802 FrameType::Push => {
1803 let Some(endpoint) = self
1804 .forwarding
1805 .module_endpoint_for_connection(ctx.connection_id)
1806 .map_err(RouterError::Forwarding)?
1807 else {
1808 return Ok(vec![control_error_frame(
1809 &frame,
1810 "unsupported_control_frame",
1811 "client-originated channel-0 PUSH is not supported",
1812 )?]);
1813 };
1814 self.handle_status_update(endpoint, frame)
1815 }
1816 FrameType::Response | FrameType::Error
1817 if self
1818 .forwarding
1819 .module_endpoint_for_connection(ctx.connection_id)
1820 .map_err(RouterError::Forwarding)?
1821 .is_some() =>
1822 {
1823 self.handle_module_relay_response(ctx.connection_id, frame)
1824 }
1825 ty => Ok(vec![control_error_frame(
1826 &frame,
1827 "unsupported_control_frame",
1828 format!("unsupported channel-0 frame {ty:?}"),
1829 )?]),
1830 }
1831 }
1832
1833 pub fn cleanup_connection(
1834 &self,
1835 connection_id: ConnectionId,
1836 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1837 self.caller_endpoints
1838 .lock()
1839 .unwrap_or_else(|p| p.into_inner())
1840 .remove(&connection_id);
1841 self.forwarding
1842 .operator_confirms()
1843 .connection_closed(connection_id);
1844 self.cleanup_connection_with_end_reason(
1845 connection_id,
1846 RegistrationEndReason::ConnectionClosed,
1847 )
1848 }
1849
1850 fn cleanup_connection_with_end_reason(
1851 &self,
1852 connection_id: ConnectionId,
1853 requested_reason: RegistrationEndReason,
1854 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1855 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1856 self.registry
1857 .get_module_by_connection(connection_id)?
1858 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
1859 .and_then(|module| module.registration_end_reason().ok().flatten())
1860 .unwrap_or(requested_reason)
1861 } else {
1862 requested_reason
1863 };
1864 let crash_closed = self
1865 .registry
1866 .get_module_by_connection(connection_id)?
1867 .and_then(|registration| {
1868 self.forwarding
1869 .module_endpoint_for_connection(connection_id)
1870 .ok()
1871 .flatten()
1872 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1873 .map(|routes| (registration.manifest.module_id, routes))
1874 });
1875 let crash_closed = crash_closed.map(|(module_id, routes)| {
1876 let terminal = match self.supervisor.get(&module_id) {
1877 None => false,
1878 Some(module) => match module.will_recover_after_connection_loss() {
1879 Ok(will_recover) => !will_recover,
1880 Err(err) => {
1881 warn!(
1882 %module_id,
1883 error = %err,
1884 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1885 );
1886 false
1887 }
1888 },
1889 };
1890 let (reason, drained) = match self
1894 .forwarding
1895 .daemon_drain_result_for_connection(connection_id)
1896 {
1897 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
1898 Ok(None) => (RouteCloseReason::Crash, false),
1899 Err(err) => {
1900 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1901 (RouteCloseReason::Crash, false)
1902 }
1903 };
1904 (module_id, routes, reason, drained, terminal)
1905 });
1906 let registrations = self.deregister_connection(connection_id, end_reason);
1907 let cleanup = if crash_closed.is_some() {
1908 self.forwarding.cleanup_connection_counted(connection_id)
1909 } else {
1910 self.forwarding
1916 .cleanup_connection(connection_id)
1917 .map(|released| crate::forwarding::ConnectionCleanup {
1918 released,
1919 abandoned_relays: 0,
1920 })
1921 };
1922 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
1927 let abandoned = cleanup
1928 .as_ref()
1929 .map_or(0, |cleanup| cleanup.abandoned_relays);
1930 send_route_control_pushes(
1931 &self.forwarding,
1932 routes,
1933 ClientControlPush::RouteClosed {
1934 module_id,
1935 channels: Vec::new(),
1936 reason,
1937 drained,
1938 abandoned,
1939 excluded_subscriptions: 0,
1940 terminal: Some(terminal),
1941 },
1942 );
1943 }
1944 if let Ok(cleanup) = cleanup {
1945 self.emit_route_goodbyes(cleanup.released);
1946 }
1947 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1951 crate::supervise::notify_registration_release();
1952 self.capability_evaluator.wake_deadline_loop();
1953 self.refresh_capability_requirements();
1954 }
1955 self.supervisor.remove_spawn_subscribers(connection_id);
1956 self.activity_streams
1957 .lock()
1958 .unwrap_or_else(|p| p.into_inner())
1959 .retain(|(id, _), task| {
1960 if *id == connection_id {
1961 task.abort();
1962 false
1963 } else {
1964 true
1965 }
1966 });
1967 self.hello_launch_nonces
1970 .lock()
1971 .unwrap_or_else(|poisoned| poisoned.into_inner())
1972 .forget(connection_id);
1973 self.scopes
1974 .write()
1975 .unwrap_or_else(|poisoned| poisoned.into_inner())
1976 .release_connection(connection_id);
1977 registrations
1978 }
1979
1980 pub(crate) fn handle_route_goodbye(
1981 &self,
1982 connection_id: ConnectionId,
1983 route_channel: u16,
1984 route_epoch: u32,
1985 ) -> Result<bool, RouterError> {
1986 debug!(
1987 connection_id = connection_id.get(),
1988 route_channel, route_epoch, "handling route GOODBYE"
1989 );
1990 let RouteRelease::Removed(released_route) = self
1991 .forwarding
1992 .release_client_route(connection_id, route_channel, route_epoch)
1993 .map_err(RouterError::Forwarding)?
1994 else {
1995 return Ok(false);
1996 };
1997 self.emit_route_goodbyes(vec![released_route]);
1998 Ok(true)
1999 }
2000
2001 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2002 for released in released_routes {
2003 let frame = match Frame::build_with_version(
2004 released.negotiated_ver,
2005 FrameType::Goodbye,
2006 control_flags(),
2007 released.channel,
2008 released.epoch,
2009 0,
2010 Vec::new(),
2011 ) {
2012 Ok(frame) => frame,
2013 Err(err) => {
2014 warn!(
2015 route_channel = released.channel,
2016 error = %err,
2017 "failed to build route GOODBYE frame"
2018 );
2019 continue;
2020 }
2021 };
2022 if !released.close_on_delivery_failure() {
2023 crate::forwarding::send_module_route_goodbye(
2024 &self.counters,
2025 &released.sink,
2026 frame,
2027 released.module_id.as_deref(),
2028 "client route released",
2029 );
2030 continue;
2031 }
2032 if let Err(err) = released.sink.try_send(frame) {
2033 warn!(
2034 target_connection_id = released.connection_id.get(),
2035 route_channel = released.channel,
2036 error = %err,
2037 "route GOODBYE was not delivered to client; closing target connection"
2038 );
2039 if self
2040 .forwarding
2041 .escalate_client_delivery_failure(
2042 released.connection_id,
2043 released.channel,
2044 released.epoch,
2045 CloseReason::new(
2046 "route_goodbye_delivery_failed",
2047 format!(
2048 "failed to enqueue route GOODBYE for channel {}: {err}",
2049 released.channel
2050 ),
2051 ),
2052 crate::forwarding::UndeliveredFrame {
2053 module_id: released.module_id.as_deref(),
2054 sink: &released.sink,
2055 },
2056 )
2057 .unwrap_or(false)
2058 {
2059 self.counters.increment_goodbye_relay_client_failed();
2060 }
2061 }
2062 }
2063 }
2064
2065 fn send_abandoned_route_bind_goodbye(
2077 &self,
2078 module_sink: &crate::FrameSink,
2079 negotiated_ver: u8,
2080 module_channel: u16,
2081 module_epoch: u32,
2082 ) {
2083 let frame = match Frame::build_with_version(
2084 negotiated_ver,
2085 FrameType::Goodbye,
2086 control_flags(),
2087 module_channel,
2088 module_epoch,
2089 0,
2090 Vec::new(),
2091 ) {
2092 Ok(frame) => frame,
2093 Err(err) => {
2094 warn!(
2095 route_channel = module_channel,
2096 error = %err,
2097 "failed to build GOODBYE for abandoned route.bind"
2098 );
2099 return;
2100 }
2101 };
2102 crate::forwarding::send_module_route_goodbye(
2103 &self.counters,
2104 module_sink,
2105 frame,
2106 None,
2107 "abandoned route.bind",
2108 );
2109 }
2110
2111 fn handle_hello(
2112 &self,
2113 connection_id: ConnectionId,
2114 sink: Option<crate::FrameSink>,
2115 frame: Frame,
2116 ) -> Result<Vec<Frame>, RouterError> {
2117 debug!(
2118 connection_id = connection_id.get(),
2119 corr = frame.header.corr,
2120 "handling HELLO"
2121 );
2122 if self
2126 .registry
2127 .get_module_by_connection(connection_id)
2128 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2129 .is_some()
2130 {
2131 return Ok(vec![control_error_frame(
2132 &frame,
2133 "invalid_hello",
2134 "connection is already registered as a module",
2135 )?]);
2136 }
2137 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2138 Ok(value) => value,
2139 Err(err) => {
2140 return Ok(vec![control_error_frame(
2141 &frame,
2142 "invalid_hello",
2143 format!("malformed HELLO body: {err}"),
2144 )?])
2145 }
2146 };
2147 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2148 return Ok(vec![control_error_frame(
2149 &frame,
2150 "invalid_capability_grammar",
2151 err.to_string(),
2152 )?]);
2153 }
2154 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2155 return Ok(vec![control_error_frame(
2156 &frame,
2157 "invalid_manifest",
2158 err.to_string(),
2159 )?]);
2160 }
2161 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2162 return Ok(vec![control_error_frame(
2163 &frame,
2164 "invalid_event_declaration",
2165 err.to_string(),
2166 )?]);
2167 }
2168 if let Some(provenance) = hello_value
2169 .get("manifest")
2170 .and_then(|manifest| manifest.get("provenance"))
2171 {
2172 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2173 return Ok(vec![control_error_frame(
2174 &frame,
2175 "invalid_manifest",
2176 format!("malformed manifest provenance: {err}"),
2177 )?]);
2178 }
2179 }
2180 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2181 Ok(hello) => hello,
2182 Err(err) => {
2183 return Ok(vec![control_error_frame(
2184 &frame,
2185 "invalid_hello",
2186 format!("malformed HELLO body: {err}"),
2187 )?])
2188 }
2189 };
2190
2191 if hello.protocol_ver != hello.manifest.protocol_ver {
2192 return Ok(vec![control_error_frame(
2193 &frame,
2194 "invalid_manifest",
2195 format!(
2196 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2197 hello.protocol_ver, hello.manifest.protocol_ver
2198 ),
2199 )?]);
2200 }
2201
2202 if hello.manifest.module_id.trim().is_empty() {
2203 return Ok(vec![control_error_frame(
2204 &frame,
2205 "invalid_manifest",
2206 "manifest module_id must not be empty",
2207 )?]);
2208 }
2209
2210 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2211 Ok(negotiated_ver) => negotiated_ver,
2212 Err(message) => {
2213 return Ok(vec![control_error_frame(
2214 &frame,
2215 "version_unsupported",
2216 message,
2217 )?])
2218 }
2219 };
2220
2221 let swap_admission = self
2229 .supervisor
2230 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2231 if swap_admission == SwapHelloAdmission::Refused {
2232 warn!(
2233 module_id = %hello.manifest.module_id,
2234 connection_id = connection_id.get(),
2235 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2236 );
2237 return Ok(vec![control_error_frame(
2238 &frame,
2239 "swap_token_invalid",
2240 format!(
2241 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2242 hello.manifest.module_id
2243 ),
2244 )?]);
2245 }
2246 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2247
2248 if let Some(rejection) = (!swap_candidate)
2256 .then(|| {
2257 self.supervisor.reserved_hello_rejection(
2258 &hello.manifest.module_id,
2259 hello.launch_nonce.as_deref(),
2260 )
2261 })
2262 .flatten()
2263 {
2264 let message = match rejection {
2265 ReservedHelloRejection::Exact { module_id } => format!(
2266 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2267 ),
2268 ReservedHelloRejection::Prefix {
2269 prefix,
2270 owner_module_id,
2271 } => format!(
2272 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2273 hello.manifest.module_id
2274 ),
2275 };
2276 return Ok(vec![control_error_frame(
2277 &frame,
2278 "reserved_module",
2279 message,
2280 )?]);
2281 }
2282
2283 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2284 &hello.manifest.module_id,
2285 hello.manifest.capabilities.as_ref(),
2286 );
2287 if let Some(refusal) = reserved_capability_refusals.first() {
2288 let capability = refusal.capability.clone();
2289 let bound_module = refusal.claimants[0].clone();
2290 log_duplicate_claim_events(reserved_capability_refusals);
2291 return Ok(vec![control_error_frame(
2292 &frame,
2293 "reserved_capability",
2294 format!(
2295 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2296 capability, bound_module, hello.manifest.module_id
2297 ),
2298 )?]);
2299 }
2300
2301 if self
2304 .forwarding
2305 .connection_has_client_routes(connection_id)
2306 .map_err(RouterError::Forwarding)?
2307 {
2308 return Ok(vec![control_error_frame(
2309 &frame,
2310 "invalid_hello",
2311 "connection has open client routes and cannot also register as a module",
2312 )?]);
2313 }
2314
2315 self.hello_launch_nonces
2320 .lock()
2321 .unwrap_or_else(|poisoned| poisoned.into_inner())
2322 .record(connection_id, hello.launch_nonce.as_deref());
2323 let control_ops = effective_module_control_ops(hello.control_ops);
2324 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2327 if swap_candidate {
2328 return self.register_swap_candidate(
2329 connection_id,
2330 sink,
2331 &frame,
2332 hello.manifest,
2333 negotiated_ver,
2334 control_ops,
2335 hello_ack,
2336 );
2337 }
2338 let registration = match self.registry.register_unpublished_with_control_ops(
2345 hello.manifest,
2346 negotiated_ver,
2347 connection_id,
2348 control_ops,
2349 ) {
2350 Ok(registration) => registration,
2351 Err(RegistryError::DuplicateModuleId { module_id }) => {
2352 return Ok(vec![control_error_frame(
2353 &frame,
2354 "duplicate_module_id",
2355 format!(
2356 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2357 ),
2358 )?])
2359 }
2360 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2361 return Ok(vec![control_error_frame(
2362 &frame,
2363 "invalid_module_id",
2364 err.to_string(),
2365 )?])
2366 }
2367 Err(err) => {
2368 return Ok(vec![control_error_frame(
2369 &frame,
2370 "registry_error",
2371 err.to_string(),
2372 )?])
2373 }
2374 };
2375 #[cfg(test)]
2376 self.hold_between_registry_and_forwarding();
2377
2378 let reply = if let Some(sink) = sink {
2379 let concurrency = manifest_concurrency(®istration.manifest);
2395 if let Err(err) = self.forwarding.register_module_connection_acked(
2396 connection_id,
2397 registration.manifest.module_id.clone(),
2398 negotiated_ver,
2399 concurrency,
2400 sink,
2401 hello_ack,
2402 ) {
2403 if matches!(
2408 self.deregister_connection(
2409 connection_id,
2410 RegistrationEndReason::RegistrationFailed,
2411 ),
2412 Ok(r) if !r.is_empty()
2413 ) {
2414 crate::supervise::notify_registration_release();
2415 }
2416 return Ok(vec![control_error_frame(
2417 &frame,
2418 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2419 "invalid_hello"
2420 } else {
2421 forwarding_error_code(&err)
2422 },
2423 err.to_string(),
2424 )?]);
2425 }
2426 Vec::new()
2427 } else {
2428 vec![hello_ack]
2431 };
2432
2433 if !self
2438 .registry
2439 .publish_registration(connection_id)
2440 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2441 {
2442 debug!(
2443 module_id = %registration.manifest.module_id,
2444 connection_id = connection_id.get(),
2445 "registration was removed before it could be published"
2446 );
2447 }
2448
2449 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2458 info!(
2459 module_id = %registration.manifest.module_id,
2460 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2461 );
2462 }
2463
2464 self.apply_registration_capabilities(®istration);
2465
2466 self.supervisor
2467 .record_activity_registration(®istration.manifest.module_id);
2468
2469 info!(
2470 module_id = %registration.manifest.module_id,
2471 module_version = %registration.manifest.module_version,
2472 negotiated_ver,
2473 routable_provider = manifest_provides_routable_role(®istration.manifest),
2474 connection_id = connection_id.get(),
2475 "module registered"
2476 );
2477
2478 Ok(reply)
2479 }
2480
2481 #[allow(clippy::too_many_arguments)]
2498 fn register_swap_candidate(
2499 &self,
2500 connection_id: ConnectionId,
2501 sink: Option<crate::FrameSink>,
2502 frame: &Frame,
2503 manifest: ModuleManifest,
2504 negotiated_ver: u8,
2505 control_ops: Vec<String>,
2506 hello_ack: Frame,
2507 ) -> Result<Vec<Frame>, RouterError> {
2508 let module_id = manifest.module_id.clone();
2509 let registration = match self.registry.register_unpublished_candidate_with_control_ops(
2510 manifest,
2511 negotiated_ver,
2512 connection_id,
2513 control_ops,
2514 ) {
2515 Ok(registration) => registration,
2516 Err(RegistryError::DuplicateModuleId { module_id }) => {
2517 return Ok(vec![control_error_frame(
2518 frame,
2519 "duplicate_module_id",
2520 format!(
2521 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2522 ),
2523 )?])
2524 }
2525 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2526 return Ok(vec![control_error_frame(
2527 frame,
2528 "invalid_module_id",
2529 err.to_string(),
2530 )?])
2531 }
2532 Err(err) => {
2533 return Ok(vec![control_error_frame(
2534 frame,
2535 "registry_error",
2536 err.to_string(),
2537 )?])
2538 }
2539 };
2540 #[cfg(test)]
2541 self.hold_between_registry_and_forwarding();
2542 let reply = if let Some(sink) = sink {
2543 let concurrency = manifest_concurrency(®istration.manifest);
2547 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2548 connection_id,
2549 module_id.clone(),
2550 negotiated_ver,
2551 concurrency,
2552 sink,
2553 hello_ack,
2554 ) {
2555 if matches!(
2556 self.deregister_connection(
2557 connection_id,
2558 RegistrationEndReason::RegistrationFailed,
2559 ),
2560 Ok(r) if !r.is_empty()
2561 ) {
2562 crate::supervise::notify_registration_release();
2563 }
2564 return Ok(vec![control_error_frame(
2565 frame,
2566 forwarding_error_code(&err),
2567 err.to_string(),
2568 )?]);
2569 }
2570 Vec::new()
2571 } else {
2572 vec![hello_ack]
2573 };
2574 if !self
2575 .registry
2576 .publish_registration(connection_id)
2577 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2578 {
2579 debug!(
2580 module_id = %module_id,
2581 connection_id = connection_id.get(),
2582 "swap candidate registration was removed before it could be published"
2583 );
2584 }
2585 self.supervisor.mark_swap_candidate_admitted(&module_id);
2586 info!(
2587 module_id = %module_id,
2588 module_version = %registration.manifest.module_version,
2589 negotiated_ver,
2590 ready = registration.ready,
2591 connection_id = connection_id.get(),
2592 "swap candidate registered; not routable until cutover"
2593 );
2594 Ok(reply)
2595 }
2596
2597 fn build_hello_ack(
2598 &self,
2599 frame: &Frame,
2600 negotiated_ver: u8,
2601 module_id: &str,
2602 ) -> Result<Frame, RouterError> {
2603 let ack = ModuleHelloAckBody {
2604 negotiated_ver,
2605 subc_ops: module_subc_ops(),
2606 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2607 storage: self
2608 .storage_config
2609 .as_ref()
2610 .map(|cfg| cfg.descriptor_for(module_id)),
2611 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2612 };
2613 let body = serde_json::to_vec(&ack).map_err(|err| {
2614 RouterError::backend(
2615 0,
2616 frame.header.corr,
2617 format!("failed to encode HELLO_ACK: {err}"),
2618 )
2619 })?;
2620
2621 Frame::build_with_version(
2622 negotiated_ver,
2623 FrameType::HelloAck,
2624 control_flags(),
2625 0,
2626 0,
2627 frame.header.corr,
2628 body,
2629 )
2630 .map_err(RouterError::FrameBuild)
2631 }
2632
2633 async fn handle_client_control_request(
2634 &self,
2635 ctx: &RouteCtx,
2636 frame: Frame,
2637 request: ClientControlRequest,
2638 ) -> Result<Vec<Frame>, RouterError> {
2639 if let Some(operation) = privileged_operation(&request) {
2640 let caller = self.resolve_caller(ctx.connection_id).await;
2641 let op = client_control_request_op(&request);
2642 let text = prompt_text(operation, &caller);
2643 match self.privileged_operator_prompt {
2644 PrivilegedOperatorPrompt::Log => {
2645 info!(target: "subc_daemon::privileged_prompt", operation = op,
2646 target = ?operation, caller = %caller.description(), summary = %text,
2647 "privileged request would prompt the operator");
2648 }
2649 PrivilegedOperatorPrompt::Enforce => {
2650 let key =
2651 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2652 let outcome = self
2653 .forwarding
2654 .operator_confirms()
2655 .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2656 .await;
2657 if outcome.code != "confirmed" {
2658 return Ok(vec![outcome.frame(&frame)]);
2659 }
2660 }
2661 }
2662 }
2663 match request {
2664 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2665 ClientControlRequest::CatalogList { module_id } => {
2666 self.handle_catalog_list(frame, module_id)
2667 }
2668 ClientControlRequest::RouteOpen {
2669 target,
2670 identity,
2671 consumer_identity,
2672 consumer_capabilities,
2673 role_versions,
2674 admission_facts,
2675 scope,
2676 } => {
2677 self.handle_route_open(
2678 ctx,
2679 frame,
2680 RouteOpenRequest {
2681 target,
2682 identity,
2683 consumer_identity,
2684 consumer_capabilities,
2685 role_versions,
2686 admission_facts,
2687 scope,
2688 },
2689 )
2690 .await
2691 }
2692 ClientControlRequest::RoutePoll {
2693 route_channel,
2694 route_epoch,
2695 kind,
2696 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2697 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2698 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2699 self.handle_supervisor_spawn_snapshot(frame)
2700 }
2701 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2702 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2703 }
2704 ClientControlRequest::SupervisorRestart {
2705 module_id,
2706 drain_timeout_ms,
2707 } => {
2708 self.log_supervisor_request_received(
2709 ctx,
2710 frame.header.corr,
2711 ops::SUPERVISOR_RESTART,
2712 Some(&module_id),
2713 None,
2714 )?;
2715 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2716 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2717 .await
2718 }
2719 ClientControlRequest::SupervisorSwap {
2720 module_id,
2721 ready_timeout_ms,
2722 } => {
2723 self.log_supervisor_request_received(
2724 ctx,
2725 frame.header.corr,
2726 ops::SUPERVISOR_SWAP,
2727 Some(&module_id),
2728 None,
2729 )?;
2730 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2731 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2732 .await
2733 }
2734 ClientControlRequest::SupervisorReload { module_id } => {
2735 self.log_supervisor_request_received(
2736 ctx,
2737 frame.header.corr,
2738 ops::SUPERVISOR_RELOAD,
2739 Some(&module_id),
2740 None,
2741 )?;
2742 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2743 self.handle_supervisor_reload(frame, module_id, requester)
2744 .await
2745 }
2746 ClientControlRequest::SupervisorRescan { preview } => {
2747 if !preview {
2748 self.log_supervisor_request_received(
2749 ctx,
2750 frame.header.corr,
2751 ops::SUPERVISOR_RESCAN,
2752 None,
2753 None,
2754 )?;
2755 }
2756 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2757 self.handle_supervisor_rescan(frame, preview, requester)
2758 .await
2759 }
2760 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2761 self.log_supervisor_request_received(
2762 ctx,
2763 frame.header.corr,
2764 ops::SUPERVISOR_RELEASE_RESERVED,
2765 Some(&module_id),
2766 None,
2767 )?;
2768 self.handle_supervisor_release_reserved(frame, module_id)
2769 .await
2770 }
2771 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2772 self.log_supervisor_request_received(
2773 ctx,
2774 frame.header.corr,
2775 ops::SUPERVISOR_SET_ENABLED,
2776 Some(&module_id),
2777 Some(enabled),
2778 )?;
2779 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2780 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2781 .await
2782 }
2783 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2784 self.handle_supervisor_health_probe(frame, module_id).await
2785 }
2786 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2787 ClientControlRequest::SupervisorRoutes { module_id } => {
2788 self.handle_supervisor_routes(frame, module_id)
2789 }
2790 ClientControlRequest::SupervisorProvenance { module_id } => {
2791 self.handle_supervisor_provenance(frame, module_id).await
2792 }
2793 ClientControlRequest::SupervisorStderrTail {
2794 module_id,
2795 max_lines,
2796 max_bytes,
2797 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2798 ClientControlRequest::SupervisorTerminals { module_id } => {
2799 self.handle_supervisor_terminals(frame, module_id).await
2800 }
2801 }
2802 }
2803
2804 fn log_supervisor_request_received(
2805 &self,
2806 ctx: &RouteCtx,
2807 corr: u64,
2808 op: &'static str,
2809 module_id: Option<&str>,
2810 enabled: Option<bool>,
2811 ) -> Result<(), RouterError> {
2812 let caller = self.supervisor_requester(ctx, corr)?;
2813 let caller = principal_label(&caller);
2814 info!(
2817 target: "control",
2818 op,
2819 module_id,
2820 enabled,
2821 connection_id = ctx.connection_id.get(),
2822 caller = %caller,
2823 "supervisor request received"
2824 );
2825 Ok(())
2826 }
2827
2828 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
2829 Ok(self
2830 .registry
2831 .get_module_by_connection(ctx.connection_id)
2832 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2833 .map(|registration| Principal::Reserved {
2834 module_id: registration.manifest.module_id,
2835 })
2836 .unwrap_or(Principal::Direct))
2837 }
2838
2839 fn handle_module_control_request(
2840 &self,
2841 connection_id: ConnectionId,
2842 frame: Frame,
2843 request: ModuleControlRequestFromModule,
2844 ) -> Result<Vec<Frame>, RouterError> {
2845 match request {
2846 ModuleControlRequestFromModule::CatalogUpdate {
2847 provides,
2848 capabilities,
2849 ready,
2850 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2851 ModuleControlRequestFromModule::LiveRoots {} => {
2852 let registered = self
2853 .registry
2854 .get_module_by_connection(connection_id)
2855 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2856 let Some(registration) = registered else {
2857 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2858 };
2859 let response = self
2860 .forwarding
2861 .live_roots(®istration.manifest.module_id)
2862 .map_err(RouterError::Forwarding)?;
2863 Ok(vec![control_response_body_frame(
2864 &frame,
2865 &response,
2866 "ModuleControlResponseToModule::LiveRoots",
2867 )?])
2868 }
2869 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2870 self.handle_scope_sync(connection_id, frame, generation, scopes)
2871 }
2872 ModuleControlRequestFromModule::ScopeApply {
2873 generation,
2874 upsert,
2875 end,
2876 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
2877 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2878 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2879 }
2880 }
2881 }
2882
2883 fn handle_operator_confirm(
2884 &self,
2885 ctx: &RouteCtx,
2886 frame: Frame,
2887 ) -> Result<Vec<Frame>, RouterError> {
2888 use crate::operator_confirm::{audit, Outcome};
2889 let registration = self
2890 .registry
2891 .get_module_by_connection(ctx.connection_id)
2892 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2893 let Some(registration) = registration else {
2894 let outcome = Outcome::refusal("not_registered");
2895 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2896 return Ok(vec![outcome.frame(&frame)]);
2897 };
2898 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
2899 Ok(request) => request,
2900 Err(_) => {
2901 let outcome = Outcome::refusal("invalid_control_body");
2902 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2903 return Ok(vec![outcome.frame(&frame)]);
2904 }
2905 };
2906 let module_id = registration.manifest.module_id;
2907 let nonce = self
2911 .hello_launch_nonces
2912 .lock()
2913 .unwrap_or_else(|p| p.into_inner())
2914 .nonce(ctx.connection_id)
2915 .map(str::to_owned);
2916 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
2917 self.supervisor
2918 .spawned_consumer_authorized(&module_id, nonce)
2919 });
2920 let confirms = self.forwarding.operator_confirms();
2921 self.forwarding
2922 .with_operator_route(
2923 ctx.connection_id,
2924 request.route_channel,
2925 request.route_epoch,
2926 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
2927 )
2928 .map_err(RouterError::Forwarding)
2929 }
2930
2931 fn handle_scope_sync(
2935 &self,
2936 connection_id: ConnectionId,
2937 frame: Frame,
2938 generation: u64,
2939 scopes: Vec<ScopeRecord>,
2940 ) -> Result<Vec<Frame>, RouterError> {
2941 self.handle_scope_change(connection_id, frame, generation, scopes, None)
2942 }
2943
2944 fn handle_scope_change(
2945 &self,
2946 connection_id: ConnectionId,
2947 frame: Frame,
2948 generation: u64,
2949 scopes: Vec<ScopeRecord>,
2950 end: Option<Vec<ScopeEnd>>,
2951 ) -> Result<Vec<Frame>, RouterError> {
2952 let is_apply = end.is_some();
2953 let Some(registration) = self
2954 .registry
2955 .get_module_by_connection(connection_id)
2956 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2957 else {
2958 return Ok(vec![control_error_frame(
2959 &frame,
2960 "not_registered",
2961 "scope.sync and scope.apply require an active module registration owned by this connection",
2962 )?]);
2963 };
2964 let owner = registration.manifest.module_id;
2965 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2966 let is_current_launch = |connection: ConnectionId| {
2967 self.hello_launch_nonces
2968 .lock()
2969 .unwrap_or_else(|poisoned| poisoned.into_inner())
2970 .presented(connection, current_nonce.as_deref())
2971 };
2972 let mut table = self
2977 .scopes
2978 .write()
2979 .unwrap_or_else(|poisoned| poisoned.into_inner());
2980 let outcome = match end {
2981 Some(end) => table.apply(
2982 &owner,
2983 connection_id,
2984 is_current_launch,
2985 generation,
2986 scopes,
2987 end,
2988 ),
2989 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
2990 };
2991 let drained = match &outcome {
2992 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
2993 Err(_) => Vec::new(),
2994 };
2995 let has_deadlines = outcome.is_ok() && table.has_deadlines();
2996 drop(table);
2997 if has_deadlines {
2998 self.scope_deadline_added.notify_one();
2999 }
3000 match outcome {
3001 Ok(applied) => {
3002 let counts = ScopeOutcomeCounts::of(&applied.results);
3003 info!(
3004 owner = %owner,
3005 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3006 generation,
3007 records = applied.results.len(),
3008 created = counts.created,
3009 replaced = counts.replaced,
3010 updated = counts.updated,
3011 unchanged = counts.unchanged,
3012 refused = counts.refused,
3013 ended = applied.ended.len(),
3014 tag_changes = applied.tag_changes.len(),
3015 routes_closed = drained.len(),
3016 "scope change accepted"
3017 );
3018 for refused in applied
3024 .results
3025 .iter()
3026 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3027 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3028 {
3029 warn!(
3030 owner = %owner,
3031 generation,
3032 scope_ref = %refused.scope_ref,
3033 scope_epoch = refused.scope_epoch,
3034 code = refused.code.as_deref().unwrap_or(""),
3035 "scope record refused"
3036 );
3037 }
3038 self.close_scope_drained_routes(drained);
3039 let response = if is_apply {
3040 ModuleControlResponseToModule::ScopeApply {
3041 generation,
3042 results: applied.results,
3043 end_results: applied.end_results,
3044 ended: applied.ended,
3045 }
3046 } else {
3047 ModuleControlResponseToModule::ScopeSync {
3048 generation,
3049 results: applied.results,
3050 ended: applied.ended,
3051 }
3052 };
3053 Ok(vec![control_response_body_frame(
3054 &frame,
3055 &response,
3056 "scope change response",
3057 )?])
3058 }
3059 Err(refusal) => {
3060 info!(
3061 owner = %owner,
3062 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3063 generation,
3064 code = refusal.code,
3065 "scope change refused"
3066 );
3067 Ok(vec![control_error_frame(
3068 &frame,
3069 refusal.code,
3070 refusal.message,
3071 )?])
3072 }
3073 }
3074 }
3075
3076 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3082 if drained.is_empty() {
3083 return;
3084 }
3085 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3086 BTreeMap::new();
3087 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3088 for route in drained {
3089 warn!(
3090 module_id = %route.module_id,
3091 reason = ?route.reason,
3092 client_connection_id = route.client.connection_id.get(),
3093 route_channel = route.client.channel,
3094 "closing route because its scope changed"
3095 );
3096 pushes
3097 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3098 .or_insert_with(|| (route.reason, Vec::new()))
3099 .1
3100 .push(EndpointRoute {
3101 goodbye_target: route.client.clone(),
3102 principal: Principal::Unverified,
3103 bound_at: Instant::now(),
3104 draining: false,
3105 drain_reason: None,
3106 });
3107 goodbyes.push(route.module);
3108 goodbyes.push(route.client);
3109 }
3110 for ((module_id, _), (reason, routes)) in pushes {
3111 send_route_control_pushes(
3112 &self.forwarding,
3113 routes,
3114 ClientControlPush::RouteClosed {
3115 module_id,
3116 channels: Vec::new(),
3117 reason,
3118 drained: false,
3119 abandoned: 0,
3120 excluded_subscriptions: 0,
3121 terminal: Some(false),
3122 },
3123 );
3124 }
3125 self.emit_route_goodbyes(goodbyes);
3126 }
3127
3128 fn handle_scope_describe(
3131 &self,
3132 connection_id: ConnectionId,
3133 frame: Frame,
3134 owner: Principal,
3135 scope_ref: String,
3136 ) -> Result<Vec<Frame>, RouterError> {
3137 let registered = self
3138 .registry
3139 .get_module_by_connection(connection_id)
3140 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3141 if registered.is_none() {
3142 return Ok(vec![control_error_frame(
3143 &frame,
3144 "not_registered",
3145 "scope.describe requires an active module registration owned by this connection",
3146 )?]);
3147 }
3148 let description = self
3149 .scopes
3150 .read()
3151 .unwrap_or_else(|poisoned| poisoned.into_inner())
3152 .describe(&owner, &scope_ref);
3153 let owner_configured = match &owner {
3154 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3159 _ => false,
3160 };
3161 let response = ModuleControlResponseToModule::ScopeDescribe {
3162 status: description.status,
3163 scope_epoch: description.scope_epoch,
3164 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3165 owner_synced: description.owner_synced,
3166 owner_configured,
3167 scope: description.stamp,
3168 };
3169 Ok(vec![control_response_body_frame(
3170 &frame,
3171 &response,
3172 "ModuleControlResponseToModule::ScopeDescribe",
3173 )?])
3174 }
3175
3176 fn handle_catalog_update(
3177 &self,
3178 connection_id: ConnectionId,
3179 frame: Frame,
3180 provides: Vec<ProviderRole>,
3181 capabilities: Option<CapabilityDeclarations>,
3182 ready: Option<bool>,
3183 ) -> Result<Vec<Frame>, RouterError> {
3184 self.refresh_capability_requirements();
3185 let Some(registration) = self
3186 .registry
3187 .get_module_by_connection(connection_id)
3188 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3189 else {
3190 return Ok(vec![control_error_frame(
3191 &frame,
3192 "not_registered",
3193 "catalog.update requires an active module registration owned by this connection",
3194 )?]);
3195 };
3196
3197 if let Some(message) =
3198 catalog_update_frozen_field_message(®istration.manifest, &provides)
3199 {
3200 return Ok(vec![control_error_frame(
3201 &frame,
3202 "catalog_update_frozen_field",
3203 message,
3204 )?]);
3205 }
3206
3207 let mut candidate = registration.manifest.clone();
3208 candidate.provides = provides.clone();
3209 candidate.capabilities = capabilities
3210 .clone()
3211 .or_else(|| registration.manifest.capabilities.clone());
3212 if let Err(err) = candidate.validate_capability_grammar() {
3213 return Ok(vec![control_error_frame(
3214 &frame,
3215 "invalid_capability_grammar",
3216 err.to_string(),
3217 )?]);
3218 }
3219
3220 let mut conflicts = self
3223 .capability_evaluator
3224 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3225 if let Some(conflict) = conflicts.first() {
3226 let message = format!(
3227 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3228 conflict.capability, conflict.claimants[0], candidate.module_id
3229 );
3230 for conflict in &mut conflicts {
3231 conflict.source = DuplicateClaimSource::CatalogUpdate;
3232 }
3233 log_duplicate_claim_events(conflicts);
3234 return Ok(vec![control_error_frame(
3235 &frame,
3236 "reserved_capability",
3237 message,
3238 )?]);
3239 }
3240
3241 let updated = self
3242 .registry
3243 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3244 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3245 if updated.is_none() {
3246 return Ok(vec![control_error_frame(
3247 &frame,
3248 "not_registered",
3249 "catalog.update requires an active module registration owned by this connection",
3250 )?]);
3251 }
3252 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3253 log_duplicate_claim_events(
3254 self.capability_evaluator
3255 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3256 );
3257 }
3258 if capability_census_trigger(
3259 registration.manifest.capabilities.as_ref(),
3260 updated
3261 .as_ref()
3262 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3263 ) {
3264 self.enforce_capability_denies();
3265 }
3266 self.refresh_capability_requirements();
3267
3268 let response = ModuleControlResponseToModule::CatalogUpdate {};
3269 control_response_body_frame(
3270 &frame,
3271 &response,
3272 "ModuleControlResponseToModule::CatalogUpdate",
3273 )
3274 .map(|frame| vec![frame])
3275 }
3276
3277 pub(crate) fn with_activity_ledger_config(
3278 mut self,
3279 run_dir: &Path,
3280 incarnation: String,
3281 config: crate::activity::ActivityConfig,
3282 ) -> Self {
3283 self.activity = Some(crate::activity::ActivityLedger::open_config(
3284 run_dir.join("activity.jsonl"),
3285 incarnation,
3286 config,
3287 ));
3288 self.supervisor.set_activity(self.activity.clone());
3289 self
3290 }
3291
3292 pub(crate) async fn configure_activity(
3293 &self,
3294 config: crate::activity::ActivityConfig,
3295 ) -> Result<(), &'static str> {
3296 match &self.activity {
3297 Some(ledger) => ledger.configure(config).await,
3298 None => Ok(()),
3299 }
3300 }
3301
3302 fn cancel_activity_subscription(
3303 &self,
3304 ctx: &RouteCtx,
3305 frame: &Frame,
3306 ) -> Result<bool, RouterError> {
3307 let task = self
3308 .activity_streams
3309 .lock()
3310 .unwrap_or_else(|p| p.into_inner())
3311 .remove(&(ctx.connection_id, frame.header.corr));
3312 let Some(task) = task else {
3313 return Ok(false);
3314 };
3315 task.abort();
3316 let end = Frame::build_with_version(
3317 response_version(frame),
3318 FrameType::StreamEnd,
3319 control_flags(),
3320 0,
3321 0,
3322 frame.header.corr,
3323 Vec::new(),
3324 )
3325 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3326 let sink = ctx.egress.clone();
3327 tokio::spawn(async move {
3328 let _ = sink.send(end).await;
3329 });
3330 Ok(true)
3331 }
3332
3333 async fn handle_activity_read(
3334 &self,
3335 ctx: &RouteCtx,
3336 frame: Frame,
3337 ) -> Result<Vec<Frame>, RouterError> {
3338 use subc_control::{
3339 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3340 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3341 };
3342 #[derive(Deserialize)]
3343 struct Identity {
3344 #[serde(default)]
3345 consumer_identity: Option<ConsumerIdentity>,
3346 }
3347 let principal = match self
3351 .forwarding
3352 .module_id_for_connection(ctx.connection_id)
3353 .map_err(RouterError::Forwarding)?
3354 {
3355 Some(module_id) => Ok(Principal::Reserved { module_id }),
3356 None => {
3357 let identity: Identity = match serde_json::from_slice(&frame.body) {
3358 Ok(identity) => identity,
3359 Err(_) => {
3360 return Ok(vec![control_error_frame(
3361 &frame,
3362 ACTIVITY_MALFORMED,
3363 "malformed activity read",
3364 )?])
3365 }
3366 };
3367 self.route_open_principal(&frame, identity.consumer_identity)?
3368 }
3369 };
3370 let admitted = matches!(principal, Ok(Principal::Direct))
3371 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3372 if !admitted {
3373 return Ok(vec![control_error_frame(
3374 &frame,
3375 ACTIVITY_READ_FORBIDDEN,
3376 "activity reads require an operator or trusted mirror principal",
3377 )?]);
3378 }
3379 let Some(ledger) = &self.activity else {
3380 return Ok(vec![control_error_frame(
3381 &frame,
3382 ACTIVITY_UNAVAILABLE,
3383 "activity ledger unavailable",
3384 )?]);
3385 };
3386 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3387 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3388 {
3389 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3390 Ok(request) => ledger.query(request).await,
3391 Err(_) => Err(ACTIVITY_MALFORMED),
3392 };
3393 return match result {
3394 Ok(reply) => Ok(vec![control_response_body_frame(
3395 &frame,
3396 &reply,
3397 ops::ACTIVITY_QUERY,
3398 )?]),
3399 Err(code) => Ok(vec![control_error_frame(
3400 &frame,
3401 code,
3402 "activity query refused",
3403 )?]),
3404 };
3405 }
3406 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3407 Ok(request) => request,
3408 Err(_) => {
3409 return Ok(vec![control_error_frame(
3410 &frame,
3411 ACTIVITY_MALFORMED,
3412 "malformed activity subscription",
3413 )?])
3414 }
3415 };
3416 let mut subscription = match ledger.subscribe(request.since).await {
3417 Ok(subscription) => subscription,
3418 Err(code) => {
3419 return Ok(vec![control_error_frame(
3420 &frame,
3421 code,
3422 "activity subscription refused",
3423 )?])
3424 }
3425 };
3426 let key = (ctx.connection_id, frame.header.corr);
3427 let sink = ctx.egress.clone();
3428 let mut streams = self
3429 .activity_streams
3430 .lock()
3431 .unwrap_or_else(|p| p.into_inner());
3432 streams.retain(|_, task| !task.is_finished());
3433 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3434 return Ok(vec![control_error_frame(
3435 &frame,
3436 "duplicate_subscription",
3437 "activity subscription correlation id already in use",
3438 )?]);
3439 }
3440 let task = tokio::spawn(async move {
3441 loop {
3442 let entry = if let Some(entry) = subscription.replay.pop_front() {
3443 Some(entry)
3444 } else {
3445 subscription.receiver.recv().await
3446 };
3447 let Some(entry) = entry else {
3448 break;
3449 };
3450 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3451 return;
3452 };
3453 let Ok(next) = Frame::build_with_version(
3454 response_version(&frame),
3455 FrameType::StreamData,
3456 control_flags(),
3457 0,
3458 0,
3459 frame.header.corr,
3460 body,
3461 ) else {
3462 return;
3463 };
3464 if sink.send(next).await.is_err() {
3465 return;
3466 }
3467 }
3468 if let Some(code) = subscription.terminal_code() {
3471 if let Ok(end) =
3472 control_error_frame(&frame, code, "activity subscriber fell behind")
3473 {
3474 let _ = sink.send(end).await;
3475 }
3476 }
3477 });
3478 streams.insert(key, task.abort_handle());
3479 Ok(Vec::new())
3480 }
3481
3482 async fn handle_activity_record(
3483 &self,
3484 ctx: &RouteCtx,
3485 frame: Frame,
3486 ) -> Result<Vec<Frame>, RouterError> {
3487 use subc_control::{
3488 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3489 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3490 };
3491 let result = async {
3492 let module = self
3493 .forwarding
3494 .module_id_for_connection(ctx.connection_id)
3495 .ok()
3496 .flatten()
3497 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3498 let request: ActivityRecordRequest =
3499 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3500 crate::activity::validate(&module, &request)?;
3501 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3502 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3503 let at_ms = std::time::SystemTime::now()
3504 .duration_since(std::time::UNIX_EPOCH)
3505 .unwrap_or_default()
3506 .as_millis() as u64;
3507 ledger.record(module, request, actor, at_ms).await
3508 }
3509 .await;
3510 match result {
3511 Ok(entry) => Ok(vec![control_response_body_frame(
3512 &frame,
3513 &ActivityRecordResponse { seq: entry.seq },
3514 "activity.record",
3515 )?]),
3516 Err(code) => Ok(vec![control_error_frame(
3517 &frame,
3518 code,
3519 "activity recording refused",
3520 )?]),
3521 }
3522 }
3523
3524 fn activity_actor(
3525 &self,
3526 connection: ConnectionId,
3527 module: &str,
3528 on: Option<&subc_control::ActivityOn>,
3529 ) -> Result<subc_control::ActivityActor, &'static str> {
3530 use crate::forwarding::{DataRoute, DataRouteState};
3531 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3532 let principal = |id: &str| Principal::Reserved {
3533 module_id: id.to_owned(),
3534 };
3535 let mut actor = ActivityActor {
3536 principal: principal(module),
3537 scope_owner: None,
3538 agent_id: None,
3539 run_id: None,
3540 flow_id: None,
3541 project_root: None,
3542 };
3543 let scope = match on {
3544 None => return Ok(actor),
3545 Some(ActivityOn::Scope(scope)) => {
3546 let selector = subc_protocol::scope::ScopeSelector {
3547 owner: principal(module),
3548 scope_ref: scope.scope_ref.clone(),
3549 scope_epoch: Some(scope.epoch),
3550 };
3551 self.scopes
3552 .read()
3553 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3554 .admit(&principal(module), module, &selector, true)
3555 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3556 }
3557 Some(ActivityOn::Route(route)) => {
3558 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3559 .forwarding
3560 .lookup_data_route(connection, route.channel, route.epoch)
3561 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3562 else {
3563 return Err(ACTIVITY_ACTOR_UNKNOWN);
3564 };
3565 if binding.module_id != module {
3566 return Err(ACTIVITY_ACTOR_UNKNOWN);
3567 }
3568 actor.principal = binding.principal.clone();
3569 actor.project_root = binding
3570 .project_root
3571 .as_ref()
3572 .map(|root| root.as_path().to_path_buf());
3573 let Some(bound) = &binding.scope else {
3574 return Ok(actor);
3575 };
3576 let owner = principal(&bound.owner);
3577 let selector = subc_protocol::scope::ScopeSelector {
3578 owner: owner.clone(),
3579 scope_ref: bound.scope_ref.clone(),
3580 scope_epoch: Some(bound.tag.scope_epoch),
3581 };
3582 let admitted = self
3583 .scopes
3584 .read()
3585 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3586 .admit(&owner, module, &selector, true)
3587 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3588 if admitted.tag != bound.tag {
3589 return Err(ACTIVITY_ACTOR_UNKNOWN);
3590 }
3591 admitted
3592 }
3593 };
3594 actor.scope_owner = Some(scope.stamp.owner);
3595 actor.agent_id = scope.stamp.attributes.agent_id;
3596 actor.run_id = scope.stamp.attributes.run_id;
3597 actor.flow_id = scope.stamp.attributes.flow_id;
3598 Ok(actor)
3599 }
3600
3601 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3602 self.refresh_capability_requirements();
3603 let mut counters = self.counters.snapshot();
3609 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3610 obj.insert(
3611 "activity_unparseable_lines".into(),
3612 ledger.read_errors().into(),
3613 );
3614 obj.insert(
3615 "activity_lifecycle_append_failures".into(),
3616 ledger.lifecycle_failures().into(),
3617 );
3618 obj.insert(
3619 "activity_started_digest_skips".into(),
3620 ledger.lifecycle_digest_skips().into(),
3621 );
3622 }
3623 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3624 self.forwarding.client_route_concentration(),
3625 counters.as_object_mut(),
3626 ) {
3627 obj.insert(
3628 "client_connections_with_routes".into(),
3629 connections_with_routes.into(),
3630 );
3631 obj.insert("max_routes_on_one_connection".into(), max.into());
3632 }
3633 if let (Some(open_breakers), Some(obj)) = (
3639 self.route_bind_breakers.open_snapshot(),
3640 counters.as_object_mut(),
3641 ) {
3642 obj.insert("route_bind_breakers_open".into(), open_breakers);
3643 }
3644 let response = ClientControlResponse::ServerDescribe {
3645 protocol_ver: PROTOCOL_VERSION,
3646 subc_ops: subc_ops(),
3647 capabilities: self.subc_capabilities.as_ref().to_vec(),
3648 connected_clients: self.connected_clients.count(),
3649 counters: Some(counters),
3650 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3651 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3652 capability_requirements: self.capability_requirement_statuses(),
3653 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3654 };
3655 Ok(vec![control_response_body_frame(
3656 &frame,
3657 &response,
3658 "ClientControlResponse::ServerDescribe",
3659 )?])
3660 }
3661
3662 fn handle_catalog_list(
3663 &self,
3664 frame: Frame,
3665 module_id: Option<String>,
3666 ) -> Result<Vec<Frame>, RouterError> {
3667 let (generation, modules) = self.registry.list_modules().map_err(|err| {
3668 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3669 })?;
3670 let entries = modules
3671 .into_iter()
3672 .filter(|registration| {
3673 module_id
3674 .as_deref()
3675 .map(|wanted| registration.manifest.module_id == wanted)
3676 .unwrap_or(true)
3677 })
3678 .map(|registration| {
3679 let not_ready = self.not_ready_reason(®istration);
3680 let roles = registration.manifest.provides;
3681 CatalogEntry::new(
3682 registration.manifest.module_id,
3683 roles,
3684 registration.control_ops,
3685 )
3686 .with_ready(not_ready.is_none())
3687 .with_not_ready(not_ready)
3688 .with_module_version(Some(registration.manifest.module_version))
3689 .with_capabilities(registration.manifest.capabilities)
3690 .with_self_signals(registration.manifest.self_signals)
3691 .with_events(registration.manifest.events)
3692 })
3693 .collect();
3694 let response = ClientControlResponse::CatalogList {
3695 generation,
3696 modules: entries,
3697 subc_ops: subc_ops(),
3698 };
3699 Ok(vec![control_response_body_frame(
3700 &frame,
3701 &response,
3702 "ClientControlResponse::CatalogList",
3703 )?])
3704 }
3705
3706 fn route_open_principal(
3707 &self,
3708 frame: &Frame,
3709 consumer_identity: Option<ConsumerIdentity>,
3710 ) -> Result<Result<Principal, Frame>, RouterError> {
3711 let Some(consumer_identity) = consumer_identity else {
3712 return Ok(Ok(Principal::Direct));
3713 };
3714
3715 if self.supervisor.spawned_consumer_authorized(
3716 &consumer_identity.module_id,
3717 &consumer_identity.launch_nonce,
3718 ) {
3719 return Ok(Ok(Principal::Reserved {
3720 module_id: consumer_identity.module_id,
3721 }));
3722 }
3723
3724 Ok(Err(control_error_frame(
3725 frame,
3726 "bad_consumer_identity",
3727 format!(
3728 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3729 consumer_identity.module_id
3730 ),
3731 )?))
3732 }
3733
3734 fn route_open_refusal_frame(
3741 &self,
3742 ctx: &RouteCtx,
3743 frame: &Frame,
3744 module_id: &str,
3745 reason: &'static str,
3746 code: &'static str,
3747 message: impl Into<String>,
3748 ) -> Result<Frame, RouterError> {
3749 self.observe_route_open_refusal(ctx, module_id, reason, code);
3750 control_error_frame(frame, code, message.into())
3751 }
3752
3753 fn route_open_breaker_refusal_frame(
3771 &self,
3772 ctx: &RouteCtx,
3773 frame: &Frame,
3774 module_id: &str,
3775 consecutive_timeouts: u32,
3776 retry_in: Duration,
3777 probe_in_flight: bool,
3778 ) -> Result<Frame, RouterError> {
3779 self.counters
3780 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
3781 debug!(
3782 target: "control",
3783 code = "module_timeout",
3784 module_id = ?module_id,
3785 connection_id = ctx.connection_id.get(),
3786 consecutive_timeouts,
3787 retry_in_ms = retry_in.as_millis() as u64,
3788 probe_in_flight,
3789 "route.open refused by open bind-relay breaker"
3790 );
3791 let detail = if probe_in_flight {
3798 "one test route is already being tried; retry once it settles".to_string()
3799 } else {
3800 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
3801 };
3802 control_error_frame(
3803 frame,
3804 "module_timeout",
3805 format!(
3806 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
3807 timed out in a row); {detail}; its established routes are unaffected"
3808 ),
3809 )
3810 }
3811
3812 fn observe_route_open_refusal(
3825 &self,
3826 ctx: &RouteCtx,
3827 module_id: &str,
3828 reason: &'static str,
3829 code: &'static str,
3830 ) {
3831 self.counters.increment_route_open_refused(code);
3832 info!(
3833 target: "control",
3834 code,
3835 reason,
3836 module_id = ?module_id,
3837 connection_id = ctx.connection_id.get(),
3838 "route.open refused"
3839 );
3840 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
3841 self.route_outages.record_not_serving(module_id, reason);
3842 }
3843 }
3844
3845 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
3890 self.route_outages.record_accepted(module_id);
3891 self.counters.increment_route_open_accepted(principal);
3892 info!(
3893 target: "control",
3894 principal,
3895 module_id,
3896 connection_id = ctx.connection_id.get(),
3897 "route.open accepted"
3898 );
3899 }
3900
3901 fn supervised_absent_route_open_refusal_frame(
3902 &self,
3903 ctx: &RouteCtx,
3904 frame: &Frame,
3905 module_id: &str,
3906 code: &'static str,
3907 status: &crate::supervise::ModuleStatus,
3908 ) -> Result<Frame, RouterError> {
3909 self.counters.increment_route_open_refused(code);
3910 info!(
3911 target: "control",
3912 code,
3913 reason = "supervised_not_registered",
3914 module_id = ?module_id,
3915 connection_id = ctx.connection_id.get(),
3916 state = %status.state,
3917 enabled = status.enabled,
3918 live = status.live,
3919 "route.open refused"
3920 );
3921 self.route_outages
3925 .record_not_serving(module_id, "supervised_not_registered");
3926 let message = if let Some(pid) = status.waiting_for_previous_pid {
3927 format!("waiting: previous instance (pid {pid}) is still running but hidden; it cannot be identified")
3928 } else {
3929 format!(
3930 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
3931 status.state, status.enabled, status.live
3932 )
3933 };
3934 control_error_frame(frame, code, message)
3935 }
3936
3937 async fn handle_route_open(
3938 &self,
3939 ctx: &RouteCtx,
3940 frame: Frame,
3941 request: RouteOpenRequest,
3942 ) -> Result<Vec<Frame>, RouterError> {
3943 let RouteOpenRequest {
3944 target,
3945 mut identity,
3946 consumer_identity,
3947 consumer_capabilities,
3948 role_versions,
3949 admission_facts,
3950 scope,
3951 } = request;
3952 let target_module_id = target_module_id(&target).to_string();
3953 if self
3954 .registry
3955 .get_module_by_connection(ctx.connection_id)
3956 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3957 .is_some()
3958 {
3959 return Ok(vec![control_error_frame(
3960 &frame,
3961 "invalid_request",
3962 "module connections cannot open client routes",
3963 )?]);
3964 }
3965 debug!(
3966 connection_id = ctx.connection_id.get(),
3967 corr = frame.header.corr,
3968 module_id = %target_module_id,
3969 "handling route.open"
3970 );
3971
3972 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
3978 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
3979 self.observe_route_open_refusal(
3980 ctx,
3981 &target_module_id,
3982 "invalid_role_versions",
3983 error_codes::INVALID_REQUEST,
3984 );
3985 return Ok(vec![control_error_body_frame(
3986 &frame,
3987 ErrorBody {
3988 code: error_codes::INVALID_REQUEST.to_string(),
3989 message: error.to_string(),
3990 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
3991 },
3992 )?]);
3993 }
3994
3995 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
3996 if status.waiting_for_previous_pid.is_some() {
3997 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3998 ctx,
3999 &frame,
4000 &target_module_id,
4001 "module_warming",
4002 &status,
4003 )?]);
4004 }
4005 }
4006
4007 let Some(registration) = self
4026 .registry
4027 .get_module(&target_module_id)
4028 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4029 else {
4030 if let Some((status, warming)) =
4031 self.supervisor_status(&target_module_id, frame.header.corr)?
4032 {
4033 if status.protocol == ModuleProtocol::None {
4040 return Ok(vec![self.route_open_refusal_frame(
4041 ctx,
4042 &frame,
4043 &target_module_id,
4044 "protocol_none",
4045 error_codes::MODULE_NO_PROTOCOL,
4046 format!(
4047 "module_id '{target_module_id}' is declared protocol: none; \
4048 it speaks no subc wire and serves no routes"
4049 ),
4050 )?]);
4051 }
4052 let code = if warming {
4053 "module_warming"
4054 } else {
4055 "target_unavailable"
4056 };
4057 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4058 ctx,
4059 &frame,
4060 &target_module_id,
4061 code,
4062 &status,
4063 )?]);
4064 }
4065 if let Some(removed_ago_ms) =
4066 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4067 {
4068 return Ok(vec![self.route_open_refusal_frame(
4069 ctx,
4070 &frame,
4071 &target_module_id,
4072 "removed",
4073 error_codes::MODULE_REMOVED,
4074 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4075 )?]);
4076 }
4077 return Ok(vec![self.route_open_refusal_frame(
4078 ctx,
4079 &frame,
4080 &target_module_id,
4081 "not_registered",
4082 error_codes::UNKNOWN_MODULE,
4083 format!("module_id '{target_module_id}' is not registered"),
4084 )?]);
4085 };
4086
4087 if !registration.ready {
4091 self.counters
4092 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4093 info!(
4094 target: "control",
4095 code = error_codes::MODULE_WARMING,
4096 module_id = ?target_module_id,
4097 connection_id = ctx.connection_id.get(),
4098 reason = "declared_not_ready",
4099 "route.open refused"
4100 );
4101 self.route_outages
4104 .record_not_serving(&target_module_id, "declared_not_ready");
4105 return Ok(vec![control_error_body_frame(
4106 &frame,
4107 ErrorBody {
4108 code: error_codes::MODULE_WARMING.to_string(),
4109 message: format!(
4110 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4111 ),
4112 detail: Some(serde_json::json!({
4113 "reason": "declared_not_ready"
4114 })),
4115 },
4116 )?]);
4117 }
4118
4119 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4138 self.counters
4139 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4140 info!(
4141 target: "control",
4142 code = error_codes::MODULE_WARMING,
4143 module_id = ?target_module_id,
4144 connection_id = ctx.connection_id.get(),
4145 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4146 capability = %capability,
4147 "route.open refused"
4148 );
4149 return Ok(vec![control_error_body_frame(
4150 &frame,
4151 ErrorBody {
4152 code: error_codes::MODULE_WARMING.to_string(),
4153 message: format!(
4154 "module_id '{target_module_id}' requires capability '{capability}', \
4155 which no registered module provides; retry"
4156 ),
4157 detail: Some(serde_json::json!({
4158 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4159 "capability": capability,
4160 })),
4161 },
4162 )?]);
4163 }
4164
4165 if !target_has_required_role(&target, ®istration.manifest.provides) {
4166 return Ok(vec![self.route_open_refusal_frame(
4167 ctx,
4168 &frame,
4169 &target_module_id,
4170 "role_not_provided",
4171 "target_unavailable",
4172 format!("module_id '{target_module_id}' does not provide the requested target"),
4173 )?]);
4174 }
4175
4176 if registration.state != ChannelState::Active {
4177 return Ok(vec![self.route_open_refusal_frame(
4178 ctx,
4179 &frame,
4180 &target_module_id,
4181 "registration_not_active",
4182 "target_unavailable",
4183 format!("module_id '{target_module_id}' is not active"),
4184 )?]);
4185 }
4186
4187 if self
4188 .forwarding
4189 .module_is_draining(&target_module_id)
4190 .map_err(RouterError::Forwarding)?
4191 {
4192 return Ok(vec![self.route_open_refusal_frame(
4193 ctx,
4194 &frame,
4195 &target_module_id,
4196 "reloading",
4197 "module_reloading",
4198 format!("module_id '{target_module_id}' is reloading"),
4199 )?]);
4200 }
4201
4202 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4203 process_liveness.process_live(&target_module_id) == Some(false)
4204 }) {
4205 if process_liveness.process_replacing(&target_module_id) {
4212 return Ok(vec![self.route_open_refusal_frame(
4213 ctx,
4214 &frame,
4215 &target_module_id,
4216 "reloading",
4217 "module_reloading",
4218 format!("module_id '{target_module_id}' is reloading"),
4219 )?]);
4220 }
4221 return Ok(vec![self.route_open_refusal_frame(
4222 ctx,
4223 &frame,
4224 &target_module_id,
4225 "supervisor_not_live",
4226 "target_unavailable",
4227 format!("module_id '{target_module_id}' is not live"),
4228 )?]);
4229 }
4230
4231 if !self
4232 .forwarding
4233 .has_live_module_connection(&target_module_id)
4234 .map_err(RouterError::Forwarding)?
4235 {
4236 return Ok(vec![self.route_open_refusal_frame(
4237 ctx,
4238 &frame,
4239 &target_module_id,
4240 "no_forwarding_connection",
4241 "target_unavailable",
4242 format!("module_id '{target_module_id}' has no live forwarding connection"),
4243 )?]);
4244 }
4245
4246 if let Some(error) =
4247 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4248 {
4249 self.observe_route_open_refusal(
4250 ctx,
4251 &target_module_id,
4252 "op_not_allowed",
4253 "op_not_allowed",
4254 );
4255 return Ok(vec![error]);
4256 }
4257
4258 let principal = match self.route_open_principal(&frame, consumer_identity)? {
4259 Ok(principal) => principal,
4260 Err(error) => {
4261 self.observe_route_open_refusal(
4262 ctx,
4263 &target_module_id,
4264 "bad_consumer_identity",
4265 "bad_consumer_identity",
4266 );
4267 return Ok(vec![error]);
4268 }
4269 };
4270
4271 if let Principal::Reserved {
4275 module_id: opening_module_id,
4276 } = &principal
4277 {
4278 if let Some(opening_registration) = self
4279 .registry
4280 .get_module(opening_module_id)
4281 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4282 {
4283 if let Some(capability) =
4284 denied_capability(&opening_registration.manifest, ®istration.manifest)
4285 {
4286 warn!(
4287 opening_module_id,
4288 target_module_id,
4289 capability,
4290 "refusing route.open because an attested capability deny edge matches"
4291 );
4292 return Ok(vec![self.route_open_refusal_frame(
4293 ctx,
4294 &frame,
4295 &target_module_id,
4296 "capability_deny_edge",
4297 "capability_forbidden",
4298 format!(
4299 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4300 ),
4301 )?]);
4302 }
4303 }
4304 }
4305
4306 if admission_facts.is_some() {
4307 let carrier_matches = matches!(
4308 &principal,
4309 Principal::Reserved { module_id }
4310 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4311 );
4312 if !carrier_matches {
4313 return Ok(vec![self.route_open_refusal_frame(
4314 ctx,
4315 &frame,
4316 &target_module_id,
4317 "admission_facts_carrier_not_permitted",
4318 "admission_facts_not_permitted",
4319 "admission facts may only be carried by the configured reserved module",
4320 )?]);
4321 }
4322
4323 let target_allowed = self
4324 .admission_facts_targets
4325 .as_ref()
4326 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4327 if !target_allowed {
4328 return Ok(vec![self.route_open_refusal_frame(
4329 ctx,
4330 &frame,
4331 &target_module_id,
4332 "admission_facts_target_not_listed",
4333 "admission_facts_target_not_allowed",
4334 format!(
4335 "admission facts are not permitted for target module_id '{target_module_id}'"
4336 ),
4337 )?]);
4338 }
4339
4340 }
4344
4345 let (bound_scope, mut scope_stamp) = match scope {
4351 None => (None, None),
4352 Some(selector) => {
4353 let owner_configured = match &selector.owner {
4354 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4360 _ => false,
4361 };
4362 let admitted = self
4363 .scopes
4364 .read()
4365 .unwrap_or_else(|poisoned| poisoned.into_inner())
4366 .admit(&principal, &target_module_id, &selector, owner_configured)
4367 .and_then(|admission| {
4368 crate::scopes::check_target_flow_support(
4369 &admission.stamp,
4370 &target_module_id,
4371 registration.manifest.capabilities.as_ref(),
4372 )?;
4373 crate::scopes::check_target_agent_run_support(
4374 &admission.stamp,
4375 &target_module_id,
4376 registration.manifest.capabilities.as_ref(),
4377 )?;
4378 Ok(admission)
4379 });
4380 match admitted {
4381 Ok(admission) => (
4382 Some(BoundScope {
4383 owner: admission.owner,
4384 scope_ref: admission.stamp.scope_ref.clone(),
4385 tag: admission.tag,
4386 }),
4387 Some(admission.stamp),
4388 ),
4389 Err(refusal) => {
4390 return Ok(vec![self.route_open_refusal_frame(
4391 ctx,
4392 &frame,
4393 &target_module_id,
4394 refusal.code,
4395 refusal.code,
4396 refusal.message,
4397 )?]);
4398 }
4399 }
4400 }
4401 };
4402
4403 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4425 Ok(project_root) => project_root,
4426 Err(err) => {
4427 return Ok(vec![control_error_frame(
4428 &frame,
4429 "invalid_project_root",
4430 err.to_string(),
4431 )?])
4432 }
4433 };
4434 identity.project_root = project_root.as_path().to_path_buf();
4435
4436 let _concurrency_guard = match self
4447 .route_bind_concurrency
4448 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4449 {
4450 Ok(guard) => guard,
4451 Err(in_flight) => {
4452 return Ok(vec![self.route_open_target_capacity_refusal(
4453 ctx,
4454 &frame,
4455 &target_module_id,
4456 in_flight,
4457 )?]);
4458 }
4459 };
4460
4461 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4464 RouteBindAdmission::Admitted { guard, probe } => {
4465 if probe {
4466 info!(
4467 module_id = %target_module_id,
4468 connection_id = ctx.connection_id.get(),
4469 "route.bind breaker half-open: admitting one probe"
4470 );
4471 }
4472 guard
4473 }
4474 RouteBindAdmission::Refused {
4475 consecutive_timeouts,
4476 retry_in,
4477 probe_in_flight,
4478 } => {
4479 return Ok(vec![self.route_open_breaker_refusal_frame(
4480 ctx,
4481 &frame,
4482 &target_module_id,
4483 consecutive_timeouts,
4484 retry_in,
4485 probe_in_flight,
4486 )?]);
4487 }
4488 };
4489
4490 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4495 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4496 let pending = match self
4497 .forwarding
4498 .begin_route_bind_relay_for(
4499 ctx.connection_id,
4500 ctx.egress.clone(),
4501 response_version(&frame),
4502 frame.header.corr,
4503 &target_module_id,
4504 principal.clone(),
4505 bound_scope,
4506 Some(project_root),
4507 relay_deadline,
4508 )
4509 .await
4510 {
4511 Ok(pending) => pending,
4512 Err(err) => {
4513 return Ok(vec![self.route_open_refusal_frame(
4514 ctx,
4515 &frame,
4516 &target_module_id,
4517 "relay_reservation_failed",
4518 forwarding_error_code(&err),
4519 err.to_string(),
4520 )?])
4521 }
4522 };
4523 let crate::forwarding::PendingRouteBindRelay {
4524 endpoint,
4525 module_sink,
4526 negotiated_ver,
4527 client_channel,
4528 client_epoch,
4529 module_channel,
4530 module_epoch,
4531 corr: relay_corr,
4532 receiver,
4533 } = pending;
4534 let mut reservation =
4535 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4536
4537 let relay_registration = self
4542 .registry
4543 .get_module_by_connection(endpoint.connection_id)
4544 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4545 let relay_capabilities = relay_registration
4546 .as_ref()
4547 .and_then(|registration| registration.manifest.capabilities.as_ref());
4548 if let Some(stamp) = scope_stamp
4549 .as_ref()
4550 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4551 {
4552 if let Err(refusal) = crate::scopes::check_target_flow_support(
4553 stamp,
4554 &target_module_id,
4555 relay_capabilities,
4556 )
4557 .and_then(|()| {
4558 crate::scopes::check_target_agent_run_support(
4559 stamp,
4560 &target_module_id,
4561 relay_capabilities,
4562 )
4563 }) {
4564 reservation.release_and_disarm();
4565 return Ok(vec![self.route_open_refusal_frame(
4566 ctx,
4567 &frame,
4568 &target_module_id,
4569 refusal.code,
4570 refusal.code,
4571 refusal.message,
4572 )?]);
4573 }
4574 }
4575
4576 let call_mode = if relay_capabilities
4579 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4580 {
4581 Some(derive_call_mode(scope_stamp.as_ref()))
4582 } else {
4583 if let Some(stamp) = scope_stamp.as_mut() {
4584 stamp.expires_at_ms = None;
4585 }
4586 None
4587 };
4588
4589 debug!(
4590 connection_id = ctx.connection_id.get(),
4591 client_channel,
4592 client_epoch,
4593 module_channel,
4594 module_epoch,
4595 "reserved route handle pair"
4596 );
4597 let principal_label = principal_label(&principal);
4600 let relay = ModuleControlRequest::RouteBind {
4601 route_channel: module_channel,
4602 epoch: module_epoch,
4603 target,
4604 identity,
4605 principal: Some(principal),
4606 consumer_capabilities,
4607 role_versions,
4608 admission_facts,
4609 scope: scope_stamp,
4610 call_mode,
4611 };
4612 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4613 RouterError::backend(
4614 0,
4615 frame.header.corr,
4616 format!("failed to encode route.bind request: {err}"),
4617 )
4618 })?;
4619 let relay_frame = Frame::build_with_version(
4620 negotiated_ver,
4621 FrameType::Request,
4622 control_flags(),
4623 0,
4624 0,
4625 relay_corr,
4626 relay_body,
4627 )
4628 .map_err(RouterError::FrameBuild)?;
4629
4630 if let Err(err) = module_sink.send(relay_frame).await {
4631 reservation.release_and_disarm();
4632 return Ok(vec![self.route_open_refusal_frame(
4633 ctx,
4634 &frame,
4635 &target_module_id,
4636 "relay_send_failed",
4637 "target_unavailable",
4638 err.to_string(),
4639 )?]);
4640 }
4641
4642 #[cfg(test)]
4643 if let Some(hold) = &self.route_bind_relay_sent_hold {
4644 hold.wait().await;
4648 hold.wait().await;
4649 }
4650
4651 if self
4656 .forwarding
4657 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4658 .map_err(RouterError::Forwarding)?
4659 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4660 {
4661 self.send_abandoned_route_bind_goodbye(
4662 &module_sink,
4663 negotiated_ver,
4664 module_channel,
4665 module_epoch,
4666 );
4667 }
4668
4669 match timeout_at(relay_deadline, receiver).await {
4670 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4671 reservation.disarm();
4672 if breaker.record_accepted() {
4673 info!(
4674 module_id = %target_module_id,
4675 "route.bind breaker closed: the probe was accepted"
4676 );
4677 }
4678 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
4679 Ok(Vec::new())
4680 }
4681 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4682 reservation.release_and_disarm();
4683 breaker.record_inconclusive();
4687 let scope_code = match body.code.as_str() {
4691 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4692 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4693 _ => None,
4694 };
4695 if let Some(code) = scope_code {
4696 self.observe_route_open_refusal(
4697 ctx,
4698 &target_module_id,
4699 "scope_changed_before_commit",
4700 code,
4701 );
4702 return Ok(vec![control_error_body_frame(&frame, body)?]);
4703 }
4704 self.counters
4705 .increment_route_open_refused("module_rejected");
4706 info!(
4707 target: "control",
4708 code = "module_rejected",
4709 module_code = ?body.code,
4710 module_id = ?target_module_id,
4711 connection_id = ctx.connection_id.get(),
4712 "route.open refused"
4713 );
4714 Ok(vec![control_error_body_frame(&frame, body)?])
4715 }
4716 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
4717 reservation.release_and_disarm();
4718 breaker.record_inconclusive();
4719 tracing::warn!(
4726 module_id = %target_module_id,
4727 "route.bind relay abandoned: {message}"
4728 );
4729 Ok(vec![self.route_open_refusal_frame(
4730 ctx,
4731 &frame,
4732 &target_module_id,
4733 "relay_abandoned",
4734 "target_unavailable",
4735 message,
4736 )?])
4737 }
4738 Ok(Err(_)) => {
4739 reservation.release_and_disarm();
4740 breaker.record_inconclusive();
4741 Ok(vec![self.route_open_refusal_frame(
4742 ctx,
4743 &frame,
4744 &target_module_id,
4745 "relay_waiter_canceled",
4746 "target_unavailable",
4747 "route.bind relay waiter was canceled before the module responded",
4748 )?])
4749 }
4750 Err(_) => {
4751 reservation.release_and_disarm();
4752 if let Some(opened) = breaker.record_timeout(
4756 self.route_bind_breaker_threshold,
4757 self.route_bind_breaker_cooldown,
4758 ) {
4759 warn!(
4760 module_id = %target_module_id,
4761 consecutive_timeouts = opened.consecutive_timeouts,
4762 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
4763 reopened_after_probe = opened.reopened_after_probe,
4764 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
4765 );
4766 }
4767 tracing::warn!(
4775 module_id = %target_module_id,
4776 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
4777 "route.bind relay timed out: module did not ack within budget"
4778 );
4779 Ok(vec![self.route_open_refusal_frame(
4780 ctx,
4781 &frame,
4782 &target_module_id,
4783 "relay_timed_out",
4784 "module_timeout",
4785 format!(
4786 "module_id '{target_module_id}' did not answer route.bind within {:?}",
4787 route_bind_relay_timeout
4788 ),
4789 )?])
4790 }
4791 }
4792 }
4793
4794 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4795 let response = ClientControlResponse::SupervisorSpawnSnapshot {
4796 snapshot: self.supervisor.spawn_snapshot(),
4797 };
4798 Ok(vec![control_response_body_frame(
4799 &frame,
4800 &response,
4801 "ClientControlResponse::SupervisorSpawnSnapshot",
4802 )?])
4803 }
4804
4805 fn handle_supervisor_spawn_subscribe(
4806 &self,
4807 ctx: &RouteCtx,
4808 frame: Frame,
4809 since: Option<SpawnCursor>,
4810 ) -> Result<Vec<Frame>, RouterError> {
4811 match self.supervisor.subscribe_spawns(
4812 ctx.connection_id,
4813 frame.header.corr,
4814 response_version(&frame),
4815 since,
4816 ctx.egress.clone(),
4817 ) {
4818 Ok(()) => Ok(Vec::new()),
4819 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
4820 Ok(vec![control_error_body_frame(
4821 &frame,
4822 ErrorBody {
4823 code: "spawn_cursor_incarnation_mismatch".to_string(),
4824 message: "spawn cursor belongs to a different daemon incarnation"
4825 .to_string(),
4826 detail: Some(serde_json::json!({
4827 "current_daemon_incarnation": current
4828 })),
4829 },
4830 )?])
4831 }
4832 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
4833 &frame,
4834 ErrorBody {
4835 code: "spawn_cursor_too_old".to_string(),
4836 message: "spawn cursor predates the retained event ring".to_string(),
4837 detail: Some(serde_json::json!({
4838 "oldest_retained_cursor": oldest
4839 })),
4840 },
4841 )?]),
4842 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
4843 0,
4844 frame.header.corr,
4845 format!("failed to open supervisor spawn subscription: {error}"),
4846 )),
4847 }
4848 }
4849
4850 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4851 let generation = self
4852 .registry
4853 .generation()
4854 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4855 let mut modules = Vec::new();
4856 for module in self.supervisor.list() {
4857 let status = module.status_for_control("list").map_err(|err| {
4858 RouterError::backend(
4859 0,
4860 frame.header.corr,
4861 format!("failed to read supervisor status: {err}"),
4862 )
4863 })?;
4864 let (configured, _) = module.configuration().map_err(|err| {
4865 RouterError::backend(
4866 0,
4867 frame.header.corr,
4868 format!("failed to read module configuration: {err}"),
4869 )
4870 })?;
4871 let image = module.running_image_agreement().await;
4873 let resources = Some(module.child_resource_usage());
4876 let pending_reload = Some(reload_verdict(
4877 &configured.program,
4878 status.spawned_from.as_deref(),
4879 image,
4880 ));
4881 modules.push(
4884 SupervisorEntry::new(
4885 status.module_id,
4886 status.state.to_string(),
4887 status.enabled,
4888 status.live,
4889 status.health.status,
4890 )
4891 .with_launch_nonce_env(Some(!cfg!(unix)))
4892 .with_launch_nonce_source(status.launch_nonce_source)
4893 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
4894 .with_protocol(status.protocol)
4895 .with_pending_reload(pending_reload)
4896 .with_last_probe_ms(status.health.last_probe_ms)
4897 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
4898 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
4899 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
4900 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
4901 .with_restart_count(Some(status.restart_count))
4902 .with_max_restarts(Some(status.max_restarts))
4903 .with_lifetime_restarts(Some(status.lifetime_restarts))
4904 .with_spawn_generation(Some(status.spawn_generation))
4905 .with_restart_window_secs(Some(status.restart_window.as_secs()))
4906 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
4907 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
4908 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
4909 .with_resources(resources),
4910 );
4911 }
4912 let response = ClientControlResponse::SupervisorList {
4913 generation,
4914 modules,
4915 };
4916 Ok(vec![control_response_body_frame(
4917 &frame,
4918 &response,
4919 "ClientControlResponse::SupervisorList",
4920 )?])
4921 }
4922
4923 fn handle_supervisor_stderr_tail(
4924 &self,
4925 frame: Frame,
4926 module_id: String,
4927 max_lines: Option<u32>,
4928 max_bytes: Option<u32>,
4929 ) -> Result<Vec<Frame>, RouterError> {
4930 let Some(module) = self.supervisor.get(&module_id) else {
4931 return Ok(vec![control_error_frame(
4932 &frame,
4933 "unknown_module",
4934 format!("module_id '{module_id}' is not supervised"),
4935 )?]);
4936 };
4937
4938 let snapshot = module.stderr_tail(
4939 max_lines.map(|value| value as usize),
4940 max_bytes.map(|value| value as usize),
4941 );
4942
4943 let response = ClientControlResponse::SupervisorStderrTail {
4944 module_id,
4945 tail: StderrTail {
4946 capture: match snapshot.capture {
4947 CaptureState::Captured => StderrCaptureState::Captured,
4948 CaptureState::Incomplete { reason } => {
4949 StderrCaptureState::Incomplete { reason }
4950 }
4951 CaptureState::NotCaptured { reason } => {
4952 StderrCaptureState::NotCaptured { reason }
4953 }
4954 },
4955 entries: snapshot
4956 .entries
4957 .into_iter()
4958 .map(|entry| match entry {
4959 TailEntry::Line {
4960 text,
4961 truncated,
4962 at_ms,
4963 } => StderrTailEntry::Line {
4964 text,
4965 truncated,
4966 at_ms,
4967 },
4968 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
4969 })
4970 .collect(),
4971 dropped_lines: snapshot.dropped_lines,
4972 },
4973 };
4974 Ok(vec![control_response_body_frame(
4975 &frame,
4976 &response,
4977 "ClientControlResponse::SupervisorStderrTail",
4978 )?])
4979 }
4980
4981 async fn handle_supervisor_terminals(
4982 &self,
4983 frame: Frame,
4984 module_id: String,
4985 ) -> Result<Vec<Frame>, RouterError> {
4986 let Some(module) = self.supervisor.get(&module_id) else {
4987 return Ok(vec![control_error_frame(
4988 &frame,
4989 "unknown_module",
4990 format!("module_id '{module_id}' is not supervised"),
4991 )?]);
4992 };
4993
4994 let terminals = module
4997 .read_durable_terminal_history()
4998 .await
4999 .map_err(|error| {
5000 RouterError::backend(
5001 0,
5002 frame.header.corr,
5003 format!("failed to read terminal history: {error}"),
5004 )
5005 })?;
5006 let response = ClientControlResponse::SupervisorTerminals {
5007 module_id,
5008 terminals,
5009 };
5010 Ok(vec![control_response_body_frame(
5011 &frame,
5012 &response,
5013 "ClientControlResponse::SupervisorTerminals",
5014 )?])
5015 }
5016
5017 fn handle_supervisor_routes(
5018 &self,
5019 frame: Frame,
5020 module_id: Option<String>,
5021 ) -> Result<Vec<Frame>, RouterError> {
5022 let modules = self
5023 .forwarding
5024 .route_census(module_id.as_deref())
5025 .map_err(RouterError::Forwarding)?
5026 .into_iter()
5027 .map(|(module_id, routes)| SupervisorRouteModule {
5028 module_id,
5029 routes: routes
5030 .into_iter()
5031 .map(|route| SupervisorRoute {
5032 consumer: match route.principal {
5033 Principal::Reserved { module_id } => {
5034 SupervisorRouteConsumer::Reserved { module_id }
5035 }
5036 Principal::Direct | Principal::Unverified => {
5037 SupervisorRouteConsumer::Direct {
5038 connection_id: route.goodbye_target.connection_id.get(),
5039 }
5040 }
5041 },
5042 age_ms: Instant::now()
5043 .saturating_duration_since(route.bound_at)
5044 .as_millis()
5045 .try_into()
5046 .unwrap_or(u64::MAX),
5047 draining: route.draining,
5048 drain_reason: route.drain_reason,
5049 })
5050 .collect(),
5051 })
5052 .collect();
5053 let response = ClientControlResponse::SupervisorRoutes { modules };
5054 Ok(vec![control_response_body_frame(
5055 &frame,
5056 &response,
5057 "ClientControlResponse::SupervisorRoutes",
5058 )?])
5059 }
5060
5061 async fn handle_supervisor_provenance(
5062 &self,
5063 frame: Frame,
5064 module_id: Option<String>,
5065 ) -> Result<Vec<Frame>, RouterError> {
5066 let mut selected = if let Some(module_id) = module_id {
5067 let Some(module) = self.supervisor.get(&module_id) else {
5068 return Ok(vec![control_error_frame(
5069 &frame,
5070 "unknown_module",
5071 format!("module_id '{module_id}' is not supervised"),
5072 )?]);
5073 };
5074 vec![module]
5075 } else {
5076 self.supervisor.list()
5077 };
5078
5079 let mut modules = Vec::with_capacity(selected.len());
5080 for module in selected.drain(..) {
5081 let (status, observed_image) = module
5082 .status_and_running_image_agreement()
5083 .await
5084 .map_err(|err| {
5085 RouterError::backend(
5086 0,
5087 frame.header.corr,
5088 format!("failed to read supervisor status: {err}"),
5089 )
5090 })?;
5091 let module_declared = self
5092 .registry
5093 .get_module(&status.module_id)
5094 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5095 .and_then(|registration| registration.manifest.provenance)
5096 .map(|build| ModuleDeclaredProvenance::Reported { build })
5097 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5098 #[cfg(test)]
5099 let running_image = match &self.provenance_probe_override {
5100 Some(result) => result.clone(),
5101 None => observed_image,
5102 };
5103 #[cfg(not(test))]
5104 let running_image = observed_image;
5105 modules.push(SupervisorModuleProvenance {
5106 module_id: status.module_id,
5107 module_declared,
5108 daemon_observed: SupervisorObservedProcess {
5109 launch_nonce_source: status.launch_nonce_source,
5110 pid: status.pid,
5111 spawned_at_ms: status.spawned_at_ms,
5112 spawned_from: status.spawned_from,
5113 running_image,
5114 },
5115 });
5116 }
5117 let daemon = SupervisorDaemonProvenance {
5118 daemon_build: self.daemon_provenance.build.clone(),
5119 daemon_observed: DaemonObservedProcess {
5120 pid: self.daemon_provenance.pid,
5121 started_at_ms: self
5122 .daemon_provenance
5123 .start_clock
5124 .map(|clock| clock.started_at_ms())
5125 .or(self.daemon_provenance.started_at_ms),
5126 running_image: self
5127 .daemon_provenance
5128 .probe
5129 .observe(
5130 self.daemon_provenance.pid,
5131 self.daemon_provenance.executable_path.as_deref(),
5132 self.daemon_provenance.executable_identity.clone(),
5133 self.daemon_provenance.process_start_time,
5134 )
5135 .await,
5136 },
5137 };
5138 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5139 Ok(vec![control_response_body_frame(
5140 &frame,
5141 &response,
5142 "ClientControlResponse::SupervisorProvenance",
5143 )?])
5144 }
5145
5146 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5147 self.refresh_capability_requirements();
5148 let generation = self
5149 .registry
5150 .generation()
5151 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5152 let modules = self
5153 .supervisor
5154 .list()
5155 .into_iter()
5156 .map(|module| {
5157 let status = module.status_for_control("health").map_err(|err| {
5158 RouterError::backend(
5159 0,
5160 frame.header.corr,
5161 format!("failed to read supervisor health: {err}"),
5162 )
5163 })?;
5164 let module_id = status.module_id;
5165 let capability_detail = self
5166 .capability_evaluator
5167 .required_problem_detail(&module_id);
5168 Ok(SupervisorHealthEntry {
5169 module_id,
5170 status: status.health.status,
5171 detail: append_capability_problem_detail(
5172 status.health.detail,
5173 capability_detail,
5174 ),
5175 metrics: status.health.metrics,
5176 consecutive_failures: status.health.consecutive_failures,
5177 late_answer_count: status.health.late_answer_count,
5178 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5179 last_action: status.health.last_action,
5180 last_action_ms: status.health.last_action_ms,
5181 last_probe_ms: status.health.last_probe_ms,
5182 })
5183 })
5184 .collect::<Result<Vec<_>, RouterError>>()?;
5185 let response = ClientControlResponse::SupervisorHealth {
5186 generation,
5187 modules,
5188 };
5189 Ok(vec![control_response_body_frame(
5190 &frame,
5191 &response,
5192 "ClientControlResponse::SupervisorHealth",
5193 )?])
5194 }
5195
5196 async fn handle_supervisor_restart(
5197 &self,
5198 frame: Frame,
5199 module_id: String,
5200 drain_timeout_ms: Option<u64>,
5201 requester: Principal,
5202 ) -> Result<Vec<Frame>, RouterError> {
5203 let operation_lock = self.supervisor.operation_lock();
5204 let _operation_guard = operation_lock.lock().await;
5205 let Some(module) = self.supervisor.get(&module_id) else {
5206 return Ok(vec![control_error_frame(
5207 &frame,
5208 "unknown_module",
5209 format!("module_id '{module_id}' is not supervised"),
5210 )?]);
5211 };
5212
5213 self.route_outages.mark_operator_action(&module_id);
5214 if let Err(err) = module
5215 .restart_requested(drain_timeout_ms, Some(requester))
5216 .await
5217 {
5218 self.route_outages
5219 .operator_action_ended_unrefused(&module_id);
5220 let (code, message) = match err {
5221 crate::supervise::SuperviseError::Disabled { .. } => {
5222 ("module_disabled", err.to_string())
5223 }
5224 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5225 ("swap_in_progress", err.to_string())
5226 }
5227 _ => (
5228 "target_unavailable",
5229 format!("failed to restart module_id '{module_id}': {err}"),
5230 ),
5231 };
5232 return Ok(vec![control_error_frame(&frame, code, message)?]);
5233 }
5234
5235 let response = ClientControlResponse::SupervisorAck {
5236 module_id,
5237 applied: true,
5238 };
5239 Ok(vec![control_response_body_frame(
5240 &frame,
5241 &response,
5242 "ClientControlResponse::SupervisorAck",
5243 )?])
5244 }
5245
5246 async fn handle_supervisor_swap(
5250 &self,
5251 frame: Frame,
5252 module_id: String,
5253 ready_timeout_ms: Option<u64>,
5254 requester: Principal,
5255 ) -> Result<Vec<Frame>, RouterError> {
5256 let module = {
5263 let operation_lock = self.supervisor.operation_lock();
5264 let _operation_guard = operation_lock.lock().await;
5265 self.supervisor.get(&module_id)
5266 };
5267 let Some(module) = module else {
5268 return Ok(vec![control_error_frame(
5269 &frame,
5270 "unknown_module",
5271 format!("module_id '{module_id}' is not supervised"),
5272 )?]);
5273 };
5274
5275 self.route_outages.mark_operator_action(&module_id);
5276 if let Err(err) = module
5277 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5278 .await
5279 {
5280 self.route_outages
5281 .operator_action_ended_unrefused(&module_id);
5282 use crate::supervise::SuperviseError;
5283 let message = err.to_string();
5284 let error = match err {
5285 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5286 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5287 code: "swap_refused".to_string(),
5288 message,
5289 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5290 },
5291 SuperviseError::SwapFailed {
5292 arm,
5293 candidate_exit,
5294 ..
5295 } => ErrorBody {
5296 code: "swap_failed".to_string(),
5297 message,
5298 detail: Some(serde_json::json!({
5299 "arm": arm.as_str(),
5300 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5301 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5302 })),
5303 },
5304 _ => ErrorBody::new(
5305 "target_unavailable",
5306 format!("failed to swap module_id '{module_id}': {message}"),
5307 ),
5308 };
5309 return Ok(vec![control_error_body_frame(&frame, error)?]);
5310 }
5311 self.route_outages
5315 .operator_action_ended_unrefused(&module_id);
5316
5317 let response = ClientControlResponse::SupervisorAck {
5318 module_id,
5319 applied: true,
5320 };
5321 Ok(vec![control_response_body_frame(
5322 &frame,
5323 &response,
5324 "ClientControlResponse::SupervisorAck",
5325 )?])
5326 }
5327
5328 async fn handle_supervisor_reload(
5329 &self,
5330 frame: Frame,
5331 module_id: String,
5332 requester: Principal,
5333 ) -> Result<Vec<Frame>, RouterError> {
5334 let operation_lock = self.supervisor.operation_lock();
5335 let _operation_guard = operation_lock.lock().await;
5336 let Some(module) = self.supervisor.get(&module_id) else {
5337 return Ok(vec![control_error_frame(
5338 &frame,
5339 "unknown_module",
5340 format!("module_id '{module_id}' is not supervised"),
5341 )?]);
5342 };
5343
5344 self.route_outages.mark_operator_action(&module_id);
5345 if let Err(err) = module.reload_requested(Some(requester)).await {
5346 self.route_outages
5347 .operator_action_ended_unrefused(&module_id);
5348 let (code, message) = match err {
5349 crate::supervise::SuperviseError::Disabled { .. } => {
5350 ("module_disabled", err.to_string())
5351 }
5352 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5353 ("swap_in_progress", err.to_string())
5354 }
5355 _ => (
5356 "reload_failed",
5357 format!("failed to reload module_id '{module_id}': {err}"),
5358 ),
5359 };
5360 return Ok(vec![control_error_frame(&frame, code, message)?]);
5361 }
5362
5363 let response = ClientControlResponse::SupervisorAck {
5364 module_id,
5365 applied: true,
5366 };
5367 Ok(vec![control_response_body_frame(
5368 &frame,
5369 &response,
5370 "ClientControlResponse::SupervisorAck",
5371 )?])
5372 }
5373
5374 async fn handle_supervisor_rescan(
5375 &self,
5376 frame: Frame,
5377 preview: bool,
5378 requester: Principal,
5379 ) -> Result<Vec<Frame>, RouterError> {
5380 let Some(context) = self.rescan.clone() else {
5381 return Ok(vec![control_error_frame(
5382 &frame,
5383 "rescan_unavailable",
5384 "the daemon was not started with a reloadable config path".to_string(),
5385 )?]);
5386 };
5387
5388 let operation_lock = self.supervisor.operation_lock();
5389 let _operation_guard = operation_lock.lock().await;
5390 let loaded = match crate::daemon_config::load_with_activity(&context.config_path) {
5391 Ok(config) => config,
5392 Err(err) => {
5393 return Ok(vec![control_error_frame(
5394 &frame,
5395 "invalid_daemon_config",
5396 format!("supervisor rescan rejected daemon config: {err}"),
5397 )?])
5398 }
5399 };
5400 let Some((config, activity_config)) = loaded else {
5408 return Ok(vec![control_error_frame(
5409 &frame,
5410 "invalid_daemon_config",
5411 format!(
5412 "daemon config not found at {}; refusing to rescan (an absent config would \
5413 retire every supervised module)",
5414 context.config_path.display()
5415 ),
5416 )?]);
5417 };
5418 let (
5419 configured_port,
5420 storage_config,
5421 admission_facts_carrier_module_id,
5422 admission_facts_targets,
5423 scope_authority_owners,
5424 privileged_operator_prompt,
5425 modules,
5426 reserved_capabilities,
5427 ) = (
5428 config.port,
5429 config.storage,
5430 config.admission_facts_carrier_module_id,
5431 config.admission_facts_targets,
5432 config.scope_authority_owners,
5433 config.privileged_operator_prompt,
5434 config.modules,
5435 config.reserved_capabilities,
5436 );
5437
5438 let mut restart_required = Vec::new();
5447 for section in RestartRequiredSection::ALL {
5448 let changed = match section {
5449 RestartRequiredSection::Port => configured_port != context.configured_port,
5450 RestartRequiredSection::Storage => storage_config != context.storage_config,
5451 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5452 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5453 }
5454 RestartRequiredSection::AdmissionFactsTargets => {
5455 admission_facts_targets != context.admission_facts_targets
5456 }
5457 RestartRequiredSection::ScopeAuthorityOwners => {
5458 scope_authority_owners != context.scope_authority_owners
5459 }
5460 RestartRequiredSection::PrivilegedOperatorPrompt => {
5461 privileged_operator_prompt != context.privileged_operator_prompt
5462 }
5463 };
5464 if changed {
5465 restart_required.push(section.label().to_string());
5466 }
5467 }
5468 if !restart_required.is_empty() {
5469 warn!(
5470 config_path = %context.config_path.display(),
5471 sections = %restart_required.join(", "),
5472 "daemon config changed outside the modules section; restart the daemon to apply those changes"
5473 );
5474 }
5475
5476 for configured in &modules {
5477 if let Err(err) = validate_spec(&configured.module_spec()) {
5478 return Ok(vec![control_error_frame(
5479 &frame,
5480 "invalid_daemon_config",
5481 format!("supervisor rescan rejected daemon config: {err}"),
5482 )?]);
5483 }
5484 }
5485
5486 let configured_capabilities = modules
5487 .iter()
5488 .map(|module| (module.module_id.clone(), module.enabled))
5489 .collect::<Vec<_>>();
5490 let preview_capability_warnings = if preview {
5491 let (_, registrations) = self.runtime_capability_snapshot()?;
5492 let current_modules = self
5493 .supervisor
5494 .list()
5495 .into_iter()
5496 .map(|module| module.module_id().to_string())
5497 .collect::<BTreeSet<_>>();
5498 let resulting_modules = configured_capabilities.clone();
5499 let removed = current_modules
5500 .into_iter()
5501 .filter(|module_id| {
5502 !resulting_modules
5503 .iter()
5504 .any(|(configured_id, _)| configured_id == module_id)
5505 })
5506 .collect::<Vec<_>>();
5507 self.capability_evaluator.preview_removal_warnings(
5508 resulting_modules,
5509 &removed,
5510 ®istrations,
5511 )
5512 } else {
5513 Vec::new()
5514 };
5515 let result = match self
5516 .reconcile_supervised_modules(&context.supervisor, modules, preview, requester.clone())
5517 .await
5518 {
5519 Ok(result) => result,
5520 Err(message) => {
5521 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5522 }
5523 };
5524 if !preview {
5525 if let Some(ledger) = &self.activity {
5526 ledger.module_rescanned(&result.added, &result.removed, requester);
5527 }
5528 if let Err(code) = self.configure_activity(activity_config).await {
5529 warn!(
5530 code,
5531 "activity retention could not be applied during rescan"
5532 );
5533 }
5534 self.capability_evaluator
5535 .configure(configured_capabilities, reserved_capabilities);
5536 self.capability_evaluator.wake_deadline_loop();
5537 self.refresh_capability_requirements();
5538 }
5539 let mut result = result;
5540 result.restart_required = restart_required;
5541 result.capability_warnings = preview_capability_warnings;
5542 let response = ClientControlResponse::SupervisorRescan { result };
5543 Ok(vec![control_response_body_frame(
5544 &frame,
5545 &response,
5546 "ClientControlResponse::SupervisorRescan",
5547 )?])
5548 }
5549
5550 async fn handle_supervisor_release_reserved(
5551 &self,
5552 frame: Frame,
5553 module_id: String,
5554 ) -> Result<Vec<Frame>, RouterError> {
5555 let Some(context) = self.rescan.clone() else {
5556 return Ok(vec![control_error_frame(
5557 &frame,
5558 "release_unavailable",
5559 "reserved-id release requires a daemon started with a reloadable config path",
5560 )?]);
5561 };
5562 let operation_lock = self.supervisor.operation_lock();
5563 let _operation_guard = operation_lock.lock().await;
5564 let loaded = match crate::daemon_config::load(&context.config_path) {
5565 Ok(Some(config)) => config,
5566 Ok(None) => {
5567 return Ok(vec![control_error_frame(
5568 &frame,
5569 "invalid_daemon_config",
5570 format!(
5571 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5572 context.config_path.display()
5573 ),
5574 )?])
5575 }
5576 Err(err) => {
5577 return Ok(vec![control_error_frame(
5578 &frame,
5579 "invalid_daemon_config",
5580 format!("unable to verify reserved-id release against daemon config: {err}"),
5581 )?])
5582 }
5583 };
5584 if loaded
5585 .modules
5586 .iter()
5587 .any(|configured| configured.module_id == module_id)
5588 {
5589 return Ok(vec![control_error_frame(
5590 &frame,
5591 "reserved_module_configured",
5592 format!(
5593 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5594 ),
5595 )?]);
5596 }
5597 if !self.supervisor.release_retained_reserved_gate(&module_id) {
5598 return Ok(vec![control_error_frame(
5599 &frame,
5600 "reserved_gate_not_retained",
5601 format!(
5602 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
5603 ),
5604 )?]);
5605 }
5606
5607 let response = ClientControlResponse::SupervisorAck {
5608 module_id,
5609 applied: true,
5610 };
5611 Ok(vec![control_response_body_frame(
5612 &frame,
5613 &response,
5614 "ClientControlResponse::SupervisorAck",
5615 )?])
5616 }
5617
5618 async fn reconcile_supervised_modules(
5627 &self,
5628 supervisor: &Supervisor,
5629 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
5630 preview: bool,
5631 requester: Principal,
5632 ) -> Result<SupervisorRescanResult, String> {
5633 let mut current = BTreeMap::new();
5634 for module in self.supervisor.list() {
5635 let (spec, health) = module.configuration().map_err(|err| {
5636 format!(
5637 "failed to read configuration for module_id '{}': {err}",
5638 module.module_id()
5639 )
5640 })?;
5641 let enabled = module
5642 .status()
5643 .map_err(|err| {
5644 format!(
5645 "failed to read status for module_id '{}': {err}",
5646 module.module_id()
5647 )
5648 })?
5649 .enabled;
5650 current.insert(
5651 module.module_id().to_string(),
5652 (module, spec, health, enabled),
5653 );
5654 }
5655 let configured = configured_modules
5656 .into_iter()
5657 .map(|module| (module.module_id.clone(), module))
5658 .collect::<BTreeMap<_, _>>();
5659
5660 let added = configured
5661 .keys()
5662 .filter(|module_id| !current.contains_key(*module_id))
5663 .cloned()
5664 .collect::<Vec<_>>();
5665 let removed = current
5666 .keys()
5667 .filter(|module_id| !configured.contains_key(*module_id))
5668 .cloned()
5669 .collect::<Vec<_>>();
5670 let mut changed_pending_reload = Vec::new();
5671 let mut configuration_changes = BTreeSet::new();
5672 let mut enabled_changes = BTreeSet::new();
5673 let mut unchanged = 0_u32;
5674
5675 for (module_id, configured_module) in &configured {
5676 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5677 else {
5678 continue;
5679 };
5680 let launch_changed = *current_spec != configured_module.module_spec();
5684 let configuration_changed =
5685 launch_changed || *current_health != configured_module.health;
5686 let enabled_changed = *current_enabled != configured_module.enabled;
5687 if configuration_changed {
5688 configuration_changes.insert(module_id.clone());
5689 }
5690 if launch_changed {
5691 changed_pending_reload.push(module_id.clone());
5692 }
5693 if enabled_changed {
5694 enabled_changes.insert(module_id.clone());
5695 }
5696 if !configuration_changed && !enabled_changed {
5697 unchanged = unchanged.saturating_add(1);
5698 }
5699 }
5700
5701 if preview {
5707 return Ok(SupervisorRescanResult {
5708 added,
5709 removed,
5710 changed_pending_reload,
5711 enabled_changes: enabled_changes.iter().cloned().collect(),
5712 unchanged,
5713 preview: true,
5714 restart_required: Vec::new(),
5718 capability_warnings: Vec::new(),
5719 });
5720 }
5721
5722 for module_id in &removed {
5723 let module = ¤t
5724 .get(module_id)
5725 .expect("removed module came from current supervisor state")
5726 .0;
5727 module
5728 .retire_requested(Some(requester.clone()))
5729 .await
5730 .map_err(|err| {
5731 format!("failed to retire module_id '{module_id}' during rescan: {err}")
5732 })?;
5733 self.supervisor.record_rescan_removal(module_id);
5767 self.supervisor.retire(module_id);
5768 self.route_outages.forget(module_id);
5769 }
5770
5771 for module_id in configured.keys() {
5772 let Some((module, _, _, _)) = current.get(module_id) else {
5773 continue;
5774 };
5775 let configured_module = configured
5776 .get(module_id)
5777 .expect("configured module id came from configured map");
5778 if configuration_changes.contains(module_id) {
5779 module
5780 .update_configuration(
5781 configured_module.module_spec(),
5782 configured_module.health.clone(),
5783 configured_module.drain_timeout_ms,
5784 )
5785 .await
5786 .map_err(|err| {
5787 format!(
5788 "failed to update module_id '{module_id}' configuration during rescan: {err}"
5789 )
5790 })?;
5791 }
5792 if enabled_changes.contains(module_id) {
5793 self.route_outages.mark_operator_action(module_id);
5796 module
5797 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
5798 .await
5799 .map_err(|err| {
5800 self.route_outages.operator_action_ended_unrefused(module_id);
5801 format!(
5802 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
5803 configured_module.enabled
5804 )
5805 })?;
5806 }
5807 }
5808
5809 for module_id in &added {
5810 let configured_module = configured
5811 .get(module_id)
5812 .expect("added module id came from configured map");
5813 supervisor
5814 .supervise_configured_requested(
5815 configured_module.module_spec(),
5816 configured_module.enabled,
5817 configured_module.health.clone(),
5818 configured_module.drain_timeout_ms,
5819 configured_module.restart,
5820 Some(requester.clone()),
5821 )
5822 .map_err(|err| {
5823 format!("failed to add module_id '{module_id}' during rescan: {err}")
5824 })?;
5825 }
5826
5827 Ok(SupervisorRescanResult {
5828 added,
5829 removed,
5830 changed_pending_reload,
5831 enabled_changes: enabled_changes.iter().cloned().collect(),
5832 unchanged,
5833 preview: false,
5834 restart_required: Vec::new(),
5837 capability_warnings: Vec::new(),
5838 })
5839 }
5840
5841 async fn handle_supervisor_set_enabled(
5842 &self,
5843 frame: Frame,
5844 module_id: String,
5845 enabled: bool,
5846 requester: Principal,
5847 ) -> Result<Vec<Frame>, RouterError> {
5848 let operation_lock = self.supervisor.operation_lock();
5849 let _operation_guard = operation_lock.lock().await;
5850 let Some(module) = self.supervisor.get(&module_id) else {
5851 return Ok(vec![control_error_frame(
5852 &frame,
5853 "unknown_module",
5854 format!("module_id '{module_id}' is not supervised"),
5855 )?]);
5856 };
5857
5858 self.route_outages.mark_operator_action(&module_id);
5861 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
5862 Ok(applied) => applied,
5863 Err(err) => {
5864 self.route_outages
5865 .operator_action_ended_unrefused(&module_id);
5866 return Ok(vec![control_error_frame(
5867 &frame,
5868 "target_unavailable",
5869 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
5870 )?]);
5871 }
5872 };
5873 if !applied {
5874 self.route_outages
5877 .operator_action_ended_unrefused(&module_id);
5878 }
5879
5880 self.capability_evaluator.wake_deadline_loop();
5881 self.refresh_capability_requirements();
5882 let response = ClientControlResponse::SupervisorAck { module_id, applied };
5883 Ok(vec![control_response_body_frame(
5884 &frame,
5885 &response,
5886 "ClientControlResponse::SupervisorAck",
5887 )?])
5888 }
5889
5890 async fn handle_supervisor_health_probe(
5891 &self,
5892 frame: Frame,
5893 module_id: String,
5894 ) -> Result<Vec<Frame>, RouterError> {
5895 self.refresh_capability_requirements();
5896 let Some(registration) = self
5897 .registry
5898 .get_module(&module_id)
5899 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5900 else {
5901 return Ok(vec![control_error_frame(
5902 &frame,
5903 "unknown_module",
5904 format!("module_id '{module_id}' is not registered"),
5905 )?]);
5906 };
5907
5908 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
5921 {
5922 return Ok(vec![control_error_frame(
5923 &frame,
5924 "health_not_advertised",
5925 format!("module_id '{module_id}' did not advertise health.check"),
5926 )?]);
5927 }
5928
5929 let deadline = Instant::now() + self.health_probe_timeout;
5930 let pending = match self.forwarding.begin_module_control_rpc_for(
5931 &module_id,
5932 MODULE_CONTROL_OP_HEALTH_CHECK,
5933 deadline,
5934 ) {
5935 Ok(pending) => pending,
5936 Err(err) => {
5937 return Ok(vec![control_error_frame(
5938 &frame,
5939 forwarding_error_code(&err),
5940 err.to_string(),
5941 )?])
5942 }
5943 };
5944
5945 let PendingModuleControlRpc {
5946 endpoint,
5947 module_sink,
5948 negotiated_ver,
5949 corr: probe_corr,
5950 receiver,
5951 } = pending;
5952 let mut guard =
5953 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
5954 let probe_body =
5955 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5956 RouterError::backend(
5957 0,
5958 frame.header.corr,
5959 format!("failed to encode health.check request: {err}"),
5960 )
5961 })?;
5962 let probe_frame = Frame::build_with_version(
5963 negotiated_ver,
5964 FrameType::Request,
5965 control_flags(),
5966 0,
5967 0,
5968 probe_corr,
5969 probe_body,
5970 )
5971 .map_err(RouterError::FrameBuild)?;
5972
5973 if let Err(err) = module_sink.send(probe_frame).await {
5974 return Ok(vec![control_error_frame(
5975 &frame,
5976 "target_unavailable",
5977 err.to_string(),
5978 )?]);
5979 }
5980
5981 match timeout_at(deadline, receiver).await {
5982 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5983 guard.disarm();
5984 let Some(report) = response.health_report() else {
5985 return Ok(vec![control_error_frame(
5986 &frame,
5987 "invalid_control_body",
5988 "health.check RPC returned a non-health response",
5989 )?]);
5990 };
5991 let HealthReport {
5996 status,
5997 detail,
5998 metrics,
5999 } = report;
6000 let capability_detail = self
6001 .capability_evaluator
6002 .required_problem_detail(&module_id);
6003 let response = ClientControlResponse::SupervisorHealthProbe {
6004 module_id,
6005 status,
6006 detail: append_capability_problem_detail(detail, capability_detail),
6007 metrics,
6008 };
6009 Ok(vec![control_response_body_frame(
6010 &frame,
6011 &response,
6012 "ClientControlResponse::SupervisorHealthProbe",
6013 )?])
6014 }
6015 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6016 guard.disarm();
6017 Ok(vec![control_error_body_frame(&frame, body)?])
6018 }
6019 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6020 guard.disarm();
6021 Ok(vec![control_error_frame(
6022 &frame,
6023 "target_unavailable",
6024 message,
6025 )?])
6026 }
6027 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6028 guard.disarm();
6029 Ok(vec![control_error_frame(
6030 &frame,
6031 "invalid_control_body",
6032 message,
6033 )?])
6034 }
6035 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6036 guard.disarm();
6037 Ok(vec![control_error_frame(
6038 &frame,
6039 "invalid_control_body",
6040 format!("expected module-control op '{expected}', got '{actual}'"),
6041 )?])
6042 }
6043 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6044 guard.disarm();
6045 Ok(vec![control_error_frame(
6046 &frame,
6047 "module_timeout",
6048 format!(
6049 "module_id '{module_id}' answered health.check after {:?}",
6050 self.health_probe_timeout
6051 ),
6052 )?])
6053 }
6054 Ok(Err(_)) => Ok(vec![control_error_frame(
6055 &frame,
6056 "target_unavailable",
6057 "health.check waiter was canceled before the module responded",
6058 )?]),
6059 Err(_) => Ok(vec![control_error_frame(
6060 &frame,
6061 "module_timeout",
6062 format!(
6063 "module_id '{module_id}' did not answer health.check within {:?}",
6064 self.health_probe_timeout
6065 ),
6066 )?]),
6067 }
6068 }
6069
6070 fn supervisor_status(
6071 &self,
6072 module_id: &str,
6073 corr: u64,
6074 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6075 self.supervisor
6076 .get(module_id)
6077 .map(|module| {
6078 let warming = module.is_warming_for_control("status").map_err(|err| {
6079 RouterError::backend(
6080 0,
6081 corr,
6082 format!(
6083 "failed to read supervisor warming state for module_id '{module_id}': {err}"
6084 ),
6085 )
6086 })?;
6087 module.status_for_control("status").map_err(|err| {
6088 RouterError::backend(
6089 0,
6090 corr,
6091 format!(
6092 "failed to read supervisor status for module_id '{module_id}': {err}"
6093 ),
6094 )
6095 }).map(|status| (status, warming))
6096 })
6097 .transpose()
6098 }
6099
6100 fn guard_module_control_op(
6101 &self,
6102 frame: &Frame,
6103 module_id: &str,
6104 op: &str,
6105 ) -> Result<Option<Frame>, RouterError> {
6106 if self.module_grants_op(module_id, op, frame.header.corr)? {
6107 return Ok(None);
6108 }
6109
6110 Ok(Some(control_error_frame(
6111 frame,
6112 "op_not_allowed",
6113 format!("module_id '{module_id}' did not grant control op '{op}'"),
6114 )?))
6115 }
6116
6117 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6118 let Some(registration) = self
6119 .registry
6120 .get_module(module_id)
6121 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6122 else {
6123 return Ok(false);
6124 };
6125 Ok(module_registration_grants_op(®istration.control_ops, op))
6126 }
6127
6128 fn handle_status_update(
6129 &self,
6130 endpoint: ModuleEndpointId,
6131 frame: Frame,
6132 ) -> Result<Vec<Frame>, RouterError> {
6133 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6134 Ok(update) => update,
6135 Err(err) => {
6136 if is_known_module_push_op(&frame.body) {
6141 return Ok(vec![control_error_frame(
6142 &frame,
6143 "invalid_control_body",
6144 format!("malformed module control push body: {err}"),
6145 )?]);
6146 }
6147 return Ok(Vec::new());
6148 }
6149 };
6150
6151 match update {
6152 ModuleControlPush::RouteStatus {
6153 route_channel,
6154 route_epoch,
6155 status,
6156 } => {
6157 self.forwarding
6158 .cache_status(endpoint, route_channel, route_epoch, status)
6159 .map_err(RouterError::Forwarding)?;
6160 }
6161 }
6162 Ok(Vec::new())
6163 }
6164
6165 fn handle_route_poll(
6166 &self,
6167 ctx: &RouteCtx,
6168 frame: Frame,
6169 route_channel: u16,
6170 route_epoch: u32,
6171 kind: PollKind,
6172 ) -> Result<Vec<Frame>, RouterError> {
6173 let snapshot = self
6174 .forwarding
6175 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6176 .map_err(RouterError::Forwarding)?;
6177 let response = match (kind, snapshot) {
6178 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6179 ClientControlResponse::RoutePoll {
6180 route_channel,
6181 route_epoch,
6182 status,
6183 live: None,
6184 }
6185 }
6186 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6187 route_channel,
6188 route_epoch,
6189 status: None,
6190 live: None,
6191 },
6192 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6193 let live = self
6219 .process_liveness
6220 .as_ref()
6221 .and_then(|source| source.process_live(&module_id))
6222 .unwrap_or(true);
6223 ClientControlResponse::RoutePoll {
6224 route_channel,
6225 route_epoch,
6226 status: None,
6227 live: Some(live),
6228 }
6229 }
6230 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6231 route_channel,
6232 route_epoch,
6233 status: None,
6234 live: Some(false),
6235 },
6236 };
6237
6238 Ok(vec![control_response_body_frame(
6239 &frame,
6240 &response,
6241 "ClientControlResponse::RoutePoll",
6242 )?])
6243 }
6244
6245 pub(crate) fn observe_module_control_completion(
6246 &self,
6247 completion: ModuleControlRpcCompletion,
6248 ) -> bool {
6249 match completion {
6250 ModuleControlRpcCompletion::Unknown => false,
6251 ModuleControlRpcCompletion::Settled => true,
6252 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6253 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6254 info!(
6255 module_id = %module_id,
6256 latency_ms,
6257 "late health.check answer proves the module is alive"
6258 );
6259 match self
6260 .supervisor
6261 .record_late_health_answer(&module_id, latency_ms)
6262 {
6263 Ok(true) => {}
6264 Ok(false) => debug!(
6265 module_id = %module_id,
6266 latency_ms,
6267 "late health.check answer has no active supervisor snapshot"
6268 ),
6269 Err(err) => warn!(
6270 module_id = %module_id,
6271 latency_ms,
6272 error = %err,
6273 "failed to record late health.check answer"
6274 ),
6275 }
6276 true
6277 }
6278 }
6279 }
6280
6281 fn refuse_to_end_module_connection_for_a_client(
6302 &self,
6303 module_connection_id: ConnectionId,
6304 corr: u64,
6305 err: ForwardingError,
6306 ) -> Result<(), RouterError> {
6307 if let ForwardingError::ConnectionClosing { connection_id } = err {
6308 if connection_id != module_connection_id {
6309 warn!(
6310 module_connection_id = module_connection_id.get(),
6311 client_connection_id = connection_id.get(),
6312 corr,
6313 "dropping a route.bind response for a closing client; the module connection keeps serving"
6314 );
6315 return Ok(());
6316 }
6317 }
6318 Err(RouterError::Forwarding(err))
6319 }
6320
6321 fn handle_module_relay_response(
6322 &self,
6323 connection_id: ConnectionId,
6324 frame: Frame,
6325 ) -> Result<Vec<Frame>, RouterError> {
6326 let mut secondary_error = None;
6327 let outcome = match frame.header.ty {
6328 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6329 Ok(probe) if probe.op == "route.bind" => {
6330 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6331 Ok(ModuleControlResponse::RouteBindAck {}) => {
6332 RouteBindRelayOutcome::Accepted
6333 }
6334 Ok(other) => {
6335 let message =
6336 format!("route.bind response carried unexpected body: {other:?}");
6337 secondary_error = Some(control_error_frame(
6338 &frame,
6339 "invalid_control_body",
6340 message.clone(),
6341 )?);
6342 RouteBindRelayOutcome::ModuleGone(message)
6343 }
6344 Err(err) => {
6345 let message = format!("malformed route.bind response body: {err}");
6346 secondary_error = Some(control_error_frame(
6347 &frame,
6348 "invalid_control_body",
6349 message.clone(),
6350 )?);
6351 RouteBindRelayOutcome::ModuleGone(message)
6352 }
6353 }
6354 }
6355 Ok(probe) => {
6356 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6357 {
6358 Ok(response) => ModuleControlRpcOutcome::Response(response),
6359 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6360 "malformed {} response body: {err}",
6361 probe.op
6362 )),
6363 };
6364 let completion = self
6365 .forwarding
6366 .complete_module_control_rpc(
6367 connection_id,
6368 frame.header.corr,
6369 Some(&probe.op),
6370 outcome,
6371 )
6372 .map_err(RouterError::Forwarding)?;
6373 if !self.observe_module_control_completion(completion) {
6374 debug!(
6375 connection_id = connection_id.get(),
6376 corr = frame.header.corr,
6377 op = %probe.op,
6378 "dropping late or unknown module-control RPC response"
6379 );
6380 }
6381 return Ok(Vec::new());
6382 }
6383 Err(err) => {
6384 if let Some(expected_op) = self
6385 .forwarding
6386 .pending_module_control_op(connection_id, frame.header.corr)
6387 .map_err(RouterError::Forwarding)?
6388 {
6389 let completion = self
6390 .forwarding
6391 .complete_module_control_rpc(
6392 connection_id,
6393 frame.header.corr,
6394 None,
6395 ModuleControlRpcOutcome::MalformedResponse(format!(
6396 "malformed {expected_op} response body: {err}"
6397 )),
6398 )
6399 .map_err(RouterError::Forwarding)?;
6400 if !self.observe_module_control_completion(completion) {
6401 debug!(
6402 connection_id = connection_id.get(),
6403 corr = frame.header.corr,
6404 "dropping late malformed module-control RPC response"
6405 );
6406 }
6407 return Ok(Vec::new());
6408 }
6409 let message = format!("malformed route.bind response body: {err}");
6410 secondary_error = Some(control_error_frame(
6411 &frame,
6412 "invalid_control_body",
6413 message.clone(),
6414 )?);
6415 RouteBindRelayOutcome::ModuleGone(message)
6416 }
6417 },
6418 FrameType::Error => {
6419 if self
6420 .forwarding
6421 .pending_module_control_op(connection_id, frame.header.corr)
6422 .map_err(RouterError::Forwarding)?
6423 .is_some()
6424 {
6425 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6426 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
6427 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6428 "malformed module-control ERROR body: {err}"
6429 )),
6430 };
6431 let completion = self
6432 .forwarding
6433 .complete_module_control_rpc(
6434 connection_id,
6435 frame.header.corr,
6436 None,
6437 outcome,
6438 )
6439 .map_err(RouterError::Forwarding)?;
6440 if !self.observe_module_control_completion(completion) {
6441 debug!(
6442 connection_id = connection_id.get(),
6443 corr = frame.header.corr,
6444 "dropping late or unknown module-control RPC error"
6445 );
6446 }
6447 return Ok(Vec::new());
6448 }
6449 match serde_json::from_slice::<ErrorBody>(&frame.body) {
6450 Ok(body) => RouteBindRelayOutcome::Rejected(body),
6451 Err(err) => {
6452 let message = format!("malformed route.bind ERROR body: {err}");
6453 secondary_error = Some(control_error_frame(
6454 &frame,
6455 "invalid_control_body",
6456 message.clone(),
6457 )?);
6458 RouteBindRelayOutcome::ModuleGone(message)
6459 }
6460 }
6461 }
6462 ty => {
6463 return Ok(vec![control_error_frame(
6464 &frame,
6465 "unsupported_control_frame",
6466 format!("unsupported module channel-0 frame {ty:?}"),
6467 )?])
6468 }
6469 };
6470
6471 let settled =
6472 self.forwarding
6473 .complete_pending_relay(connection_id, frame.header.corr, outcome);
6474 let completion = match settled {
6475 Ok(completion) => completion,
6476 Err(err) => {
6477 self.refuse_to_end_module_connection_for_a_client(
6478 connection_id,
6479 frame.header.corr,
6480 err,
6481 )?;
6482 return Ok(secondary_error.into_iter().collect());
6483 }
6484 };
6485 if let Some(target) = completion.abandoned.as_ref() {
6486 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6487 }
6488 if !completion.settled {
6489 debug!(
6490 connection_id = connection_id.get(),
6491 corr = frame.header.corr,
6492 frame_type = ?frame.header.ty,
6493 "dropping late or unknown route.bind relay response"
6494 );
6495 }
6496 Ok(secondary_error.into_iter().collect())
6497 }
6498
6499 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6500 debug!(connection_id = connection_id.get(), "handling GOODBYE");
6501 self.cleanup_connection_with_end_reason(
6505 connection_id,
6506 RegistrationEndReason::ExplicitGoodbye,
6507 )
6508 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6509 Ok(Vec::new())
6510 }
6511}
6512
6513impl Default for ControlHandler {
6514 fn default() -> Self {
6515 Self::new(Arc::new(Registry::default()))
6516 }
6517}
6518
6519impl crate::supervise::SwapPromotionObserver for ControlHandler {
6520 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6521 self.apply_registration_capabilities(registration);
6522 }
6523}
6524
6525fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6526 CapabilityRequirementStatus {
6527 consumer: status.consumer,
6528 capability: status.capability,
6529 need: match status.need {
6530 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6531 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6532 },
6533 verdict: status.verdict.as_str().to_string(),
6534 episode_seq: status.episode_seq,
6535 config_satisfiable: status.config_satisfiable,
6536 runtime_available: status.runtime_available,
6537 detail: status.detail,
6538 }
6539}
6540
6541fn append_capability_problem_detail(
6542 detail: Option<String>,
6543 capability_detail: Option<String>,
6544) -> Option<String> {
6545 match (detail, capability_detail) {
6546 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
6547 (Some(detail), None) => Some(detail),
6548 (None, Some(capability_detail)) => Some(capability_detail),
6549 (None, None) => None,
6550 }
6551}
6552
6553fn subc_ops() -> Vec<String> {
6554 SUBC_CONTROL_OPS
6555 .iter()
6556 .map(|op| (*op).to_string())
6557 .collect()
6558}
6559
6560fn module_subc_ops() -> Vec<String> {
6561 SUBC_CONTROL_OPS
6562 .iter()
6563 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
6564 .map(|op| (*op).to_string())
6565 .collect()
6566}
6567
6568#[cfg(test)]
6569fn module_baseline_control_ops() -> Vec<String> {
6570 MODULE_BASELINE_CONTROL_OPS
6571 .iter()
6572 .map(|op| (*op).to_string())
6573 .collect()
6574}
6575
6576fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
6577 let mut seen = HashSet::new();
6578 let mut effective = Vec::new();
6579 for op in MODULE_BASELINE_CONTROL_OPS {
6580 if seen.insert((*op).to_string()) {
6581 effective.push((*op).to_string());
6582 }
6583 }
6584 for op in declared.unwrap_or_default() {
6585 if seen.insert(op.clone()) {
6586 effective.push(op);
6587 }
6588 }
6589 effective
6590}
6591
6592fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
6593 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
6594}
6595
6596fn target_module_id(target: &RouteTarget) -> &str {
6597 match target {
6598 RouteTarget::ToolProvider { module_id }
6599 | RouteTarget::ManagementSurface { module_id }
6600 | RouteTarget::InternalService { module_id, .. } => module_id,
6601 }
6602}
6603
6604fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
6605 roles.iter().any(|role| match (target, role) {
6606 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
6607 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
6608 (
6609 RouteTarget::InternalService { service_id, .. },
6610 ProviderRole::InternalService {
6611 service_id: provided,
6612 ..
6613 },
6614 ) => service_id == provided,
6615 _ => false,
6616 })
6617}
6618
6619fn is_routable_role(role: &ProviderRole) -> bool {
6620 matches!(
6621 role,
6622 ProviderRole::ToolProvider { .. }
6623 | ProviderRole::ManagementSurface { .. }
6624 | ProviderRole::InternalService { .. }
6625 )
6626}
6627
6628#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6629enum ControlRequestBodyError {
6630 UnknownOp,
6631 InvalidBody,
6632}
6633
6634#[derive(Debug, Deserialize)]
6635struct ControlOpProbe {
6636 op: String,
6637}
6638
6639const MODULE_PUSH_OPS: &[&str] = &["route.status"];
6642
6643fn is_known_module_push_op(body: &[u8]) -> bool {
6644 serde_json::from_slice::<ControlOpProbe>(body)
6645 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
6646 .unwrap_or(false)
6647}
6648
6649fn is_known_module_request_op(body: &[u8]) -> bool {
6650 serde_json::from_slice::<ControlOpProbe>(body)
6651 .map(|probe| is_module_to_subc_op(&probe.op))
6652 .unwrap_or(false)
6653}
6654
6655fn is_module_to_subc_op(op: &str) -> bool {
6656 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
6657}
6658
6659fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
6660 debug!(
6661 op = %op,
6662 connection_id = connection_id.get(),
6663 corr,
6664 "control dispatch"
6665 );
6666}
6667
6668fn log_slow_control_dispatch(
6669 dispatch_started_at: Option<StdInstant>,
6670 op: &'static str,
6671 connection_id: ConnectionId,
6672 corr: u64,
6673) {
6674 let Some(dispatch_started_at) = dispatch_started_at else {
6675 return;
6676 };
6677 let elapsed = dispatch_started_at.elapsed();
6678 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
6679 warn!(
6680 op = %op,
6681 connection_id = connection_id.get(),
6682 corr,
6683 elapsed_ms = elapsed.as_millis() as u64,
6684 "slow control dispatch"
6685 );
6686 }
6687}
6688
6689#[cfg(test)]
6691type TestCallerLookup =
6692 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
6693
6694fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
6696 match request {
6697 ClientControlRequest::SupervisorRestart { module_id, .. } => {
6698 Some(PrivilegedOperation::Restart { module_id })
6699 }
6700 ClientControlRequest::SupervisorSwap { module_id, .. } => {
6701 Some(PrivilegedOperation::Swap { module_id })
6702 }
6703 ClientControlRequest::SupervisorReload { module_id } => {
6704 Some(PrivilegedOperation::Reload { module_id })
6705 }
6706 ClientControlRequest::SupervisorRescan { preview } => {
6707 if *preview {
6708 None
6709 } else {
6710 Some(PrivilegedOperation::Rescan)
6711 }
6712 }
6713 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
6714 Some(PrivilegedOperation::SetEnabled {
6715 module_id,
6716 enabled: *enabled,
6717 })
6718 }
6719 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
6720 Some(PrivilegedOperation::ReleaseReserved { module_id })
6721 }
6722 ClientControlRequest::ServerDescribe {}
6723 | ClientControlRequest::CatalogList { .. }
6724 | ClientControlRequest::RouteOpen { .. }
6725 | ClientControlRequest::RoutePoll { .. }
6726 | ClientControlRequest::SupervisorList {}
6727 | ClientControlRequest::SupervisorHealth {}
6728 | ClientControlRequest::SupervisorHealthProbe { .. }
6729 | ClientControlRequest::SupervisorRoutes { .. }
6730 | ClientControlRequest::SupervisorProvenance { .. }
6731 | ClientControlRequest::SupervisorStderrTail { .. }
6732 | ClientControlRequest::SupervisorTerminals { .. }
6733 | ClientControlRequest::SupervisorSpawnSnapshot {}
6734 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
6735 }
6736}
6737
6738fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
6739 match request {
6740 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
6741 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
6742 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
6743 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
6744 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
6745 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
6746 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
6747 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
6748 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
6749 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
6750 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
6751 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
6752 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
6753 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
6754 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
6755 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
6756 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
6757 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
6758 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
6759 }
6760}
6761
6762fn principal_label(principal: &Principal) -> String {
6763 match principal {
6764 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
6765 Principal::Direct => "direct".to_string(),
6766 other => format!("{other:?}"),
6767 }
6768}
6769
6770fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
6771 match request {
6772 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
6773 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
6774 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
6775 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
6776 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
6777 }
6778}
6779
6780fn parse_client_control_request(
6781 body: &[u8],
6782) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
6783 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
6784 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6785 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
6786 ControlRequestBodyError::InvalidBody
6787 }
6788 Ok(_) => ControlRequestBodyError::UnknownOp,
6789 Err(_) => ControlRequestBodyError::InvalidBody,
6790 };
6791 (err, classification)
6792 })
6793}
6794
6795fn parse_module_control_request_from_module(
6796 body: &[u8],
6797) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
6798 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
6799 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6800 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
6801 Ok(_) => ControlRequestBodyError::UnknownOp,
6802 Err(_) => ControlRequestBodyError::InvalidBody,
6803 };
6804 (err, classification)
6805 })
6806}
6807
6808#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
6809enum ProviderRoleKind {
6810 ToolProvider,
6811 PipelineStage,
6812 ManagementSurface,
6813 InternalService,
6814}
6815
6816fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
6817 match role {
6818 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
6819 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
6820 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
6821 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
6822 }
6823}
6824
6825fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
6826 roles.iter().map(provider_role_kind).collect()
6827}
6828
6829const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
6832
6833#[derive(Debug, Default, PartialEq, Eq)]
6835struct ScopeOutcomeCounts {
6836 created: usize,
6837 replaced: usize,
6838 updated: usize,
6839 unchanged: usize,
6840 refused: usize,
6841}
6842
6843impl ScopeOutcomeCounts {
6844 fn of(results: &[ScopeRecordResult]) -> Self {
6845 let mut counts = Self::default();
6846 for result in results {
6847 let slot = match result.outcome {
6848 ScopeRecordOutcome::Created => &mut counts.created,
6849 ScopeRecordOutcome::Replaced => &mut counts.replaced,
6850 ScopeRecordOutcome::Updated => &mut counts.updated,
6851 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
6852 ScopeRecordOutcome::Refused => &mut counts.refused,
6853 };
6854 *slot += 1;
6855 }
6856 counts
6857 }
6858}
6859
6860#[cfg(test)]
6861mod scope_outcome_count_tests {
6862 use super::*;
6863
6864 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
6865 ScopeRecordResult {
6866 scope_ref: "r".to_string(),
6867 scope_epoch: 1,
6868 outcome,
6869 code: None,
6870 message: None,
6871 version: None,
6872 parent_state: None,
6873 }
6874 }
6875
6876 #[test]
6879 fn every_outcome_is_counted_in_its_own_field() {
6880 let results = [
6881 result(ScopeRecordOutcome::Created),
6882 result(ScopeRecordOutcome::Created),
6883 result(ScopeRecordOutcome::Replaced),
6884 result(ScopeRecordOutcome::Updated),
6885 result(ScopeRecordOutcome::Unchanged),
6886 result(ScopeRecordOutcome::Refused),
6887 result(ScopeRecordOutcome::Refused),
6888 result(ScopeRecordOutcome::Refused),
6889 ];
6890 assert_eq!(
6891 ScopeOutcomeCounts::of(&results),
6892 ScopeOutcomeCounts {
6893 created: 2,
6894 replaced: 1,
6895 updated: 1,
6896 unchanged: 1,
6897 refused: 3,
6898 }
6899 );
6900 }
6901}
6902
6903fn capability_census_trigger(
6907 old: Option<&CapabilityDeclarations>,
6908 new: Option<&CapabilityDeclarations>,
6909) -> bool {
6910 let old_provides = old
6911 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
6912 .unwrap_or_default();
6913 let old_denies = old
6914 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
6915 .unwrap_or_default();
6916 let new = new.cloned().unwrap_or(CapabilityDeclarations {
6917 provides: Vec::new(),
6918 requires: Vec::new(),
6919 must_never_reach: Vec::new(),
6920 });
6921
6922 new.provides
6923 .iter()
6924 .any(|capability| !old_provides.contains(capability))
6925 || new
6926 .must_never_reach
6927 .iter()
6928 .any(|capability| !old_denies.contains(capability))
6929}
6930
6931fn denied_capability<'a>(
6934 opening_manifest: &'a ModuleManifest,
6935 target_manifest: &ModuleManifest,
6936) -> Option<&'a str> {
6937 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
6938 let target_capabilities = target_manifest.capabilities.as_ref()?;
6939 opening_capabilities
6940 .must_never_reach
6941 .iter()
6942 .find(|denied| {
6943 target_capabilities
6944 .provides
6945 .iter()
6946 .any(|provided| provided == *denied)
6947 })
6948 .map(String::as_str)
6949}
6950
6951fn catalog_update_frozen_field_message(
6952 registered: &ModuleManifest,
6953 provides: &[ProviderRole],
6954) -> Option<String> {
6955 let old_has_provides = !registered.provides.is_empty();
6956 let new_has_provides = !provides.is_empty();
6957 if old_has_provides != new_has_provides {
6958 return Some(format!(
6959 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
6960 registered.module_id
6961 ));
6962 }
6963
6964 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
6965 return Some(format!(
6966 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
6967 registered.module_id
6968 ));
6969 }
6970
6971 let registered_concurrency = manifest_concurrency(registered);
6972 let mut candidate = registered.clone();
6973 candidate.provides = provides.to_vec();
6974 let candidate_concurrency = manifest_concurrency(&candidate);
6975 if candidate_concurrency != registered_concurrency {
6976 return Some(format!(
6977 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
6978 registered.module_id, registered_concurrency, candidate_concurrency
6979 ));
6980 }
6981
6982 None
6985}
6986
6987fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
6988 manifest.provides.iter().any(is_routable_role)
6989}
6990
6991fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
6997 manifest
6998 .provides
6999 .iter()
7000 .find_map(|provider| match provider {
7001 ProviderRole::ToolProvider { concurrency, .. }
7002 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7003 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7004 })
7005 .unwrap_or(Concurrency::ModuleManaged)
7006}
7007
7008fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7015 let has_management_surface = manifest
7016 .provides
7017 .iter()
7018 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7019 if !has_management_surface {
7020 return false;
7021 }
7022 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7023 return false;
7024 };
7025 let Some(provides) = raw
7026 .get("manifest")
7027 .and_then(|manifest| manifest.get("provides"))
7028 .and_then(serde_json::Value::as_array)
7029 else {
7030 return false;
7031 };
7032 provides.iter().any(|role| {
7037 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7038 && role.get("concurrency").is_none()
7039 })
7040}
7041
7042fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7043 if peer_version != PROTOCOL_VERSION {
7044 return Err(format!(
7045 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7046 ));
7047 }
7048 Ok(PROTOCOL_VERSION)
7049}
7050
7051fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7052 Frame::build_with_version(
7053 response_version(frame),
7054 FrameType::Pong,
7055 frame.header.flags,
7056 0,
7057 0,
7058 frame.header.corr,
7059 Vec::new(),
7060 )
7061 .map_err(RouterError::FrameBuild)
7062}
7063
7064pub(crate) fn control_error_frame(
7065 frame: &Frame,
7066 code: &'static str,
7067 message: impl Into<String>,
7068) -> Result<Frame, RouterError> {
7069 control_error_body_frame(
7070 frame,
7071 ErrorBody {
7072 code: code.to_string(),
7073 message: message.into(),
7074 detail: None,
7075 },
7076 )
7077}
7078
7079fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7080 let body = serde_json::to_vec(&error).map_err(|err| {
7081 RouterError::backend(
7082 0,
7083 frame.header.corr,
7084 format!("failed to encode control ERROR: {err}"),
7085 )
7086 })?;
7087
7088 Frame::build_with_version(
7089 response_version(frame),
7090 FrameType::Error,
7091 control_flags(),
7092 0,
7093 0,
7094 frame.header.corr,
7095 body,
7096 )
7097 .map_err(RouterError::FrameBuild)
7098}
7099
7100fn control_response_body_frame<T: Serialize>(
7101 frame: &Frame,
7102 reply: &T,
7103 label: &'static str,
7104) -> Result<Frame, RouterError> {
7105 let body = serde_json::to_vec(reply).map_err(|err| {
7106 RouterError::backend(
7107 0,
7108 frame.header.corr,
7109 format!("failed to encode {label}: {err}"),
7110 )
7111 })?;
7112
7113 Frame::build_with_version(
7114 response_version(frame),
7115 FrameType::Response,
7116 control_flags(),
7117 0,
7118 0,
7119 frame.header.corr,
7120 body,
7121 )
7122 .map_err(RouterError::FrameBuild)
7123}
7124
7125fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7155 match err {
7156 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7157 ForwardingError::NoModuleConnection => "target_unavailable",
7158 ForwardingError::ModuleReloading { .. } => "module_reloading",
7159 ForwardingError::ClientRouteChannelExhausted { .. }
7160 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7161 ForwardingError::StaleModuleEndpoint
7162 | ForwardingError::UnknownReservation { .. }
7163 | ForwardingError::ConnectionClosing { .. }
7164 | ForwardingError::ClientEgressClosed { .. }
7165 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7166 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7169 ForwardingError::RelayCorrelationExhausted
7170 | ForwardingError::RouteOpenBuild(_)
7171 | ForwardingError::Poisoned => "forwarding_error",
7172 }
7173}
7174
7175fn response_version(frame: &Frame) -> u8 {
7176 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7177 frame.header.ver
7178 } else {
7179 PROTOCOL_VERSION
7180 }
7181}
7182
7183fn control_flags() -> Flags {
7184 Flags::new(false, Priority::Passive, false)
7185}
7186
7187fn send_goodbye_target_best_effort(
7192 counters: &DaemonCounters,
7193 target: &GoodbyeTarget,
7194 context: &'static str,
7195) {
7196 let Ok(frame) = Frame::build_with_version(
7197 target.negotiated_ver,
7198 FrameType::Goodbye,
7199 control_flags(),
7200 target.channel,
7201 target.epoch,
7202 0,
7203 Vec::new(),
7204 ) else {
7205 return;
7206 };
7207 crate::forwarding::send_module_route_goodbye(
7208 counters,
7209 &target.sink,
7210 frame,
7211 target.module_id.as_deref(),
7212 context,
7213 );
7214}
7215
7216pub(crate) fn send_route_control_pushes(
7217 forwarding: &ForwardingTable,
7218 routes: Vec<EndpointRoute>,
7219 push: ClientControlPush,
7220) {
7221 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7222 for route in routes {
7223 let target = route.goodbye_target;
7224 if let Some((existing, channels)) = targets
7225 .iter_mut()
7226 .find(|(existing, _)| existing.connection_id == target.connection_id)
7227 {
7228 debug_assert_eq!(
7229 existing.negotiated_ver, target.negotiated_ver,
7230 "one connection cannot negotiate multiple frame versions"
7231 );
7232 if !channels.contains(&target.channel) {
7233 channels.push(target.channel);
7234 }
7235 continue;
7236 }
7237 let channel = target.channel;
7238 targets.push((target, vec![channel]));
7239 }
7240 for (target, mut channels) in targets {
7241 channels.sort_unstable();
7242 let mut push = push.clone();
7243 match &mut push {
7244 ClientControlPush::RouteClosing {
7245 channels: covered, ..
7246 }
7247 | ClientControlPush::RouteClosed {
7248 channels: covered, ..
7249 } => *covered = channels,
7250 }
7251 let body = match serde_json::to_vec(&push) {
7252 Ok(body) => body,
7253 Err(err) => {
7254 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7255 continue;
7256 }
7257 };
7258 let frame = match Frame::build_with_version(
7259 target.negotiated_ver,
7260 FrameType::Push,
7261 control_flags(),
7262 0,
7263 0,
7264 0,
7265 body.clone(),
7266 ) {
7267 Ok(frame) => frame,
7268 Err(err) => {
7269 warn!(
7270 route_channel = target.channel,
7271 error = %err,
7272 "failed to build route lifecycle control PUSH frame"
7273 );
7274 continue;
7275 }
7276 };
7277 if let Err(err) = target.sink.try_send(frame) {
7278 if target.close_on_delivery_failure() {
7279 warn!(
7280 target_connection_id = target.connection_id.get(),
7281 route_channel = target.channel,
7282 error = %err,
7283 "route lifecycle control PUSH was not delivered to client; closing target connection"
7284 );
7285 let _ = forwarding.escalate_client_delivery_failure(
7286 target.connection_id,
7287 target.channel,
7288 target.epoch,
7289 CloseReason::new(
7290 "route_lifecycle_push_delivery_failed",
7291 format!(
7292 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7293 target.channel
7294 ),
7295 ),
7296 crate::forwarding::UndeliveredFrame {
7297 module_id: target.module_id.as_deref(),
7298 sink: &target.sink,
7299 },
7300 );
7301 }
7302 }
7303 }
7304}
7305
7306#[cfg(test)]
7307mod tests {
7308 mod activity_read {
7309 include!("activity_read_tests.rs");
7310 }
7311
7312 mod lifecycle_activity {
7313 include!("control_lifecycle_tests.rs");
7314 }
7315 #[cfg(unix)]
7316 #[tokio::test]
7317 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
7318 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
7319 let path = dir.join("subc.jsonc");
7320 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
7321 "program":"/bin/sleep","args":["60"],"protocol":"none",
7322 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
7323 }}}).to_string()).unwrap();
7324 let mut configured = crate::daemon_config::load(&path)
7325 .unwrap()
7326 .unwrap()
7327 .modules
7328 .pop()
7329 .unwrap();
7330 let registry = std::sync::Arc::new(crate::Registry::default());
7331 let handle = crate::SupervisorHandle::new();
7332 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
7333 .with_handle(handle.clone());
7334 let module = supervisor
7335 .supervise_configured_with_health(
7336 configured.module_spec(),
7337 true,
7338 configured.health.clone(),
7339 None,
7340 configured.restart,
7341 )
7342 .unwrap();
7343 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
7344 let before = module.status().unwrap().pid;
7345 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
7346 configured.health.cadence = std::time::Duration::from_secs(3600);
7347 let health_only = handler
7348 .reconcile_supervised_modules(
7349 &supervisor,
7350 vec![configured.clone()],
7351 false,
7352 Principal::Direct,
7353 )
7354 .await
7355 .unwrap();
7356 assert!(
7357 health_only.changed_pending_reload.is_empty(),
7358 "health policy is already applied live"
7359 );
7360 assert_eq!(module.status().unwrap().pid, before);
7361 assert_eq!(
7362 module.configuration().unwrap().1.http,
7363 configured.health.http
7364 );
7365 configured.args = vec!["61".into()];
7366 let launch = handler
7367 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
7368 .await
7369 .unwrap();
7370 assert_eq!(launch.changed_pending_reload, ["stock"]);
7371 assert_eq!(
7372 module.status().unwrap().pid,
7373 before,
7374 "a launch edit is stored until reload"
7375 );
7376 module.drain().await.unwrap();
7377 }
7378 use cortexkit_test_support::ScratchDir;
7379 use std::{
7380 collections::BTreeMap,
7381 fmt,
7382 path::PathBuf,
7383 sync::{Arc, Mutex},
7384 time::Duration,
7385 };
7386
7387 use serde_json::{json, Value};
7388 use subc_protocol::{
7389 manifest::{
7390 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
7391 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
7392 },
7393 session::HealthStatus,
7394 FrameType,
7395 };
7396
7397 use super::*;
7398 use crate::{
7399 forwarding::{DataRoute, DataRouteState},
7400 registry::ChannelState,
7401 router::FrameSink,
7402 stderr_tail::DEFAULT_MAX_LINE_BYTES,
7403 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
7404 RouteCtx, Router,
7405 };
7406 use tokio::{
7407 sync::mpsc,
7408 time::{sleep, Instant},
7409 };
7410 use tracing::{
7411 field::{Field, Visit},
7412 Event, Subscriber,
7413 };
7414 use tracing_subscriber::{layer::Context, prelude::*, Layer};
7415
7416 fn fake_aft_stub_path() -> PathBuf {
7435 let mut path = std::env::current_exe().expect("current_exe available in tests");
7436 path.pop(); path.pop(); path.push(if cfg!(windows) {
7439 "fake-aft-stub.exe"
7440 } else {
7441 "fake-aft-stub"
7442 });
7443 assert!(
7444 path.exists(),
7445 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
7446 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
7447 path.display()
7448 );
7449 path
7450 }
7451
7452 fn client_retries(code: &str) -> bool {
7457 subc_protocol::error_codes::is_retryable_route_open(code)
7458 }
7459
7460 #[test]
7471 fn retryability_of_forwarding_codes_matches_the_failure() {
7472 let transient = [
7475 ForwardingError::NoModuleConnection,
7476 ForwardingError::ModuleReloading {
7477 module_id: "m".into(),
7478 },
7479 ForwardingError::StaleModuleEndpoint,
7480 ForwardingError::UnknownReservation {
7481 client_channel: 1,
7482 module_channel: 1,
7483 },
7484 ForwardingError::ConnectionClosing {
7485 connection_id: ConnectionId::new(1),
7486 },
7487 ForwardingError::ClientEgressClosed {
7488 connection_id: ConnectionId::new(1),
7489 },
7490 ForwardingError::ModuleEgressUnavailable {
7491 connection_id: ConnectionId::new(1),
7492 },
7493 ];
7494 for err in transient {
7495 let code = forwarding_error_code(&err);
7496 assert!(
7497 client_retries(code),
7498 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
7499 );
7500 }
7501
7502 let permanent = [
7507 ForwardingError::ConnectionRoleConflict {
7508 connection_id: ConnectionId::new(1),
7509 },
7510 ForwardingError::ClientRouteChannelExhausted {
7511 connection_id: ConnectionId::new(1),
7512 },
7513 ForwardingError::ModuleRouteChannelExhausted {
7514 endpoint: ModuleEndpointId {
7515 connection_id: ConnectionId::new(1),
7516 generation: 1,
7517 },
7518 },
7519 ForwardingError::RelayCorrelationExhausted,
7520 ForwardingError::RouteOpenBuild("x".into()),
7521 ForwardingError::Poisoned,
7522 ];
7523 for err in permanent {
7524 let code = forwarding_error_code(&err);
7525 assert!(
7526 !client_retries(code),
7527 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
7528 );
7529 }
7530 }
7531
7532 #[tokio::test]
7560 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
7561 let handler = ControlHandler::default();
7562 let frame =
7563 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
7564
7565 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
7569 assert_eq!(
7570 stamped,
7571 Principal::Direct,
7572 "a caller that proved nothing must not be stamped as a supervised module"
7573 );
7574
7575 let forged = handler
7580 .route_open_principal(
7581 &frame,
7582 Some(ConsumerIdentity {
7583 module_id: "aft".to_string(),
7584 launch_nonce: "not-a-real-nonce".to_string(),
7585 }),
7586 )
7587 .unwrap();
7588 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
7589 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
7590 }
7591
7592 #[test]
7609 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
7610 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
7611 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
7612 let ClientControlRequest::RouteOpen {
7613 consumer_identity, ..
7614 } = parsed
7615 else {
7616 panic!("route.open body must parse as RouteOpen");
7617 };
7618 assert_eq!(
7619 consumer_identity,
7620 Some(ConsumerIdentity {
7621 module_id: "aft".to_string(),
7622 launch_nonce: "n".to_string(),
7623 }),
7624 "the wire field name must reach the value route_open_principal reads"
7625 );
7626 }
7627
7628 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
7629 ModuleManifest::builder(module_id, "0.1.0")
7630 .protocol_ver(protocol_ver)
7631 .provides(vec![ProviderRole::ToolProvider {
7632 tools: vec![Tool {
7633 name: "read".to_string(),
7634 description: None,
7635 execution_mode: ExecutionMode::Pure,
7636 schema: json!({"type": "object"}),
7637 }],
7638 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
7639 concurrency: Concurrency::ModuleManaged,
7640 emits_push: true,
7641 sub_supervises: true,
7642 }])
7643 .build()
7644 }
7645
7646 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
7647 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
7648 }
7649
7650 fn hello_frame_with_control_ops(
7651 module_id: &str,
7652 protocol_ver: u8,
7653 corr: u64,
7654 control_ops: Option<Vec<String>>,
7655 ) -> Frame {
7656 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
7657 }
7658
7659 fn hello_frame_with_nonce(
7660 module_id: &str,
7661 protocol_ver: u8,
7662 corr: u64,
7663 launch_nonce: Option<&str>,
7664 ) -> Frame {
7665 hello_frame_full(
7666 module_id,
7667 protocol_ver,
7668 corr,
7669 None,
7670 launch_nonce.map(ToOwned::to_owned),
7671 )
7672 }
7673
7674 fn hello_frame_full(
7675 module_id: &str,
7676 protocol_ver: u8,
7677 corr: u64,
7678 control_ops: Option<Vec<String>>,
7679 launch_nonce: Option<String>,
7680 ) -> Frame {
7681 let body = serde_json::to_vec(&ModuleHelloBody {
7682 manifest: manifest(module_id, protocol_ver),
7683 protocol_ver,
7684 control_ops,
7685 launch_nonce,
7686 })
7687 .unwrap();
7688 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7689 }
7690
7691 fn non_routable_hello_frame_with_control_ops(
7692 module_id: &str,
7693 corr: u64,
7694 control_ops: Option<Vec<String>>,
7695 ) -> Frame {
7696 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
7697 manifest.provides.clear();
7698 let body = serde_json::to_vec(&ModuleHelloBody {
7699 manifest,
7700 protocol_ver: PROTOCOL_VERSION,
7701 control_ops,
7702 launch_nonce: None,
7703 })
7704 .unwrap();
7705 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7706 }
7707
7708 fn capability_grammar_hello_frame(
7709 capabilities: Value,
7710 runtime_computed: Option<Value>,
7711 corr: u64,
7712 ) -> Frame {
7713 let mut body = serde_json::to_value(ModuleHelloBody {
7714 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
7715 protocol_ver: PROTOCOL_VERSION,
7716 control_ops: None,
7717 launch_nonce: None,
7718 })
7719 .expect("HELLO body serializes");
7720 body["manifest"]["capabilities"] = capabilities;
7721 if let Some(runtime_computed) = runtime_computed {
7722 body["runtime_computed"] = runtime_computed;
7723 }
7724 Frame::build(
7725 FrameType::Hello,
7726 control_flags(),
7727 0,
7728 0,
7729 corr,
7730 serde_json::to_vec(&body).expect("HELLO body reserializes"),
7731 )
7732 .expect("HELLO frame builds")
7733 }
7734
7735 fn channel_request(channel: u16, corr: u64) -> Frame {
7736 Frame::build(
7737 FrameType::Request,
7738 Flags::new(true, Priority::Interactive, false),
7739 channel,
7740 0,
7741 corr,
7742 b"opaque".to_vec(),
7743 )
7744 .unwrap()
7745 }
7746
7747 fn route_ctx(
7748 connection_id: ConnectionId,
7749 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
7750 let (tx, rx) = mpsc::channel(8);
7751 (
7752 RouteCtx {
7753 connection_id,
7754 egress: FrameSink::new(tx),
7755 },
7756 rx,
7757 )
7758 }
7759
7760 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
7761 serde_json::from_slice(&frame.body).unwrap()
7762 }
7763
7764 async fn hello_via_sink(
7769 handler: &ControlHandler,
7770 ctx: &RouteCtx,
7771 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7772 hello: Frame,
7773 ) -> Frame {
7774 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
7775 assert!(
7776 replies.is_empty(),
7777 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
7778 );
7779 let ack = rx
7780 .try_recv()
7781 .expect("HELLO_ACK is queued on the module sink")
7782 .frame;
7783 assert_eq!(ack.header.ty, FrameType::HelloAck);
7784 ack
7785 }
7786
7787 fn parse_error(frame: &Frame) -> Value {
7788 serde_json::from_slice(&frame.body).unwrap()
7789 }
7790
7791 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
7792 serde_json::from_slice(&frame.body).unwrap()
7793 }
7794
7795 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
7796 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
7797 route_channel,
7798 route_epoch: 0,
7799 kind,
7800 })
7801 .unwrap();
7802 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7803 }
7804
7805 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
7806 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
7807 module_id: module_id.to_string(),
7808 })
7809 .unwrap();
7810 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7811 }
7812
7813 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
7814 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
7815 }
7816
7817 fn route_open_frame_with_consumer_capabilities(
7818 corr: u64,
7819 module_id: &str,
7820 project_root: ScratchDir,
7821 consumer_capabilities: Option<Vec<String>>,
7822 ) -> Frame {
7823 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7824 target: RouteTarget::ToolProvider {
7825 module_id: module_id.to_string(),
7826 },
7827 identity: BindIdentity::new(
7828 project_root.path().to_path_buf(),
7829 "unit".to_string(),
7830 "session".to_string(),
7831 ),
7832 consumer_identity: None,
7833 consumer_capabilities,
7834 role_versions: None,
7835 admission_facts: None,
7836 scope: None,
7837 })
7838 .unwrap();
7839 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7840 }
7841
7842 fn route_open_frame_with_role_versions(
7843 corr: u64,
7844 module_id: &str,
7845 project_root: ScratchDir,
7846 role_versions: Option<BTreeMap<String, String>>,
7847 ) -> Frame {
7848 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7849 target: RouteTarget::ToolProvider {
7850 module_id: module_id.to_string(),
7851 },
7852 identity: BindIdentity::new(
7853 project_root.path().to_path_buf(),
7854 "unit".to_string(),
7855 format!("session-{corr}"),
7856 ),
7857 consumer_identity: None,
7858 consumer_capabilities: None,
7859 role_versions,
7860 admission_facts: None,
7861 scope: None,
7862 })
7863 .unwrap();
7864 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7865 }
7866
7867 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
7868 entries
7869 .iter()
7870 .map(|(role, version)| (role.to_string(), version.to_string()))
7871 .collect()
7872 }
7873
7874 fn route_open_frame_with_admission_facts(
7875 corr: u64,
7876 module_id: &str,
7877 project_root: ScratchDir,
7878 consumer_identity: Option<subc_control::ConsumerIdentity>,
7879 facts: Option<Value>,
7880 ) -> Frame {
7881 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7882 target: RouteTarget::ToolProvider {
7883 module_id: module_id.to_string(),
7884 },
7885 identity: BindIdentity::new(
7886 project_root.path().to_path_buf(),
7887 "unit".to_string(),
7888 format!("session-{corr}"),
7889 ),
7890 consumer_identity,
7891 consumer_capabilities: None,
7892 role_versions: None,
7893 admission_facts: facts,
7894 scope: None,
7895 })
7896 .unwrap();
7897 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7898 }
7899
7900 #[derive(Clone, Default)]
7901 struct EventCapture {
7902 events: Arc<Mutex<Vec<CapturedEvent>>>,
7903 }
7904
7905 #[derive(Clone, Debug)]
7906 struct CapturedEvent {
7907 target: String,
7908 level: tracing::Level,
7909 fields: BTreeMap<String, String>,
7910 }
7911
7912 impl EventCapture {
7913 fn events(&self) -> Vec<CapturedEvent> {
7914 self.events.lock().unwrap().clone()
7915 }
7916 }
7917
7918 impl<S> Layer<S> for EventCapture
7919 where
7920 S: Subscriber,
7921 {
7922 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
7923 let mut visitor = EventFieldVisitor::default();
7924 event.record(&mut visitor);
7925 self.events.lock().unwrap().push(CapturedEvent {
7926 target: event.metadata().target().to_string(),
7927 level: *event.metadata().level(),
7928 fields: visitor.fields,
7929 });
7930 }
7931 }
7932
7933 #[derive(Default)]
7934 struct EventFieldVisitor {
7935 fields: BTreeMap<String, String>,
7936 }
7937
7938 impl Visit for EventFieldVisitor {
7939 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
7940 self.fields
7941 .insert(field.name().to_string(), format!("{value:?}"));
7942 }
7943 }
7944
7945 fn health_response(corr: u64, status: HealthStatus) -> Frame {
7946 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
7947 status,
7948 detail: Some("warming".to_string()),
7949 metrics: Some(json!({"queue_depth": 3})),
7950 })
7951 .unwrap();
7952 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7953 }
7954
7955 fn route_bind_ack(corr: u64) -> Frame {
7956 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
7957 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7958 }
7959
7960 fn unique_project_root(label: &str) -> ScratchDir {
7961 ScratchDir::new(label)
7962 }
7963
7964 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
7965 match parse_route_poll(frame) {
7966 ClientControlResponse::RoutePoll {
7967 status: None,
7968 live: Some(live),
7969 ..
7970 } => assert_eq!(live, expected_live),
7971 other => panic!("unexpected route.poll response: {other:?}"),
7972 }
7973 }
7974
7975 fn bind_liveness_route(
7976 registry: &Registry,
7977 forwarding: &ForwardingTable,
7978 module_id: &str,
7979 ) -> (RouteCtx, u16, u32) {
7980 let module_connection = ConnectionId::new(101);
7981 let client_connection = ConnectionId::new(202);
7982 let registration = registry
7983 .register_with_control_ops(
7984 manifest(module_id, PROTOCOL_VERSION),
7985 PROTOCOL_VERSION,
7986 module_connection,
7987 module_baseline_control_ops(),
7988 )
7989 .unwrap();
7990 let (module_tx, _module_rx) = mpsc::channel(8);
7991 let endpoint = forwarding
7992 .register_module_connection(
7993 module_connection,
7994 module_id.to_string(),
7995 PROTOCOL_VERSION,
7996 manifest_concurrency(®istration.manifest),
7997 FrameSink::new(module_tx),
7998 )
7999 .unwrap();
8000 let (client_ctx, _client_rx) = route_ctx(client_connection);
8001 let pending = forwarding
8002 .begin_route_bind_relay_for_test(
8003 client_connection,
8004 client_ctx.egress.clone(),
8005 1,
8006 module_id,
8007 )
8008 .unwrap();
8009 assert_eq!(pending.endpoint, endpoint);
8010 let route_channel = pending.client_channel;
8011 let route_epoch = pending.client_epoch;
8012 forwarding
8013 .complete_pending_relay(
8014 module_connection,
8015 pending.corr,
8016 RouteBindRelayOutcome::Accepted,
8017 )
8018 .unwrap();
8019 (client_ctx, route_channel, route_epoch)
8020 }
8021
8022 struct FakeProcessLiveness {
8023 live: Option<bool>,
8024 }
8025
8026 impl ModuleProcessLiveness for FakeProcessLiveness {
8027 fn process_live(&self, _module_id: &str) -> Option<bool> {
8028 self.live
8029 }
8030 }
8031
8032 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8033 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8034 {
8035 let registry = Arc::new(Registry::default());
8036 let supervisor_handle = SupervisorHandle::new();
8037 let supervisor = Supervisor::new_for_test(
8038 Arc::clone(®istry),
8039 RestartPolicy::new(1, Duration::from_millis(10)),
8040 )
8041 .with_handle(supervisor_handle.clone());
8042 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8043 let module = supervisor
8044 .spawn(ModuleSpec {
8045 module_id: "stderr-tail-wire".to_string(),
8046 program: fake_aft_stub_path(),
8047 args: Vec::new(),
8048 env: vec![
8049 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8050 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8051 ],
8052 reserved: false,
8053 reserved_prefixes: Vec::new(),
8054 protocol: ModuleProtocol::Subc,
8055 overlap: Default::default(),
8056 })
8057 .unwrap();
8058
8059 let deadline = Instant::now() + Duration::from_secs(5);
8060 loop {
8061 let tail = module.stderr_tail(None, None);
8062 if tail
8063 .entries
8064 .iter()
8065 .any(|entry| matches!(entry, TailEntry::ProcessStart))
8066 && tail.entries.iter().any(|entry| {
8067 matches!(
8068 entry,
8069 TailEntry::Line {
8070 truncated: true,
8071 ..
8072 }
8073 )
8074 })
8075 {
8076 break;
8077 }
8078 assert!(
8079 Instant::now() < deadline,
8080 "module did not produce a truncated line and restart boundary: {tail:?}"
8081 );
8082 sleep(Duration::from_millis(10)).await;
8083 }
8084
8085 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8086 let request = ClientControlRequest::SupervisorStderrTail {
8087 module_id: "stderr-tail-wire".to_string(),
8088 max_lines: None,
8089 max_bytes: None,
8090 };
8091 let frame = Frame::build(
8092 FrameType::Request,
8093 control_flags(),
8094 0,
8095 0,
8096 1,
8097 serde_json::to_vec(&request).unwrap(),
8098 )
8099 .unwrap();
8100 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8101 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8102 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8103 serde_json::from_slice(&responses[0].body).unwrap()
8104 else {
8105 panic!("expected supervisor.stderr_tail response");
8106 };
8107
8108 assert!(
8109 tail.entries
8110 .iter()
8111 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8112 "the control response lost the restart boundary"
8113 );
8114 let Some(StderrTailEntry::Line {
8115 text,
8116 truncated,
8117 at_ms,
8118 }) = tail.entries.iter().find(|entry| {
8119 matches!(
8120 entry,
8121 StderrTailEntry::Line {
8122 truncated: true,
8123 ..
8124 }
8125 )
8126 })
8127 else {
8128 panic!("the control response lost the truncated line");
8129 };
8130 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8131 assert!(*truncated);
8132 assert!(
8133 at_ms.is_some(),
8134 "the control response lost the line's capture time"
8135 );
8136 }
8137
8138 #[tokio::test(flavor = "current_thread")]
8143 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8144 let dir = ScratchDir::new("terminals-off-worker");
8145 let journal_path = dir.join("terminals.jsonl");
8146 let registry = Arc::new(Registry::default());
8147 let supervisor_handle = SupervisorHandle::new();
8148 let supervisor =
8149 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8150 .with_handle(supervisor_handle.clone())
8151 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8152 let module = supervisor
8153 .spawn(ModuleSpec {
8154 module_id: "terminal-off-worker".to_string(),
8155 program: fake_aft_stub_path(),
8156 args: Vec::new(),
8157 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8158 reserved: false,
8159 reserved_prefixes: Vec::new(),
8160 protocol: ModuleProtocol::Subc,
8161 overlap: Default::default(),
8162 })
8163 .unwrap();
8164 let deadline = Instant::now() + Duration::from_secs(5);
8165 while module.terminal_history().entries.len() != 2 {
8166 assert!(Instant::now() < deadline, "module did not record two exits");
8167 sleep(Duration::from_millis(10)).await;
8168 }
8169
8170 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8171 let handler =
8172 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
8173 let frame = Frame::build(
8174 FrameType::Request,
8175 control_flags(),
8176 0,
8177 0,
8178 1,
8179 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8180 module_id: "terminal-off-worker".to_string(),
8181 })
8182 .unwrap(),
8183 )
8184 .unwrap();
8185 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8186 let spawned_at = std::time::Instant::now();
8187 let read = tokio::spawn({
8188 let handler = Arc::clone(&handler);
8189 async move { handler.handle_control_frame(&ctx, frame).await }
8190 });
8191 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8194 .await
8195 .unwrap()
8196 .expect("the history read reached its pause");
8197 let elapsed = spawned_at.elapsed();
8198 assert!(
8199 elapsed < Duration::from_secs(2) && !read.is_finished(),
8200 "this task could not run while the history read was paused \
8201 (resumed after {elapsed:?}, read finished: {})",
8202 read.is_finished()
8203 );
8204
8205 drop(release);
8206 let responses = read.await.unwrap().unwrap();
8207 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8208 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8209 panic!("expected supervisor.terminals response");
8210 };
8211 assert_eq!(terminals.entries.len(), 2);
8212 assert_eq!(terminals.journal_skipped_lines, 0);
8213 assert_eq!(terminals.journal_read_errors, 0);
8214 }
8215
8216 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8217 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8218 let registry = Arc::new(Registry::default());
8219 let supervisor_handle = SupervisorHandle::new();
8220 let supervisor =
8221 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8222 .with_handle(supervisor_handle.clone());
8223 let module = supervisor
8224 .spawn(ModuleSpec {
8225 module_id: "terminal-golden".to_string(),
8226 program: fake_aft_stub_path(),
8227 args: Vec::new(),
8228 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8229 reserved: false,
8230 reserved_prefixes: Vec::new(),
8231 protocol: ModuleProtocol::Subc,
8232 overlap: Default::default(),
8233 })
8234 .unwrap();
8235
8236 let deadline = Instant::now() + Duration::from_secs(5);
8237 while module.terminal_history().entries.len() != 2 {
8238 assert!(
8239 Instant::now() < deadline,
8240 "module did not retain two terminal exits: {:?}",
8241 module.terminal_history()
8242 );
8243 sleep(Duration::from_millis(10)).await;
8244 }
8245
8246 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8247 let request = ClientControlRequest::SupervisorTerminals {
8248 module_id: "terminal-golden".to_string(),
8249 };
8250 let frame = Frame::build(
8251 FrameType::Request,
8252 control_flags(),
8253 0,
8254 0,
8255 1,
8256 serde_json::to_vec(&request).unwrap(),
8257 )
8258 .unwrap();
8259 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8260 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8261 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8262 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8263 panic!("expected supervisor.terminals response");
8264 };
8265 assert_eq!(terminals.entries.len(), 2);
8266 assert_eq!(terminals.dropped, 0);
8267
8268 let mut rendered = serde_json::to_value(response).unwrap();
8269 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
8272 for (index, entry) in rendered["entries"]
8273 .as_array_mut()
8274 .expect("terminal response entries array")
8275 .iter_mut()
8276 .enumerate()
8277 {
8278 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
8279 }
8280
8281 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8282 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
8283 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
8284 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8285 std::fs::write(&golden_path, &serialized).unwrap();
8286 }
8287 let expected: Value =
8288 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
8289 assert_eq!(rendered, expected);
8290 }
8291
8292 #[test]
8293 fn hello_registers_manifest_and_returns_ack() {
8294 let registry = Arc::new(Registry::default());
8295 let handler = ControlHandler::new(Arc::clone(®istry));
8296 let conn = ConnectionId::new(1);
8297
8298 let responses = handler
8299 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
8300 .unwrap();
8301
8302 assert_eq!(responses.len(), 1);
8303 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8304 assert_eq!(responses[0].header.channel, 0);
8305 assert_eq!(responses[0].header.corr, 7);
8306 let ack = parse_ack(&responses[0]);
8307 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8308 assert!(ack
8309 .subc_capabilities
8310 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
8311 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
8312 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
8313 assert!(ack
8314 .subc_ops
8315 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
8316 assert!(ack
8317 .subc_ops
8318 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
8319
8320 let registration = registry.get_module("aft").unwrap().unwrap();
8321 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
8322 assert_eq!(registration.state, ChannelState::Active);
8323 assert_eq!(registration.connection_id, conn);
8324 assert_eq!(registration.control_ops, module_baseline_control_ops());
8325 }
8326
8327 #[test]
8328 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
8329 let invalid_identifiers = [
8330 ("case_change", "credentials-Provider/v1"),
8331 ("leading_zero", "credentials-provider/v01"),
8332 ("trailing_hyphen", "credentials-provider-/v1"),
8333 ("consecutive_hyphens", "credentials--provider/v1"),
8334 ("uppercase", "Credentials-provider/v1"),
8335 ("missing_v", "credentials-provider/1"),
8336 ("whitespace", "credentials provider/v1"),
8337 ("zero_version", "credentials-provider/v0"),
8338 ("out_of_range_version", "credentials-provider/v4294967296"),
8339 (
8340 "overlength_name",
8341 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
8342 ),
8343 ];
8344 let mut cases = invalid_identifiers
8345 .into_iter()
8346 .map(|(name, identifier)| {
8347 (
8348 format!("identifier_{name}"),
8349 "capabilities.provides[0]".to_string(),
8350 identifier.to_string(),
8351 json!({ "provides": [identifier] }),
8352 None,
8353 )
8354 })
8355 .collect::<Vec<_>>();
8356 cases.extend([
8357 (
8358 "unknown_need".to_string(),
8359 "capabilities.requires[0].need".to_string(),
8360 "deferred".to_string(),
8361 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
8362 None,
8363 ),
8364 (
8365 "duplicate_provides".to_string(),
8366 "capabilities.provides[1]".to_string(),
8367 "credentials-provider/v1".to_string(),
8368 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
8369 None,
8370 ),
8371 (
8372 "duplicate_must_never_reach".to_string(),
8373 "capabilities.must_never_reach[1]".to_string(),
8374 "credentials-provider/v1".to_string(),
8375 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
8376 None,
8377 ),
8378 (
8379 "duplicate_requires_same_need".to_string(),
8380 "capabilities.requires[1]".to_string(),
8381 "credentials-provider/v1".to_string(),
8382 json!({ "requires": [
8383 { "capability": "credentials-provider/v1", "need": "required" },
8384 { "capability": "credentials-provider/v1", "need": "required" }
8385 ] }),
8386 None,
8387 ),
8388 (
8389 "duplicate_requires_conflicting_need".to_string(),
8390 "capabilities.requires[1]".to_string(),
8391 "credentials-provider/v1".to_string(),
8392 json!({ "requires": [
8393 { "capability": "credentials-provider/v1", "need": "required" },
8394 { "capability": "credentials-provider/v1", "need": "optional" }
8395 ] }),
8396 None,
8397 ),
8398 (
8399 "capabilities_root_pointer".to_string(),
8400 "runtime_computed[0]".to_string(),
8401 "/capabilities".to_string(),
8402 json!({}),
8403 Some(json!(["/capabilities"])),
8404 ),
8405 (
8406 "capabilities_descendant_pointer".to_string(),
8407 "runtime_computed[0]".to_string(),
8408 "/capabilities/provides".to_string(),
8409 json!({}),
8410 Some(json!(["/capabilities/provides"])),
8411 ),
8412 (
8413 "malformed_pointer_without_leading_slash".to_string(),
8414 "runtime_computed[0]".to_string(),
8415 "capabilities".to_string(),
8416 json!({}),
8417 Some(json!(["capabilities"])),
8418 ),
8419 (
8420 "malformed_pointer_escape".to_string(),
8421 "runtime_computed[0]".to_string(),
8422 "/roles/~2/tools".to_string(),
8423 json!({}),
8424 Some(json!(["/roles/~2/tools"])),
8425 ),
8426 (
8427 "unknown_capabilities_field".to_string(),
8428 "capabilities.future".to_string(),
8429 "<array>".to_string(),
8430 json!({ "future": [] }),
8431 None,
8432 ),
8433 ]);
8434
8435 for (index, (name, field, value, capabilities, runtime_computed)) in
8436 cases.into_iter().enumerate()
8437 {
8438 let registry = Arc::new(Registry::default());
8439 let handler = ControlHandler::new(Arc::clone(®istry));
8440 let response = handler
8441 .handle_control(
8442 ConnectionId::new((index + 1) as u64),
8443 capability_grammar_hello_frame(
8444 capabilities,
8445 runtime_computed,
8446 index as u64 + 1,
8447 ),
8448 )
8449 .expect("invalid HELLO returns a refusal");
8450
8451 assert_eq!(response.len(), 1, "{name} must emit one refusal");
8452 let error = parse_error(&response[0]);
8453 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
8454 let message = error["message"]
8455 .as_str()
8456 .expect("error message is a string");
8457 assert!(
8458 message.contains(&field),
8459 "{name}: field missing from {message}"
8460 );
8461 assert!(
8462 message.contains(&value),
8463 "{name}: value missing from {message}"
8464 );
8465 assert_eq!(
8466 registry
8467 .active_registration_count()
8468 .expect("registry reads"),
8469 0,
8470 "{name}: refused HELLO must not create a catalog entry"
8471 );
8472 }
8473 }
8474
8475 #[test]
8476 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
8477 let registry = Arc::new(Registry::default());
8478 let handler = ControlHandler::new(Arc::clone(®istry));
8479 let capabilities = json!({
8480 "provides": ["credentials-provider/v1"],
8481 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
8482 "must_never_reach": ["federation-transport/v1"]
8483 });
8484 let response = handler
8485 .handle_control(
8486 ConnectionId::new(99),
8487 capability_grammar_hello_frame(
8488 capabilities.clone(),
8489 Some(json!(["/roles/0/tools"])),
8490 99,
8491 ),
8492 )
8493 .expect("valid HELLO registers");
8494 assert_eq!(response[0].header.ty, FrameType::HelloAck);
8495
8496 let request = Frame::build(
8497 FrameType::Request,
8498 control_flags(),
8499 0,
8500 0,
8501 100,
8502 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8503 .expect("catalog request serializes"),
8504 )
8505 .expect("catalog request frame builds");
8506 let response = handler
8507 .handle_catalog_list(request, None)
8508 .expect("catalog list succeeds");
8509 let ClientControlResponse::CatalogList { modules, .. } =
8510 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
8511 else {
8512 panic!("catalog request must return catalog.list");
8513 };
8514 assert_eq!(modules.len(), 1);
8515 assert_eq!(
8516 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
8517 capabilities
8518 );
8519 }
8520
8521 #[test]
8522 fn catalog_list_mirrors_management_operation_description() {
8523 let registry = Arc::new(Registry::default());
8524 let handler = ControlHandler::new(Arc::clone(®istry));
8525 let description = "List managed records and return their identifiers and metadata.";
8526 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
8527 manifest.provides = vec![ProviderRole::ManagementSurface {
8528 operations: vec![ManagementOperation {
8529 name: "records.list".to_string(),
8530 kind: ManagementOperationKind::Query,
8531 description: Some(description.to_string()),
8532 }],
8533 config_schema: json!({"type": "object"}),
8534 observability: vec![ObservabilitySurface {
8535 name: "records.stats".to_string(),
8536 kind: ObservabilityKind::Snapshot,
8537 }],
8538 identity_scope: vec![IdentityScope::Project],
8539 concurrency: Concurrency::ModuleManaged,
8540 }];
8541 registry
8542 .register_with_control_ops(
8543 manifest,
8544 PROTOCOL_VERSION,
8545 ConnectionId::new(99),
8546 Vec::new(),
8547 )
8548 .expect("described management manifest registers");
8549
8550 let request = Frame::build(
8551 FrameType::Request,
8552 control_flags(),
8553 0,
8554 0,
8555 100,
8556 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8557 .expect("catalog request serializes"),
8558 )
8559 .expect("catalog request frame builds");
8560 let response = handler
8561 .handle_catalog_list(request, None)
8562 .expect("catalog list succeeds");
8563 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8564 assert_eq!(
8565 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
8566 "catalog.list must preserve the declared operation description verbatim"
8567 );
8568 }
8569
8570 #[test]
8571 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
8572 let registry = Arc::new(Registry::default());
8573 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8574 [("vault".to_string(), true), ("squatter".to_string(), true)],
8575 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8576 );
8577 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
8578 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8579 provides: vec!["credentials-provider/v1".to_string()],
8580 requires: Vec::new(),
8581 must_never_reach: Vec::new(),
8582 });
8583 let frame = Frame::build(
8584 FrameType::Hello,
8585 control_flags(),
8586 0,
8587 0,
8588 77,
8589 serde_json::to_vec(&ModuleHelloBody {
8590 manifest: squatter,
8591 protocol_ver: PROTOCOL_VERSION,
8592 control_ops: None,
8593 launch_nonce: None,
8594 })
8595 .expect("HELLO serializes"),
8596 )
8597 .expect("HELLO frame builds");
8598 let response = handler
8599 .handle_control(ConnectionId::new(77), frame)
8600 .expect("reserved claim receives a typed refusal");
8601 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8602 assert_eq!(
8603 registry
8604 .active_registration_count()
8605 .expect("registry reads"),
8606 0,
8607 "a reserved capability refusal must not leave a catalog entry"
8608 );
8609 }
8610
8611 #[test]
8612 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
8613 for settlement in ["timeout", "inconclusive", "drop"] {
8614 let breakers = RouteBindBreakers::default();
8615 let RouteBindAdmission::Admitted {
8616 guard: mut old,
8617 probe: false,
8618 } = breakers.admit("prov")
8619 else {
8620 panic!("ordinary relay admitted")
8621 };
8622 let RouteBindAdmission::Admitted {
8623 guard: mut opener, ..
8624 } = breakers.admit("prov")
8625 else {
8626 panic!("second relay admitted")
8627 };
8628 assert!(
8629 !opener
8630 .record_timeout(1, Duration::ZERO)
8631 .unwrap()
8632 .reopened_after_probe
8633 );
8634 let RouteBindAdmission::Admitted {
8635 guard: mut probe,
8636 probe: true,
8637 } = breakers.admit("prov")
8638 else {
8639 panic!("one cooldown probe admitted")
8640 };
8641 match settlement {
8642 "timeout" => assert!(
8643 !old.record_timeout(1, Duration::ZERO)
8644 .unwrap()
8645 .reopened_after_probe
8646 ),
8647 "inconclusive" => old.record_inconclusive(),
8648 "drop" => drop(old),
8649 _ => unreachable!(),
8650 }
8651 assert!(
8652 matches!(
8653 breakers.admit("prov"),
8654 RouteBindAdmission::Refused {
8655 probe_in_flight: true,
8656 ..
8657 }
8658 ),
8659 "{settlement} of a pre-open relay cannot release the real probe"
8660 );
8661 assert!(
8662 probe
8663 .record_timeout(1, Duration::ZERO)
8664 .unwrap()
8665 .reopened_after_probe
8666 );
8667 assert!(matches!(
8668 breakers.admit("prov"),
8669 RouteBindAdmission::Admitted { probe: true, .. }
8670 ));
8671 }
8672 let breakers = RouteBindBreakers::default();
8673 let admit = || match breakers.admit("prov") {
8674 RouteBindAdmission::Admitted { guard, .. } => guard,
8675 _ => panic!("relay admitted"),
8676 };
8677 admit().record_timeout(1, Duration::ZERO);
8678 let mut old_probe = admit();
8679 breakers.reset_for_new_module_connection("prov");
8680 admit().record_timeout(1, Duration::ZERO);
8681 let _new_probe = admit();
8682 old_probe.record_inconclusive();
8683 assert!(matches!(
8684 breakers.admit("prov"),
8685 RouteBindAdmission::Refused {
8686 probe_in_flight: true,
8687 ..
8688 }
8689 ));
8690 }
8691
8692 #[tokio::test]
8693 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
8694 for candidate in [false, true] {
8695 let registry = Arc::new(Registry::default());
8696 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8697 [("vault".to_string(), true), ("squatter".to_string(), true)],
8698 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8699 );
8700 let conn = ConnectionId::new(77);
8701 let (ctx, mut rx) = route_ctx(conn);
8702 let initial = capability_manifest("squatter", &[], &[]);
8703 if candidate {
8704 registry
8705 .register_candidate_with_control_ops(
8706 initial.clone(),
8707 PROTOCOL_VERSION,
8708 conn,
8709 module_baseline_control_ops(),
8710 )
8711 .unwrap();
8712 handler
8713 .forwarding
8714 .register_candidate_module_connection(
8715 conn,
8716 "squatter".to_string(),
8717 PROTOCOL_VERSION,
8718 manifest_concurrency(&initial),
8719 ctx.egress.clone(),
8720 )
8721 .unwrap();
8722 } else {
8723 hello_via_sink(
8724 &handler,
8725 &ctx,
8726 &mut rx,
8727 hello_frame_with_manifest(initial.clone(), 1),
8728 )
8729 .await;
8730 }
8731 let response = handler
8732 .handle_control_frame(
8733 &ctx,
8734 catalog_update_with_capabilities_frame(
8735 2,
8736 capability_manifest("squatter", &["credentials-provider/v1"], &[])
8737 .capabilities
8738 .unwrap(),
8739 ),
8740 )
8741 .await
8742 .unwrap();
8743 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8744 assert_eq!(
8745 registry
8746 .get_module_by_connection(conn)
8747 .unwrap()
8748 .unwrap()
8749 .manifest,
8750 initial
8751 );
8752 }
8753 }
8754
8755 #[test]
8756 fn server_describe_surfaces_required_capability_verdict_fields() {
8757 let registry = Arc::new(Registry::default());
8758 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8759 [
8760 ("consumer".to_string(), true),
8761 ("provider".to_string(), false),
8762 ],
8763 BTreeMap::new(),
8764 );
8765 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
8766 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8767 provides: Vec::new(),
8768 requires: vec![subc_protocol::manifest::CapabilityRequirement {
8769 capability: "credentials-provider/v1".to_string(),
8770 need: subc_protocol::manifest::CapabilityNeed::Required,
8771 }],
8772 must_never_reach: Vec::new(),
8773 });
8774 let hello = Frame::build(
8775 FrameType::Hello,
8776 control_flags(),
8777 0,
8778 0,
8779 78,
8780 serde_json::to_vec(&ModuleHelloBody {
8781 manifest: consumer,
8782 protocol_ver: PROTOCOL_VERSION,
8783 control_ops: None,
8784 launch_nonce: None,
8785 })
8786 .expect("HELLO serializes"),
8787 )
8788 .expect("HELLO frame builds");
8789 handler
8790 .handle_control(ConnectionId::new(78), hello)
8791 .expect("consumer registers");
8792 let describe = Frame::build(
8793 FrameType::Request,
8794 control_flags(),
8795 0,
8796 0,
8797 79,
8798 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
8799 .expect("request serializes"),
8800 )
8801 .expect("describe frame builds");
8802 let response = handler
8803 .handle_server_describe(describe)
8804 .expect("server.describe succeeds");
8805 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
8806 let requirement = &rendered["capability_requirements"][0];
8807 assert_eq!(requirement["consumer"], "consumer");
8808 assert_eq!(requirement["verdict"], "never_provided");
8809 assert_eq!(requirement["episode_seq"], 1);
8810 assert_eq!(requirement["config_satisfiable"], false);
8811 assert_eq!(requirement["runtime_available"], false);
8812 assert!(requirement["detail"]
8813 .as_str()
8814 .expect("detail string")
8815 .contains("credentials-provider/v1"));
8816 }
8817
8818 #[test]
8819 fn catalog_list_omits_capabilities_for_legacy_manifest() {
8820 let registry = Arc::new(Registry::default());
8821 let handler = ControlHandler::new(Arc::clone(®istry));
8822 let hello = handler
8823 .handle_control(
8824 ConnectionId::new(101),
8825 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
8826 )
8827 .expect("legacy HELLO registers");
8828 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
8829
8830 let request = Frame::build(
8831 FrameType::Request,
8832 control_flags(),
8833 0,
8834 0,
8835 102,
8836 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8837 .expect("catalog request serializes"),
8838 )
8839 .expect("catalog request frame builds");
8840 let response = handler
8841 .handle_catalog_list(request, None)
8842 .expect("catalog list succeeds");
8843 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8844 assert!(
8845 body["modules"][0].get("capabilities").is_none(),
8846 "legacy manifest must retain an absent capabilities field on catalog.list"
8847 );
8848 }
8849
8850 #[test]
8851 fn hello_ack_omits_storage_when_no_storage_config() {
8852 let registry = Arc::new(Registry::default());
8853 let handler = ControlHandler::new(Arc::clone(®istry));
8854 let responses = handler
8855 .handle_control(
8856 ConnectionId::new(1),
8857 hello_frame("aft", PROTOCOL_VERSION, 7),
8858 )
8859 .unwrap();
8860 let ack = parse_ack(&responses[0]);
8861 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
8862 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
8863 }
8864
8865 #[tokio::test]
8866 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
8867 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
8868 let registry = Arc::new(Registry::default());
8869 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
8870 let responses = handler
8871 .handle_control(
8872 ConnectionId::new(1),
8873 hello_frame("aft", PROTOCOL_VERSION, 7),
8874 )
8875 .unwrap();
8876 let ack = parse_ack(&responses[0]);
8877 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
8878
8879 let described = handler
8880 .handle_control_frame(
8881 &route_ctx(ConnectionId::new(2)).0,
8882 Frame::build(
8883 FrameType::Request,
8884 control_flags(),
8885 0,
8886 0,
8887 9,
8888 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8889 )
8890 .unwrap(),
8891 )
8892 .await
8893 .unwrap();
8894 let ClientControlResponse::ServerDescribe { machine_id, .. } =
8895 serde_json::from_slice(&described[0].body).unwrap()
8896 else {
8897 panic!("server.describe answered with another shape");
8898 };
8899 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
8900 }
8901
8902 #[test]
8903 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
8904 let registry = Arc::new(Registry::default());
8907 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
8908 crate::daemon_config::StorageConfig::Sqlite {
8909 data_home: std::path::PathBuf::from("/data"),
8910 },
8911 ));
8912
8913 let responses = handler
8914 .handle_control(
8915 ConnectionId::new(1),
8916 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
8917 )
8918 .unwrap();
8919 let ack = parse_ack(&responses[0]);
8920 assert_eq!(
8921 ack.storage,
8922 Some(serde_json::json!({
8923 "module_id": "alfonso-routing",
8924 "storage_namespace": "default",
8925 "isolation": { "kind": "module" },
8926 "backend": {
8927 "backend": "sqlite",
8928 "path": "/data/cortexkit/alfonso-routing/store.db"
8929 }
8930 })),
8931 "the delivered descriptor is the module's own sqlite store path"
8932 );
8933 }
8934
8935 #[test]
8936 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
8937 let registry = Arc::new(Registry::default());
8938 let handler = ControlHandler::new(Arc::clone(®istry));
8939 let conn = ConnectionId::new(1);
8940 let responses = handler
8941 .handle_control(
8942 conn,
8943 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8944 )
8945 .unwrap();
8946 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8947 let registration = registry.get_module("aft").unwrap().unwrap();
8948 assert_eq!(registration.control_ops, module_baseline_control_ops());
8949
8950 let frame =
8951 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
8952 assert!(handler
8953 .guard_module_control_op(&frame, "aft", "route.bind")
8954 .unwrap()
8955 .is_none());
8956 let error = handler
8957 .guard_module_control_op(&frame, "aft", "test.synthetic")
8958 .unwrap()
8959 .expect("synthetic ungranted op should be rejected");
8960 assert_eq!(error.header.ty, FrameType::Error);
8961 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
8962 }
8963
8964 #[test]
8965 fn hello_control_ops_some_adds_optional_grants() {
8966 let registry = Arc::new(Registry::default());
8967 let handler = ControlHandler::new(Arc::clone(®istry));
8968 handler
8969 .handle_control(
8970 ConnectionId::new(1),
8971 hello_frame_with_control_ops(
8972 "aft",
8973 PROTOCOL_VERSION,
8974 7,
8975 Some(vec![
8976 "future.synthetic".to_string(),
8977 "route.bind".to_string(),
8978 ]),
8979 ),
8980 )
8981 .unwrap();
8982 let registration = registry.get_module("aft").unwrap().unwrap();
8983 assert_eq!(
8984 registration.control_ops,
8985 vec![
8986 "route.bind".to_string(),
8987 "route.status".to_string(),
8988 "future.synthetic".to_string(),
8989 ]
8990 );
8991 let frame =
8992 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
8993 assert!(handler
8994 .guard_module_control_op(&frame, "aft", "future.synthetic")
8995 .unwrap()
8996 .is_none());
8997 }
8998
8999 #[tokio::test]
9000 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
9001 let registry = Arc::new(Registry::default());
9002 let forwarding = Arc::new(ForwardingTable::default());
9003 let handler =
9004 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9005 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
9006 hello_via_sink(
9007 &handler,
9008 &module_ctx,
9009 &mut module_rx,
9010 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9011 )
9012 .await;
9013
9014 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
9015 let responses = handler
9016 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
9017 .await
9018 .unwrap();
9019 assert_eq!(responses.len(), 1);
9020 assert_eq!(responses[0].header.ty, FrameType::Error);
9021 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
9022 assert!(module_rx.try_recv().is_err());
9023 }
9024
9025 #[tokio::test]
9026 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
9027 let registry = Arc::new(Registry::default());
9028 let forwarding = Arc::new(ForwardingTable::default());
9029 let handler =
9030 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9031 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
9032 hello_via_sink(
9033 &handler,
9034 &module_ctx,
9035 &mut module_rx,
9036 hello_frame_with_control_ops(
9037 "aft",
9038 PROTOCOL_VERSION,
9039 7,
9040 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9041 ),
9042 )
9043 .await;
9044
9045 let project_root = unique_project_root("demux");
9046 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
9047 let route_handler = handler.clone();
9048 let route_task = tokio::spawn(async move {
9049 route_handler
9050 .handle_control_frame(
9051 &route_client_ctx,
9052 route_open_frame(100, "aft", project_root),
9053 )
9054 .await
9055 .unwrap()
9056 });
9057 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9058 .await
9059 .unwrap()
9060 .unwrap();
9061 assert!(matches!(
9062 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
9063 ModuleControlRequest::RouteBind { .. }
9064 ));
9065
9066 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
9067 let health_handler = handler.clone();
9068 let health_task = tokio::spawn(async move {
9069 health_handler
9070 .handle_control_frame(
9071 &health_client_ctx,
9072 supervisor_health_probe_frame(101, "aft"),
9073 )
9074 .await
9075 .unwrap()
9076 });
9077 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9078 .await
9079 .unwrap()
9080 .unwrap();
9081 assert_eq!(
9082 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9083 ModuleControlRequest::HealthCheck {}
9084 );
9085
9086 handler
9087 .handle_control_frame(
9088 &module_ctx,
9089 health_response(health_frame.header.corr, HealthStatus::Degraded),
9090 )
9091 .await
9092 .unwrap();
9093 let health_response = health_task.await.unwrap();
9094 assert_eq!(health_response.len(), 1);
9095 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9096 ClientControlResponse::SupervisorHealthProbe {
9097 module_id,
9098 status,
9099 detail,
9100 metrics,
9101 } => {
9102 assert_eq!(module_id, "aft");
9103 assert_eq!(status, HealthStatus::Degraded);
9104 assert_eq!(detail.as_deref(), Some("warming"));
9105 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
9106 }
9107 other => panic!("unexpected health response: {other:?}"),
9108 }
9109
9110 handler
9111 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9112 .await
9113 .unwrap();
9114 let route_response = route_task.await.unwrap();
9115 assert!(route_response.is_empty());
9116 let published = route_client_rx.recv().await.unwrap();
9117 assert!(matches!(
9118 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9119 ClientControlResponse::RouteOpen { .. }
9120 ));
9121 }
9122
9123 async fn relay_route_open(
9128 handler: &ControlHandler,
9129 client_connection: ConnectionId,
9130 client_egress: &FrameSink,
9131 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9132 corr: u64,
9133 module_id: &str,
9134 project_root_label: &str,
9135 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
9136 let ctx = RouteCtx {
9137 connection_id: client_connection,
9138 egress: client_egress.clone(),
9139 };
9140 let handler = handler.clone();
9141 let project_root = unique_project_root(project_root_label);
9142 let module_id = module_id.to_string();
9143 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
9144 let task = tokio::spawn(async move {
9145 let _guard = tracing::dispatcher::set_default(&dispatch);
9146 handler
9147 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
9148 .await
9149 .unwrap()
9150 });
9151 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
9152 .await
9153 .expect("module receives the relayed route.bind")
9154 .expect("module egress is open");
9155 (task, bind.frame)
9156 }
9157
9158 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
9159 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
9160 ModuleControlRequest::RouteBind {
9161 route_channel,
9162 epoch,
9163 ..
9164 } => (route_channel, epoch),
9165 other => panic!("expected a route.bind request, got {other:?}"),
9166 }
9167 }
9168
9169 fn published_route(frame: &Frame) -> (u16, u32) {
9170 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
9171 ClientControlResponse::RouteOpen {
9172 route_channel,
9173 route_epoch,
9174 } => (route_channel, route_epoch),
9175 other => panic!("expected a route.open response, got {other:?}"),
9176 }
9177 }
9178
9179 #[tokio::test]
9199 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
9200 let registry = Arc::new(Registry::default());
9201 let forwarding = Arc::new(ForwardingTable::default());
9202 let handler =
9203 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9204
9205 let module_connection = ConnectionId::new(30);
9206 let (module_ctx, mut module_rx) = route_ctx(module_connection);
9207 hello_via_sink(
9208 &handler,
9209 &module_ctx,
9210 &mut module_rx,
9211 hello_frame("aft", PROTOCOL_VERSION, 7),
9212 )
9213 .await;
9214
9215 let dying_client = ConnectionId::new(31);
9216 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
9217
9218 let (first_task, first_bind) = relay_route_open(
9221 &handler,
9222 dying_client,
9223 &dying_ctx.egress,
9224 &mut module_rx,
9225 100,
9226 "aft",
9227 "closing-first",
9228 )
9229 .await;
9230 handler
9231 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
9232 .await
9233 .unwrap();
9234 assert!(first_task.await.unwrap().is_empty());
9235 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
9236
9237 let (second_task, second_bind) = relay_route_open(
9239 &handler,
9240 dying_client,
9241 &dying_ctx.egress,
9242 &mut module_rx,
9243 101,
9244 "aft",
9245 "closing-second",
9246 )
9247 .await;
9248 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
9249
9250 assert!(forwarding
9253 .escalate_client_delivery_failure(
9254 dying_client,
9255 first_channel,
9256 first_epoch,
9257 CloseReason::new(
9258 "module_to_client_delivery_failed",
9259 "client egress refused a module frame",
9260 ),
9261 crate::forwarding::UndeliveredFrame {
9262 module_id: None,
9263 sink: &dying_ctx.egress,
9264 },
9265 )
9266 .unwrap());
9267 assert!(!dying_ctx.egress.is_closed());
9268
9269 let ack = handler
9271 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
9272 .await;
9273 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
9274 if module_loop_error.is_some() {
9275 handler.cleanup_connection(module_connection).unwrap();
9279 }
9280 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9284 .await
9285 .ok()
9286 .flatten();
9287
9288 assert!(
9290 registry
9291 .get_module_by_connection(module_connection)
9292 .unwrap()
9293 .is_some(),
9294 "one client's closing connection ended the shared module connection: \
9295 {module_loop_error:?}"
9296 );
9297 let cotenant = ConnectionId::new(32);
9299 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
9300 let (cotenant_task, cotenant_bind) = relay_route_open(
9301 &handler,
9302 cotenant,
9303 &cotenant_ctx.egress,
9304 &mut module_rx,
9305 102,
9306 "aft",
9307 "closing-cotenant",
9308 )
9309 .await;
9310 handler
9311 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
9312 .await
9313 .unwrap();
9314 assert!(cotenant_task.await.unwrap().is_empty());
9315 let (cotenant_channel, cotenant_epoch) =
9316 published_route(&cotenant_rx.recv().await.unwrap());
9317 assert!(matches!(
9318 forwarding
9319 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
9320 .unwrap(),
9321 DataRoute::Client(DataRouteState::Bound(_))
9322 ));
9323
9324 let goodbye = post_ack_module_frame
9327 .expect("module receives a GOODBYE for the abandoned route channel");
9328 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9329 assert_eq!(goodbye.header.channel, abandoned_channel);
9330 assert_eq!(goodbye.header.epoch, abandoned_epoch);
9331
9332 assert!(dying_rx.try_recv().is_err());
9336 let second_response = second_task.await.unwrap();
9337 assert_eq!(second_response.len(), 1);
9338 assert_eq!(
9339 parse_error(&second_response[0])["code"],
9340 "target_unavailable"
9341 );
9342 }
9343
9344 async fn catalog_snapshot(
9347 handler: &ControlHandler,
9348 client_ctx: &RouteCtx,
9349 corr: u64,
9350 ) -> (u64, Vec<(String, bool)>) {
9351 let body =
9352 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
9353 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
9354 let reply = handler
9355 .handle_control_frame(client_ctx, request)
9356 .await
9357 .unwrap()
9358 .pop()
9359 .unwrap();
9360 let ClientControlResponse::CatalogList {
9361 generation,
9362 modules,
9363 ..
9364 } = serde_json::from_slice(&reply.body).unwrap()
9365 else {
9366 panic!("not a catalog.list reply");
9367 };
9368 (
9369 generation,
9370 modules
9371 .into_iter()
9372 .map(|entry| (entry.module_id, entry.ready))
9373 .collect(),
9374 )
9375 }
9376
9377 fn hello_held_before_forwarding() -> (
9383 ControlHandler,
9384 Arc<std::sync::Barrier>,
9385 std::thread::JoinHandle<Vec<Frame>>,
9386 RouteCtx,
9387 mpsc::Receiver<crate::router::OutboundFrame>,
9388 ) {
9389 let hold = Arc::new(std::sync::Barrier::new(2));
9390 let handler = ControlHandler::with_forwarding(
9391 Arc::new(Registry::default()),
9392 Arc::new(ForwardingTable::default()),
9393 )
9394 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
9395 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9396 let hello = {
9397 let handler = handler.clone();
9398 let module_ctx = module_ctx.clone();
9399 std::thread::spawn(move || {
9400 handler
9401 .handle_hello(
9402 module_ctx.connection_id,
9403 Some(module_ctx.egress.clone()),
9404 hello_frame("aft", PROTOCOL_VERSION, 7),
9405 )
9406 .unwrap()
9407 })
9408 };
9409 (handler, hold, hello, module_ctx, module_rx)
9410 }
9411
9412 #[tokio::test]
9421 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
9422 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
9423 let client = ConnectionId::new(41);
9424 let (client_ctx, mut client_rx) = route_ctx(client);
9425
9426 hold.wait();
9427 assert!(handler.registry.get_module("aft").unwrap().is_some());
9429 assert!(!handler
9430 .forwarding
9431 .has_live_module_connection("aft")
9432 .unwrap());
9433 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
9434 assert!(
9435 listed.is_empty(),
9436 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
9437 );
9438 let refused = handler
9439 .handle_control_frame(
9440 &client_ctx,
9441 route_open_frame(301, "aft", unique_project_root("listed-window")),
9442 )
9443 .await
9444 .unwrap();
9445 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
9446 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
9447 let duplicate = handler
9448 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
9449 .await
9450 .unwrap();
9451 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9452
9453 hold.wait();
9454 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9455 let ack = module_rx.try_recv().unwrap().frame;
9456 assert_eq!(ack.header.ty, FrameType::HelloAck);
9457 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9458 assert_eq!(listed, vec![("aft".to_string(), true)]);
9459
9460 let (open_task, bind) = relay_route_open(
9461 &handler,
9462 client,
9463 &client_ctx.egress,
9464 &mut module_rx,
9465 303,
9466 "aft",
9467 "listed-after",
9468 )
9469 .await;
9470 assert!(handler
9471 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9472 .await
9473 .unwrap()
9474 .is_empty());
9475 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
9476 published_route(&client_rx.try_recv().unwrap().frame);
9477 }
9478
9479 #[tokio::test]
9486 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
9487 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
9488 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9489 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9490
9491 hold.wait();
9492 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
9493 assert!(listed.is_empty());
9494 assert_eq!(
9495 during, before,
9496 "the generation moved while the module was not listed"
9497 );
9498
9499 hold.wait();
9500 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9501 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9502 assert_eq!(listed, vec![("aft".to_string(), true)]);
9503 assert_eq!(
9504 after,
9505 before + 1,
9506 "the module appeared without exactly one generation bump"
9507 );
9508 }
9509
9510 #[tokio::test]
9515 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
9516 let registry = Arc::new(Registry::default());
9517 let handler = ControlHandler::with_forwarding(
9518 Arc::clone(®istry),
9519 Arc::new(ForwardingTable::default()),
9520 );
9521 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9522 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9523 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9524 drop(module_rx);
9525
9526 let replies = handler
9527 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
9528 .await
9529 .unwrap();
9530 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
9531 assert!(registry.get_module("aft").unwrap().is_none());
9532 assert!(registry
9533 .get_module_by_connection(module_ctx.connection_id)
9534 .unwrap()
9535 .is_none());
9536 assert_eq!(
9537 catalog_snapshot(&handler, &client_ctx, 301).await,
9538 (before, Vec::new())
9539 );
9540 }
9541
9542 async fn route_open_held_after_bind_send() -> (
9546 ControlHandler,
9547 Arc<tokio::sync::Barrier>,
9548 RouteCtx,
9549 mpsc::Receiver<crate::router::OutboundFrame>,
9550 ) {
9551 let hold = Arc::new(tokio::sync::Barrier::new(2));
9552 let handler = ControlHandler::with_forwarding(
9553 Arc::new(Registry::default()),
9554 Arc::new(ForwardingTable::default()),
9555 )
9556 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
9557 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
9558 hello_via_sink(
9559 &handler,
9560 &module_ctx,
9561 &mut module_rx,
9562 hello_frame("aft", PROTOCOL_VERSION, 7),
9563 )
9564 .await;
9565 (handler, hold, module_ctx, module_rx)
9566 }
9567
9568 #[tokio::test]
9575 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
9576 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9577 let client = ConnectionId::new(41);
9578 let (client_ctx, mut client_rx) = route_ctx(client);
9579
9580 let (open_task, bind) = relay_route_open(
9581 &handler,
9582 client,
9583 &client_ctx.egress,
9584 &mut module_rx,
9585 200,
9586 "aft",
9587 "acked-before-mark",
9588 )
9589 .await;
9590 let (module_channel, module_epoch) = route_bind_channel(&bind);
9591 hold.wait().await;
9593
9594 assert!(handler
9596 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9597 .await
9598 .unwrap()
9599 .is_empty());
9600 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
9601
9602 hold.wait().await;
9603 assert!(
9604 open_task.await.unwrap().is_empty(),
9605 "route.open answers accepted"
9606 );
9607
9608 match module_rx.try_recv() {
9611 Err(mpsc::error::TryRecvError::Empty) => {}
9612 Ok(outbound) => panic!(
9613 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
9614 outbound.frame.header.ty,
9615 outbound.frame.header.channel,
9616 outbound.frame.header.epoch
9617 ),
9618 Err(err) => panic!("module egress closed: {err}"),
9619 }
9620
9621 let router = Router::with_control_handler(Arc::new(handler.clone()));
9624 let request = Frame::build(
9625 FrameType::Request,
9626 Flags::new(true, Priority::Interactive, false),
9627 client_channel,
9628 client_epoch,
9629 201,
9630 b"first request".to_vec(),
9631 )
9632 .unwrap();
9633 router
9634 .route_for_connection(&client_ctx, request)
9635 .await
9636 .unwrap();
9637 let delivered = module_rx.try_recv().unwrap().frame;
9638 assert_eq!(delivered.header.ty, FrameType::Request);
9639 assert_eq!(
9640 (delivered.header.channel, delivered.header.epoch),
9641 (module_channel, module_epoch)
9642 );
9643 assert_eq!(delivered.body, b"first request");
9644 }
9645
9646 #[tokio::test]
9652 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
9653 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9654 let client = ConnectionId::new(42);
9655 let (client_ctx, _client_rx) = route_ctx(client);
9656
9657 let (open_task, bind) = relay_route_open(
9658 &handler,
9659 client,
9660 &client_ctx.egress,
9661 &mut module_rx,
9662 300,
9663 "aft",
9664 "released-before-mark",
9665 )
9666 .await;
9667 let (module_channel, module_epoch) = route_bind_channel(&bind);
9668 hold.wait().await;
9669
9670 handler.cleanup_connection(client).unwrap();
9671 assert!(
9672 module_rx.try_recv().is_err(),
9673 "client teardown must not GOODBYE a bind route.open is still queueing"
9674 );
9675
9676 hold.wait().await;
9677 let response = open_task.await.unwrap();
9678 assert_eq!(response.len(), 1);
9679 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9680
9681 let goodbye = module_rx
9682 .try_recv()
9683 .expect("module is told to drop the binding the daemon abandoned")
9684 .frame;
9685 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9686 assert_eq!(
9687 (goodbye.header.channel, goodbye.header.epoch),
9688 (module_channel, module_epoch)
9689 );
9690 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9691 }
9692
9693 #[tokio::test]
9698 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
9699 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9700 let client = ConnectionId::new(43);
9701 let (client_ctx, client_rx) = route_ctx(client);
9702
9703 let (open_task, bind) = relay_route_open(
9704 &handler,
9705 client,
9706 &client_ctx.egress,
9707 &mut module_rx,
9708 400,
9709 "aft",
9710 "gone-client-before-mark",
9711 )
9712 .await;
9713 let (module_channel, module_epoch) = route_bind_channel(&bind);
9714 hold.wait().await;
9715
9716 drop(client_rx);
9717 assert!(client_ctx.egress.is_closed());
9718 handler
9719 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9720 .await
9721 .unwrap();
9722
9723 hold.wait().await;
9724 let response = open_task.await.unwrap();
9725 assert_eq!(response.len(), 1);
9726 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9727
9728 let goodbye = module_rx
9729 .try_recv()
9730 .expect("module is told to drop the binding it acked")
9731 .frame;
9732 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9733 assert_eq!(
9734 (goodbye.header.channel, goodbye.header.epoch),
9735 (module_channel, module_epoch)
9736 );
9737 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9738 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
9739 }
9740
9741 #[test]
9748 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
9749 let handler = ControlHandler::default();
9750 let module_connection = ConnectionId::new(30);
9751 let client_connection = ConnectionId::new(31);
9752
9753 handler
9754 .refuse_to_end_module_connection_for_a_client(
9755 module_connection,
9756 77,
9757 ForwardingError::ConnectionClosing {
9758 connection_id: client_connection,
9759 },
9760 )
9761 .expect("a closing client must never end the module connection");
9762
9763 assert!(matches!(
9764 handler.refuse_to_end_module_connection_for_a_client(
9765 module_connection,
9766 78,
9767 ForwardingError::ConnectionClosing {
9768 connection_id: module_connection,
9769 },
9770 ),
9771 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
9772 connection_id
9773 })) if connection_id == module_connection
9774 ));
9775 assert!(matches!(
9776 handler.refuse_to_end_module_connection_for_a_client(
9777 module_connection,
9778 79,
9779 ForwardingError::Poisoned,
9780 ),
9781 Err(RouterError::Forwarding(ForwardingError::Poisoned))
9782 ));
9783 assert!(matches!(
9784 handler.refuse_to_end_module_connection_for_a_client(
9785 module_connection,
9786 80,
9787 ForwardingError::StaleModuleEndpoint,
9788 ),
9789 Err(RouterError::Forwarding(
9790 ForwardingError::StaleModuleEndpoint
9791 ))
9792 ));
9793 }
9794
9795 #[tokio::test]
9808 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
9809 let registry = Arc::new(Registry::default());
9810 let forwarding = Arc::new(ForwardingTable::default());
9811 let supervisor = SupervisorHandle::new();
9812 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9813 let handler =
9814 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9815 .with_supervisor(supervisor);
9816
9817 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9818 hello_via_sink(
9819 &handler,
9820 &target_ctx,
9821 &mut target_rx,
9822 hello_frame("target", PROTOCOL_VERSION, 1),
9823 )
9824 .await;
9825
9826 let wrong_nonce = handler
9830 .handle_control_frame(
9831 &route_ctx(ConnectionId::new(91)).0,
9832 route_open_frame_with_admission_facts(
9833 20,
9834 "target",
9835 unique_project_root("admission-facts"),
9836 Some(subc_control::ConsumerIdentity {
9837 module_id: "fed".to_string(),
9838 launch_nonce: "not-the-real-nonce".to_string(),
9839 }),
9840 None,
9841 ),
9842 )
9843 .await
9844 .unwrap();
9845 assert_eq!(
9846 parse_error(&wrong_nonce[0])["code"],
9847 "bad_consumer_identity",
9848 "a mismatched launch nonce must be refused, not stamped Reserved"
9849 );
9850
9851 let never_spawned = handler
9855 .handle_control_frame(
9856 &route_ctx(ConnectionId::new(92)).0,
9857 route_open_frame_with_admission_facts(
9858 21,
9859 "target",
9860 unique_project_root("admission-facts"),
9861 Some(subc_control::ConsumerIdentity {
9862 module_id: "never-spawned".to_string(),
9863 launch_nonce: "any-nonce".to_string(),
9864 }),
9865 None,
9866 ),
9867 )
9868 .await
9869 .unwrap();
9870 assert_eq!(
9871 parse_error(&never_spawned[0])["code"],
9872 "bad_consumer_identity",
9873 "an unspawned module_id must be refused rather than accepted for lack of a record"
9874 );
9875 }
9876
9877 #[tokio::test]
9900 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
9901 let registry = Arc::new(Registry::default());
9902 let forwarding = Arc::new(ForwardingTable::default());
9903 let supervisor = SupervisorHandle::new();
9904 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9905 let handler =
9906 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9907 .with_supervisor(supervisor);
9908
9909 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
9910 hello_via_sink(
9911 &handler,
9912 &target_ctx,
9913 &mut target_rx,
9914 hello_frame("target", PROTOCOL_VERSION, 1),
9915 )
9916 .await;
9917
9918 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
9919 let route_handler = handler.clone();
9920 let route_task = tokio::spawn(async move {
9921 route_handler
9922 .handle_control_frame(
9923 &client_ctx,
9924 route_open_frame_with_admission_facts(
9925 30,
9926 "target",
9927 unique_project_root("admission-facts"),
9928 Some(subc_control::ConsumerIdentity {
9929 module_id: "fed".to_string(),
9930 launch_nonce: "fed-nonce".to_string(),
9931 }),
9932 None,
9933 ),
9934 )
9935 .await
9936 .unwrap()
9937 });
9938
9939 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9946 .await
9947 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
9948 .expect("module control channel closed before route.bind");
9949 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9950 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
9951 panic!("expected route.bind")
9952 };
9953 assert_eq!(
9954 principal,
9955 Some(Principal::Reserved {
9956 module_id: "fed".to_string()
9957 }),
9958 "a correctly attested consumer must be stamped Reserved for its own id"
9959 );
9960
9961 handler
9962 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9963 .await
9964 .unwrap();
9965 assert!(route_task.await.unwrap().is_empty());
9966 assert!(
9967 matches!(
9968 serde_json::from_slice::<ClientControlResponse>(
9969 &client_rx.recv().await.unwrap().body
9970 )
9971 .unwrap(),
9972 ClientControlResponse::RouteOpen { .. }
9973 ),
9974 "the route must actually open, not merely avoid an error"
9975 );
9976 }
9977
9978 #[tokio::test(start_paused = true)]
9979 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
9980 let registry = Arc::new(Registry::default());
9981 let forwarding = Arc::new(ForwardingTable::default());
9982 let supervisor = SupervisorHandle::new();
9983 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9984 let handler =
9985 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9986 .with_supervisor(supervisor);
9987
9988 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
9989 hello_via_sink(
9990 &handler,
9991 &target_ctx,
9992 &mut target_rx,
9993 hello_frame("target", PROTOCOL_VERSION, 1),
9994 )
9995 .await;
9996
9997 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
9998 let direct_handler = handler.clone();
9999 let direct_open = tokio::spawn(async move {
10000 direct_handler
10001 .handle_control_frame(
10002 &direct_ctx,
10003 route_open_frame(2, "target", unique_project_root("route-census-direct")),
10004 )
10005 .await
10006 .unwrap()
10007 });
10008 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10009 .await
10010 .expect("no direct route.bind within 5s")
10011 .expect("target control channel closed before direct route.bind");
10012 handler
10013 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
10014 .await
10015 .unwrap();
10016 assert!(direct_open.await.unwrap().is_empty());
10017 let _ = direct_rx.recv().await.unwrap();
10018
10019 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
10020 let reserved_handler = handler.clone();
10021 let reserved_open = tokio::spawn(async move {
10022 reserved_handler
10023 .handle_control_frame(
10024 &reserved_ctx,
10025 route_open_frame_with_admission_facts(
10026 3,
10027 "target",
10028 unique_project_root("admission-facts"),
10029 Some(ConsumerIdentity {
10030 module_id: "fed".to_string(),
10031 launch_nonce: "fed-nonce".to_string(),
10032 }),
10033 None,
10034 ),
10035 )
10036 .await
10037 .unwrap()
10038 });
10039 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10040 .await
10041 .expect("no reserved route.bind within 5s")
10042 .expect("target control channel closed before reserved route.bind");
10043 handler
10044 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
10045 .await
10046 .unwrap();
10047 assert!(reserved_open.await.unwrap().is_empty());
10048 let _ = reserved_rx.recv().await.unwrap();
10049
10050 forwarding
10051 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
10052 .unwrap();
10053 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
10054 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
10055 module_id: Some("target".to_string()),
10056 })
10057 .unwrap();
10058 let census_frame =
10059 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
10060 let response = handler
10061 .handle_control_frame(&census_ctx, census_frame)
10062 .await
10063 .unwrap()
10064 .pop()
10065 .unwrap();
10066 let actual: Value = serde_json::from_slice(&response.body).unwrap();
10067 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
10068 assert!(matches!(
10069 decoded,
10070 ClientControlResponse::SupervisorRoutes { .. }
10071 ));
10072 let routes = actual["modules"][0]["routes"].as_array().unwrap();
10073 assert_eq!(routes.len(), 2);
10074 assert!(routes.iter().all(|route| route["draining"] == true));
10075 assert!(
10078 routes.iter().all(|route| route["drain_reason"] == "reload"),
10079 "draining routes must name the drain's reason: {routes:?}"
10080 );
10081 assert!(routes.iter().any(|route| {
10082 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
10083 }));
10084 assert!(routes.iter().any(|route| {
10085 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
10086 }));
10087
10088 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10089 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
10090 if std::env::var_os("UPDATE_GOLDEN").is_some() {
10091 std::fs::write(
10092 &golden_path,
10093 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
10094 )
10095 .unwrap();
10096 }
10097 let expected: Value =
10098 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
10099 assert_eq!(actual, expected);
10100 }
10101
10102 async fn query_live_roots(
10103 handler: &ControlHandler,
10104 module_ctx: &RouteCtx,
10105 ) -> ModuleControlResponseToModule {
10106 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
10107 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
10108 let response = handler
10109 .handle_control_frame(module_ctx, frame)
10110 .await
10111 .unwrap()
10112 .pop()
10113 .unwrap();
10114 serde_json::from_slice(&response.body).unwrap()
10115 }
10116
10117 #[tokio::test(start_paused = true)]
10118 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
10119 let registry = Arc::new(Registry::default());
10120 let forwarding = Arc::new(ForwardingTable::default());
10121 let handler = ControlHandler::with_forwarding(registry, forwarding);
10122 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
10123 hello_via_sink(
10124 &handler,
10125 &target_ctx,
10126 &mut target_rx,
10127 hello_frame("target", PROTOCOL_VERSION, 1),
10128 )
10129 .await;
10130 let root = unique_project_root("live-roots-known");
10131 let path = ProjectRootId::from_path_allowing_missing(root.path())
10132 .unwrap()
10133 .as_path()
10134 .to_path_buf();
10135 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
10136 let open_handler = handler.clone();
10137 let opened = tokio::spawn(async move {
10138 open_handler
10139 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
10140 .await
10141 .unwrap()
10142 });
10143 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10144 .await
10145 .unwrap()
10146 .unwrap();
10147 handler
10148 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
10149 .await
10150 .unwrap();
10151 assert!(opened.await.unwrap().is_empty());
10152 let _ = client_rx.recv().await.unwrap();
10153
10154 let root = unique_project_root("live-roots-pending");
10155 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
10156 .unwrap()
10157 .as_path()
10158 .to_path_buf();
10159 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
10160 let open_handler = handler.clone();
10161 let pending = tokio::spawn(async move {
10162 open_handler
10163 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
10164 .await
10165 .unwrap()
10166 });
10167 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10168 .await
10169 .unwrap()
10170 .unwrap();
10171 let actual = query_live_roots(&handler, &target_ctx).await;
10172 let ModuleControlResponseToModule::LiveRoots {
10173 roots,
10174 unknown_root_bindings,
10175 total_bindings,
10176 } = actual
10177 else {
10178 panic!("expected live roots")
10179 };
10180 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
10181 assert_eq!(unknown_root_bindings, 0);
10182 assert_eq!(
10183 roots.len(),
10184 2,
10185 "root-known arm must retain each canonical root"
10186 );
10187 assert_eq!(
10188 total_bindings,
10189 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10190 );
10191 let counts = roots
10192 .iter()
10193 .map(|root| (root.project_root.clone(), root.bound, root.pending))
10194 .collect::<Vec<_>>();
10195 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
10196 expected.sort_by(|a, b| a.0.cmp(&b.0));
10197 assert_eq!(
10198 counts, expected,
10199 "roots must sort by path and count pending separately"
10200 );
10201 handler
10202 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
10203 .await
10204 .unwrap();
10205 assert!(pending.await.unwrap().is_empty());
10206 }
10207
10208 #[tokio::test(start_paused = true)]
10209 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
10210 let forwarding = Arc::new(ForwardingTable::default());
10211 let handler =
10212 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10213 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
10214 hello_via_sink(
10215 &handler,
10216 &target_ctx,
10217 &mut target_rx,
10218 hello_frame("target", PROTOCOL_VERSION, 1),
10219 )
10220 .await;
10221 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
10222 let pending = forwarding
10223 .begin_route_bind_relay_for_test(
10224 client_ctx.connection_id,
10225 client_ctx.egress.clone(),
10226 2,
10227 "target",
10228 )
10229 .unwrap();
10230 forwarding
10231 .complete_pending_relay(
10232 target_ctx.connection_id,
10233 pending.corr,
10234 RouteBindRelayOutcome::Accepted,
10235 )
10236 .unwrap();
10237 let actual = query_live_roots(&handler, &target_ctx).await;
10238 let ModuleControlResponseToModule::LiveRoots {
10239 roots,
10240 unknown_root_bindings,
10241 total_bindings,
10242 } = actual
10243 else {
10244 panic!("expected live roots")
10245 };
10246 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
10247 assert_eq!(
10248 unknown_root_bindings, 1,
10249 "unknown-root arm must not read as no bindings"
10250 );
10251 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
10252 assert_eq!(
10253 total_bindings,
10254 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10255 );
10256 }
10257
10258 #[tokio::test(start_paused = true)]
10266 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
10267 let forwarding = Arc::new(ForwardingTable::default());
10268 let handler =
10269 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10270 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
10271 let replies = handler
10272 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
10273 .await
10274 .unwrap();
10275 let queued_by_hello = module_rx.len();
10276
10277 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
10278 let open_handler = handler.clone();
10279 let open = tokio::spawn(async move {
10280 open_handler
10281 .handle_control_frame(
10282 &client_ctx,
10283 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
10284 )
10285 .await
10286 .unwrap()
10287 });
10288 let mut spins = 0;
10290 while module_rx.len() == queued_by_hello {
10291 spins += 1;
10292 assert!(spins < 10_000, "route.open never queued a route.bind");
10293 tokio::task::yield_now().await;
10294 }
10295
10296 for reply in replies {
10298 module_ctx.egress.send(reply).await.unwrap();
10299 }
10300
10301 let first = module_rx.recv().await.unwrap().frame;
10302 assert_eq!(
10303 first.header.ty,
10304 FrameType::HelloAck,
10305 "the first frame a registering module reads must be its HELLO_ACK"
10306 );
10307 assert_eq!(first.header.corr, 7);
10308 let second = module_rx.recv().await.unwrap().frame;
10309 assert_eq!(second.header.ty, FrameType::Request);
10310 assert!(
10311 matches!(
10312 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
10313 ModuleControlRequest::RouteBind { .. }
10314 ),
10315 "the route.bind follows the ack"
10316 );
10317 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
10318
10319 handler
10320 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
10321 .await
10322 .unwrap();
10323 assert!(open.await.unwrap().is_empty());
10324 let _ = client_rx.recv().await.unwrap();
10325 }
10326
10327 #[tokio::test(start_paused = true)]
10328 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
10329 let handler = ControlHandler::with_forwarding(
10330 Arc::new(Registry::default()),
10331 Arc::new(ForwardingTable::default()),
10332 );
10333 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
10334 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
10335 hello_via_sink(
10336 &handler,
10337 &first_ctx,
10338 &mut first_rx,
10339 hello_frame("first", PROTOCOL_VERSION, 1),
10340 )
10341 .await;
10342 hello_via_sink(
10343 &handler,
10344 &second_ctx,
10345 &mut second_rx,
10346 hello_frame("second", PROTOCOL_VERSION, 2),
10347 )
10348 .await;
10349 let root = unique_project_root("second-only");
10350 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
10351 let cloned = handler.clone();
10352 let open = tokio::spawn(async move {
10353 cloned
10354 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
10355 .await
10356 .unwrap()
10357 });
10358 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
10359 .await
10360 .unwrap()
10361 .unwrap();
10362 let first = query_live_roots(&handler, &first_ctx).await;
10363 let second = query_live_roots(&handler, &second_ctx).await;
10364 assert!(
10365 matches!(
10366 first,
10367 ModuleControlResponseToModule::LiveRoots {
10368 total_bindings: 0,
10369 ..
10370 }
10371 ),
10372 "cross-module scope must not expose another module's roots"
10373 );
10374 assert!(
10375 matches!(
10376 second,
10377 ModuleControlResponseToModule::LiveRoots {
10378 total_bindings: 1,
10379 ..
10380 }
10381 ),
10382 "second module must see its pending route"
10383 );
10384 handler
10385 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
10386 .await
10387 .unwrap();
10388 assert!(open.await.unwrap().is_empty());
10389 }
10390
10391 #[tokio::test(start_paused = true)]
10392 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
10393 let handler = ControlHandler::with_forwarding(
10394 Arc::new(Registry::default()),
10395 Arc::new(ForwardingTable::default()),
10396 );
10397 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
10398 hello_via_sink(
10399 &handler,
10400 &target_ctx,
10401 &mut target_rx,
10402 hello_frame("target", PROTOCOL_VERSION, 1),
10403 )
10404 .await;
10405 let actual = query_live_roots(&handler, &target_ctx).await;
10406 let ModuleControlResponseToModule::LiveRoots {
10407 roots,
10408 unknown_root_bindings,
10409 total_bindings,
10410 } = actual
10411 else {
10412 panic!("expected live roots")
10413 };
10414 assert!(roots.is_empty());
10415 assert_eq!(unknown_root_bindings, 0);
10416 assert_eq!(total_bindings, 0);
10417 assert_eq!(
10418 total_bindings,
10419 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10420 );
10421 }
10422
10423 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
10430 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10431 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
10432 let text = std::fs::read_to_string(&path)
10433 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
10434 let vectors: Vec<(String, Value)> = text
10435 .lines()
10436 .filter(|line| !line.trim().is_empty())
10437 .map(|line| {
10438 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
10439 let id = entry["corpus_id"]
10440 .as_str()
10441 .expect("every vector carries a corpus_id")
10442 .to_string();
10443 (id, entry["package"].clone())
10444 })
10445 .collect();
10446 assert_eq!(
10450 vectors.len(),
10451 3,
10452 "vendored fed corpus changed size; re-sync from subc-federation"
10453 );
10454
10455 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
10470 let richest = vectors
10471 .iter()
10472 .filter_map(|(_, package)| package.as_object())
10473 .map(|object| {
10474 object
10475 .keys()
10476 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
10477 .count()
10478 })
10479 .max()
10480 .unwrap_or(0);
10481 assert!(
10482 richest >= 2,
10483 "vendored corpus no longer carries a package with unmodelled fields, \
10484 so the relay test can no longer distinguish a verbatim relay from a lossy one"
10485 );
10486
10487 vectors
10488 }
10489
10490 #[tokio::test]
10498 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
10499 for (corpus_id, package) in fed_admission_facts_vectors() {
10500 let registry = Arc::new(Registry::default());
10501 let forwarding = Arc::new(ForwardingTable::default());
10502 let supervisor = SupervisorHandle::new();
10503 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10504 let handler =
10505 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10506 .with_supervisor(supervisor)
10507 .with_admission_facts_config(
10508 Some("fed".to_string()),
10509 Some(vec!["target".to_string()]),
10510 );
10511
10512 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10513 hello_via_sink(
10514 &handler,
10515 &target_ctx,
10516 &mut target_rx,
10517 hello_frame("target", PROTOCOL_VERSION, 1),
10518 )
10519 .await;
10520
10521 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
10522 let route_handler = handler.clone();
10523 let expected = package.clone();
10524 let route_task = tokio::spawn(async move {
10525 route_handler
10526 .handle_control_frame(
10527 &client_ctx,
10528 route_open_frame_with_admission_facts(
10529 20,
10530 "target",
10531 unique_project_root("admission-facts"),
10532 Some(subc_control::ConsumerIdentity {
10533 module_id: "fed".to_string(),
10534 launch_nonce: "fed-nonce".to_string(),
10535 }),
10536 Some(package),
10537 ),
10538 )
10539 .await
10540 .unwrap()
10541 });
10542
10543 let bind_frame = target_rx.recv().await.unwrap();
10544 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10545 let ModuleControlRequest::RouteBind {
10546 admission_facts, ..
10547 } = bind
10548 else {
10549 panic!("{corpus_id}: expected route.bind")
10550 };
10551 assert_eq!(
10552 admission_facts,
10553 Some(expected),
10554 "{corpus_id}: relay must not add, drop or reshape any field"
10555 );
10556
10557 handler
10558 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10559 .await
10560 .unwrap();
10561 route_task.await.unwrap();
10562 }
10563 }
10564
10565 #[tokio::test]
10566 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
10567 let registry = Arc::new(Registry::default());
10568 let forwarding = Arc::new(ForwardingTable::default());
10569 let supervisor = SupervisorHandle::new();
10570 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10571 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
10572 let handler =
10573 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10574 .with_supervisor(supervisor)
10575 .with_admission_facts_config(
10576 Some("fed".to_string()),
10577 Some(vec!["target".to_string()]),
10578 );
10579
10580 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
10581 hello_via_sink(
10582 &handler,
10583 &target_ctx,
10584 &mut target_rx,
10585 hello_frame("target", PROTOCOL_VERSION, 1),
10586 )
10587 .await;
10588 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
10589 hello_via_sink(
10590 &handler,
10591 &other_ctx,
10592 &mut other_rx,
10593 hello_frame("other", PROTOCOL_VERSION, 2),
10594 )
10595 .await;
10596
10597 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
10598 let expected_facts = facts.clone();
10599 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
10600 let route_handler = handler.clone();
10601 let route_task = tokio::spawn(async move {
10602 route_handler
10603 .handle_control_frame(
10604 &client_ctx,
10605 route_open_frame_with_admission_facts(
10606 10,
10607 "target",
10608 unique_project_root("admission-facts"),
10609 Some(subc_control::ConsumerIdentity {
10610 module_id: "fed".to_string(),
10611 launch_nonce: "fed-nonce".to_string(),
10612 }),
10613 Some(facts.clone()),
10614 ),
10615 )
10616 .await
10617 .unwrap()
10618 });
10619 let bind_frame = target_rx.recv().await.unwrap();
10620 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10621 let ModuleControlRequest::RouteBind {
10622 admission_facts, ..
10623 } = bind
10624 else {
10625 panic!("expected route.bind")
10626 };
10627 assert_eq!(admission_facts, Some(expected_facts));
10628 handler
10629 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10630 .await
10631 .unwrap();
10632 assert!(route_task.await.unwrap().is_empty());
10633 assert!(matches!(
10634 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
10635 .unwrap(),
10636 ClientControlResponse::RouteOpen { .. }
10637 ));
10638
10639 let direct = handler
10640 .handle_control_frame(
10641 &route_ctx(ConnectionId::new(73)).0,
10642 route_open_frame_with_admission_facts(
10643 11,
10644 "target",
10645 unique_project_root("admission-facts"),
10646 None,
10647 Some(json!({"x": 1})),
10648 ),
10649 )
10650 .await
10651 .unwrap();
10652 assert_eq!(
10653 parse_error(&direct[0])["code"],
10654 "admission_facts_not_permitted"
10655 );
10656
10657 let different_reserved = handler
10658 .handle_control_frame(
10659 &route_ctx(ConnectionId::new(77)).0,
10660 route_open_frame_with_admission_facts(
10661 15,
10662 "target",
10663 unique_project_root("admission-facts"),
10664 Some(subc_control::ConsumerIdentity {
10665 module_id: "other".to_string(),
10666 launch_nonce: "other-nonce".to_string(),
10667 }),
10668 Some(json!({"x": 1})),
10669 ),
10670 )
10671 .await
10672 .unwrap();
10673 assert_eq!(
10674 parse_error(&different_reserved[0])["code"],
10675 "admission_facts_not_permitted"
10676 );
10677
10678 let other_target = handler
10679 .handle_control_frame(
10680 &route_ctx(ConnectionId::new(74)).0,
10681 route_open_frame_with_admission_facts(
10682 12,
10683 "other",
10684 unique_project_root("admission-facts"),
10685 Some(subc_control::ConsumerIdentity {
10686 module_id: "fed".to_string(),
10687 launch_nonce: "fed-nonce".to_string(),
10688 }),
10689 Some(json!({"x": 1})),
10690 ),
10691 )
10692 .await
10693 .unwrap();
10694 assert_eq!(
10695 parse_error(&other_target[0])["code"],
10696 "admission_facts_target_not_allowed"
10697 );
10698
10699 let nonexistent = handler
10700 .handle_control_frame(
10701 &route_ctx(ConnectionId::new(75)).0,
10702 route_open_frame_with_admission_facts(
10703 13,
10704 "missing",
10705 unique_project_root("admission-facts"),
10706 None,
10707 Some(json!({"x": 1})),
10708 ),
10709 )
10710 .await
10711 .unwrap();
10712 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
10713
10714 let described = handler
10715 .handle_control_frame(
10716 &route_ctx(ConnectionId::new(76)).0,
10717 Frame::build(
10718 FrameType::Request,
10719 control_flags(),
10720 0,
10721 0,
10722 14,
10723 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10724 )
10725 .unwrap(),
10726 )
10727 .await
10728 .unwrap();
10729 let ClientControlResponse::ServerDescribe { capabilities, .. } =
10730 serde_json::from_slice(&described[0].body).unwrap()
10731 else {
10732 panic!("expected server.describe response")
10733 };
10734 assert!(capabilities
10735 .iter()
10736 .any(|cap| cap == "admission_facts_relay_v1"));
10737 }
10738
10739 #[tokio::test]
10740 async fn admission_facts_without_configured_carrier_are_rejected() {
10741 let registry = Arc::new(Registry::default());
10742 let forwarding = Arc::new(ForwardingTable::default());
10743 let handler = ControlHandler::with_forwarding(registry, forwarding);
10744 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
10745 hello_via_sink(
10746 &handler,
10747 &target_ctx,
10748 &mut target_rx,
10749 hello_frame("target", PROTOCOL_VERSION, 1),
10750 )
10751 .await;
10752
10753 let responses = handler
10754 .handle_control_frame(
10755 &route_ctx(ConnectionId::new(79)).0,
10756 route_open_frame_with_admission_facts(
10757 16,
10758 "target",
10759 unique_project_root("admission-facts"),
10760 None,
10761 Some(json!({"x": 1})),
10762 ),
10763 )
10764 .await
10765 .unwrap();
10766 assert_eq!(
10767 parse_error(&responses[0])["code"],
10768 "admission_facts_not_permitted"
10769 );
10770 }
10771
10772 #[tokio::test]
10773 async fn route_open_relays_consumer_capabilities_verbatim() {
10774 let registry = Arc::new(Registry::default());
10775 let forwarding = Arc::new(ForwardingTable::default());
10776 let handler =
10777 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10778 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
10779 hello_via_sink(
10780 &handler,
10781 &module_ctx,
10782 &mut module_rx,
10783 hello_frame("aft", PROTOCOL_VERSION, 7),
10784 )
10785 .await;
10786
10787 let expected = vec!["elicitation".to_string(), "roots".to_string()];
10788 let expected_for_request = expected.clone();
10789 let project_root = unique_project_root("consumer-capabilities-present");
10790 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
10791 let route_handler = handler.clone();
10792 let route_task = tokio::spawn(async move {
10793 route_handler
10794 .handle_control_frame(
10795 &client_ctx,
10796 route_open_frame_with_consumer_capabilities(
10797 401,
10798 "aft",
10799 project_root,
10800 Some(expected_for_request),
10801 ),
10802 )
10803 .await
10804 .unwrap()
10805 });
10806 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10807 .await
10808 .unwrap()
10809 .unwrap();
10810 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10811 let ModuleControlRequest::RouteBind {
10812 consumer_capabilities,
10813 ..
10814 } = bind
10815 else {
10816 panic!("expected route.bind request, got {bind:?}");
10817 };
10818 assert_eq!(consumer_capabilities, Some(expected.clone()));
10819
10820 handler
10821 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10822 .await
10823 .unwrap();
10824 let route_response = route_task.await.unwrap();
10825 assert!(route_response.is_empty());
10826 let published = client_rx.recv().await.unwrap();
10827 assert!(matches!(
10828 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10829 ClientControlResponse::RouteOpen { .. }
10830 ));
10831 }
10832
10833 #[tokio::test]
10834 async fn route_open_without_consumer_capabilities_relays_none() {
10835 let registry = Arc::new(Registry::default());
10836 let forwarding = Arc::new(ForwardingTable::default());
10837 let handler =
10838 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10839 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
10840 hello_via_sink(
10841 &handler,
10842 &module_ctx,
10843 &mut module_rx,
10844 hello_frame("aft", PROTOCOL_VERSION, 7),
10845 )
10846 .await;
10847
10848 let project_root = unique_project_root("consumer-capabilities-absent");
10849 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
10850 let route_handler = handler.clone();
10851 let route_task = tokio::spawn(async move {
10852 route_handler
10853 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
10854 .await
10855 .unwrap()
10856 });
10857 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10858 .await
10859 .unwrap()
10860 .unwrap();
10861 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10862 let ModuleControlRequest::RouteBind {
10863 consumer_capabilities,
10864 ..
10865 } = bind
10866 else {
10867 panic!("expected route.bind request, got {bind:?}");
10868 };
10869 assert_eq!(consumer_capabilities, None);
10870
10871 handler
10872 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10873 .await
10874 .unwrap();
10875 let route_response = route_task.await.unwrap();
10876 assert!(route_response.is_empty());
10877 let published = client_rx.recv().await.unwrap();
10878 assert!(matches!(
10879 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10880 ClientControlResponse::RouteOpen { .. }
10881 ));
10882 }
10883
10884 async fn bind_role_versions_for(
10888 sent: Option<BTreeMap<String, String>>,
10889 connection: u64,
10890 ) -> Option<BTreeMap<String, String>> {
10891 let registry = Arc::new(Registry::default());
10892 let forwarding = Arc::new(ForwardingTable::default());
10893 let handler =
10894 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10895 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
10896 hello_via_sink(
10897 &handler,
10898 &module_ctx,
10899 &mut module_rx,
10900 hello_frame("aft", PROTOCOL_VERSION, 7),
10901 )
10902 .await;
10903 let project_root = unique_project_root("role-versions");
10904 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
10905 let route_handler = handler.clone();
10906 let route_task = tokio::spawn(async move {
10907 route_handler
10908 .handle_control_frame(
10909 &client_ctx,
10910 route_open_frame_with_role_versions(403, "aft", project_root, sent),
10911 )
10912 .await
10913 .unwrap()
10914 });
10915 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10916 .await
10917 .expect("a well-formed route.open reaches the module as a bind")
10918 .unwrap();
10919 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10920 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
10921 panic!("expected route.bind request, got {bind:?}");
10922 };
10923 handler
10924 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10925 .await
10926 .unwrap();
10927 assert!(route_task.await.unwrap().is_empty());
10928 let published = client_rx.recv().await.unwrap();
10929 assert!(matches!(
10930 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10931 ClientControlResponse::RouteOpen { .. }
10932 ));
10933 role_versions
10934 }
10935
10936 #[tokio::test]
10937 async fn route_open_relays_role_versions_verbatim() {
10938 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
10939 assert_eq!(
10940 bind_role_versions_for(Some(sent.clone()), 141).await,
10941 Some(sent)
10942 );
10943 }
10944
10945 #[tokio::test]
10948 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
10949 assert_eq!(bind_role_versions_for(None, 143).await, None);
10950 assert_eq!(
10951 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
10952 None
10953 );
10954 }
10955
10956 #[tokio::test]
10957 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
10958 let registry = Arc::new(Registry::default());
10959 let forwarding = Arc::new(ForwardingTable::default());
10960 let handler =
10961 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10962 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
10963 hello_via_sink(
10964 &handler,
10965 &module_ctx,
10966 &mut module_rx,
10967 hello_frame("aft", PROTOCOL_VERSION, 7),
10968 )
10969 .await;
10970
10971 let nine: BTreeMap<String, String> = (0..9)
10972 .map(|index| (format!("role-{index}"), "v1".to_string()))
10973 .collect();
10974 for (label, malformed) in [
10975 (
10976 "invalid role name",
10977 role_versions(&[("Tool_Provider", "v1")]),
10978 ),
10979 ("invalid version", role_versions(&[("tool-provider", "v0")])),
10980 ("nine entries", nine),
10981 ] {
10982 let responses = tokio::time::timeout(
10985 Duration::from_secs(1),
10986 handler.handle_control_frame(
10987 &route_ctx(ConnectionId::new(148)).0,
10988 route_open_frame_with_role_versions(
10989 404,
10990 "aft",
10991 unique_project_root("role-versions-malformed"),
10992 Some(malformed),
10993 ),
10994 ),
10995 )
10996 .await
10997 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
10998 .unwrap();
10999 assert_eq!(responses.len(), 1, "{label}");
11000 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
11001 let error = parse_error(&responses[0]);
11002 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
11003 assert_eq!(
11004 error["detail"]["field"], "role_versions",
11005 "{label}: {error}"
11006 );
11007 assert!(
11008 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
11009 "{label}: a malformed declaration is terminal"
11010 );
11011 assert!(
11012 module_rx.try_recv().is_err(),
11013 "{label}: the module must never see a bind"
11014 );
11015 }
11016 }
11017
11018 #[tokio::test]
11022 async fn route_role_versions_capability_is_advertised() {
11023 let handler = ControlHandler::new(Arc::new(Registry::default()));
11024 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
11025 let ack = hello_via_sink(
11026 &handler,
11027 &ctx,
11028 &mut rx,
11029 hello_frame("m", PROTOCOL_VERSION, 1),
11030 )
11031 .await;
11032 let ack = parse_ack(&ack);
11033 assert!(
11034 ack.subc_capabilities
11035 .iter()
11036 .any(|c| c == "route-role-versions/v1"),
11037 "{:?}",
11038 ack.subc_capabilities
11039 );
11040
11041 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11042 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11043 let reply = handler
11044 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
11045 .await
11046 .unwrap()
11047 .pop()
11048 .unwrap();
11049 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11050 serde_json::from_slice(&reply.body).unwrap()
11051 else {
11052 panic!("not a server.describe reply");
11053 };
11054 assert!(
11055 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
11056 "{capabilities:?}"
11057 );
11058 }
11059
11060 #[tokio::test]
11064 async fn catalog_events_capability_is_advertised() {
11065 let handler = ControlHandler::new(Arc::new(Registry::default()));
11066 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11067 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11068 let reply = handler
11069 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
11070 .await
11071 .unwrap()
11072 .pop()
11073 .unwrap();
11074 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11075 serde_json::from_slice(&reply.body).unwrap()
11076 else {
11077 panic!("not a server.describe reply");
11078 };
11079 assert!(
11080 capabilities
11081 .iter()
11082 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
11083 "{capabilities:?}"
11084 );
11085 }
11086
11087 #[tokio::test]
11088 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
11089 let registry = Arc::new(Registry::default());
11090 let forwarding = Arc::new(ForwardingTable::default());
11091 let handler =
11092 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11093 .with_health_probe_timeout(Duration::from_secs(5));
11094 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
11095 hello_via_sink(
11096 &handler,
11097 &module_ctx,
11098 &mut module_rx,
11099 non_routable_hello_frame_with_control_ops(
11100 "mcp",
11101 300,
11102 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11103 ),
11104 )
11105 .await;
11106 assert!(registry
11107 .get_module("mcp")
11108 .unwrap()
11109 .unwrap()
11110 .manifest
11111 .provides
11112 .is_empty());
11113
11114 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
11115 let route_response = handler
11116 .handle_control_frame(
11117 &route_client_ctx,
11118 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
11119 )
11120 .await
11121 .unwrap();
11122 assert_eq!(route_response[0].header.ty, FrameType::Error);
11123 assert_eq!(
11124 parse_error(&route_response[0])["code"],
11125 "target_unavailable"
11126 );
11127 assert!(parse_error(&route_response[0])["message"]
11128 .as_str()
11129 .unwrap()
11130 .contains("does not provide the requested target"));
11131 assert!(module_rx.try_recv().is_err());
11132
11133 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
11134 let health_handler = handler.clone();
11135 let health_task = tokio::spawn(async move {
11136 health_handler
11137 .handle_control_frame(
11138 &health_client_ctx,
11139 supervisor_health_probe_frame(302, "mcp"),
11140 )
11141 .await
11142 .unwrap()
11143 });
11144 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11145 .await
11146 .unwrap()
11147 .unwrap();
11148 assert_eq!(
11149 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11150 ModuleControlRequest::HealthCheck {}
11151 );
11152 handler
11153 .handle_control_frame(
11154 &module_ctx,
11155 health_response(health_frame.header.corr, HealthStatus::Ok),
11156 )
11157 .await
11158 .unwrap();
11159 let health_response = health_task.await.unwrap();
11160 assert_eq!(health_response[0].header.ty, FrameType::Response);
11161 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11162 ClientControlResponse::SupervisorHealthProbe {
11163 module_id, status, ..
11164 } => {
11165 assert_eq!(module_id, "mcp");
11166 assert_eq!(status, HealthStatus::Ok);
11167 }
11168 other => panic!("unexpected health response: {other:?}"),
11169 }
11170
11171 forwarding
11176 .cleanup_connection(module_ctx.connection_id)
11177 .unwrap();
11178 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
11179 let cleanup_response = tokio::time::timeout(
11180 Duration::from_millis(200),
11181 handler.handle_control_frame(
11182 &cleanup_probe_ctx,
11183 supervisor_health_probe_frame(303, "mcp"),
11184 ),
11185 )
11186 .await
11187 .expect("probe should fail immediately when the control lane is gone")
11188 .unwrap();
11189 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
11190 assert_eq!(
11191 parse_error(&cleanup_response[0])["code"],
11192 "target_unavailable"
11193 );
11194 assert!(parse_error(&cleanup_response[0])["message"]
11195 .as_str()
11196 .unwrap()
11197 .contains("no module connection"));
11198
11199 handler
11200 .cleanup_connection(module_ctx.connection_id)
11201 .unwrap();
11202 }
11203
11204 #[tokio::test]
11205 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
11206 let registry = Arc::new(Registry::default());
11207 let supervisor_handle = SupervisorHandle::new();
11208 let supervisor =
11209 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11210 .with_handle(supervisor_handle.clone())
11211 .with_connection_file_path(
11212 std::env::temp_dir()
11213 .join(format!("subc-route-open-warming-{}", std::process::id())),
11214 );
11215 let module = supervisor
11216 .supervise_configured(
11217 ModuleSpec {
11218 module_id: "warming".to_string(),
11219 program: fake_aft_stub_path(),
11220 args: Vec::new(),
11221 env: Vec::new(),
11222 reserved: false,
11223 reserved_prefixes: Vec::new(),
11224 protocol: ModuleProtocol::Subc,
11225 overlap: Default::default(),
11226 },
11227 true,
11228 )
11229 .unwrap();
11230 assert_eq!(module.state().unwrap(), ModuleState::Running);
11231
11232 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11233 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
11234 let response = handler
11235 .handle_control_frame(
11236 &ctx,
11237 route_open_frame(304, "warming", unique_project_root("warming")),
11238 )
11239 .await
11240 .unwrap();
11241 module.stop().await.unwrap();
11242
11243 assert_eq!(response[0].header.ty, FrameType::Error);
11244 let error = parse_error(&response[0]);
11245 assert_eq!(error["code"], "module_warming");
11246 assert!(error["message"]
11247 .as_str()
11248 .unwrap()
11249 .contains("state=running, enabled=true, live=false"));
11250 }
11251
11252 #[test]
11253 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
11254 let handler = ControlHandler::new(Arc::new(Registry::default()));
11255 let capture = EventCapture::default();
11256 let _subscriber =
11257 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11258 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
11259 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
11260 let pending = (0..limit).collect::<Vec<_>>();
11261 let response = handler
11262 .route_open_capacity_refusal(
11263 &ctx,
11264 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
11265 "busy",
11266 pending.len(),
11267 limit,
11268 )
11269 .unwrap();
11270 assert_eq!(parse_error(&response)["code"], "target_unavailable");
11271 let event = capture
11272 .events()
11273 .into_iter()
11274 .find(|event| {
11275 event.target == "control"
11276 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
11277 })
11278 .expect("connection admission refusal event");
11279 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11280 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11281 }
11282
11283 #[test]
11284 fn route_open_target_cap_logs_admission_reason_and_capacity() {
11285 let handler = ControlHandler::new(Arc::new(Registry::default()));
11286 let capture = EventCapture::default();
11287 let _subscriber =
11288 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11289 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
11290 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
11291 let guards = (0..limit)
11292 .map(|_| {
11293 handler
11294 .route_bind_concurrency
11295 .try_admit("busy", limit)
11296 .unwrap()
11297 })
11298 .collect::<Vec<_>>();
11299 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
11300 Err(in_flight) => in_flight,
11301 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
11302 };
11303 let response = handler
11304 .route_open_target_capacity_refusal(
11305 &ctx,
11306 &route_open_frame(397, "busy", unique_project_root("target-cap")),
11307 "busy",
11308 in_flight,
11309 )
11310 .unwrap();
11311 assert_eq!(parse_error(&response)["code"], "target_unavailable");
11312 let event = capture
11313 .events()
11314 .into_iter()
11315 .find(|event| {
11316 event.target == "control"
11317 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
11318 })
11319 .expect("target admission refusal event");
11320 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11321 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11322 drop(guards);
11323 }
11324
11325 #[tokio::test(flavor = "current_thread")]
11326 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
11327 let handler = ControlHandler::new(Arc::new(Registry::default()));
11328 let capture = EventCapture::default();
11329 let _subscriber =
11330 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11331 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
11332 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11333
11334 let _ = handler
11335 .handle_client_control_request(
11336 &ctx,
11337 frame,
11338 ClientControlRequest::SupervisorSetEnabled {
11339 module_id: "broca".to_string(),
11340 enabled: false,
11341 },
11342 )
11343 .await
11344 .unwrap();
11345
11346 let events = capture
11347 .events()
11348 .into_iter()
11349 .filter(|event| {
11350 event.target == "control"
11351 && event.fields.get("message").map(String::as_str)
11352 == Some("supervisor request received")
11353 })
11354 .collect::<Vec<_>>();
11355 assert_eq!(events.len(), 1, "one supervisor request log line");
11356 let event = &events[0];
11357 assert_eq!(event.level, tracing::Level::INFO);
11358 assert_eq!(
11359 event.fields.get("op"),
11360 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
11361 );
11362 assert_eq!(
11363 event.fields.get("module_id"),
11364 Some(&"\"broca\"".to_string())
11365 );
11366 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
11367 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
11368 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
11369 }
11370
11371 #[tokio::test(flavor = "current_thread")]
11372 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
11373 let registry = Arc::new(Registry::default());
11374 let module_connection = ConnectionId::new(303);
11375 registry
11376 .register_with_control_ops(
11377 manifest("aft", PROTOCOL_VERSION),
11378 PROTOCOL_VERSION,
11379 module_connection,
11380 module_baseline_control_ops(),
11381 )
11382 .unwrap();
11383 let handler = ControlHandler::new(registry);
11384 let capture = EventCapture::default();
11385 let _subscriber =
11386 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11387 let (ctx, _rx) = route_ctx(module_connection);
11388 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11389
11390 let _ = handler
11391 .handle_client_control_request(
11392 &ctx,
11393 frame,
11394 ClientControlRequest::SupervisorSetEnabled {
11395 module_id: "broca".to_string(),
11396 enabled: false,
11397 },
11398 )
11399 .await
11400 .unwrap();
11401
11402 let callers = capture
11403 .events()
11404 .into_iter()
11405 .filter(|event| {
11406 event.fields.get("message").map(String::as_str)
11407 == Some("supervisor request received")
11408 })
11409 .map(|event| event.fields.get("caller").cloned())
11410 .collect::<Vec<_>>();
11411 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
11412 }
11413
11414 #[tokio::test(flavor = "current_thread")]
11415 async fn supervisor_list_does_not_log_request_received() {
11416 let handler = ControlHandler::new(Arc::new(Registry::default()));
11417 let capture = EventCapture::default();
11418 let _subscriber =
11419 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11420 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
11421 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
11422
11423 handler
11424 .handle_client_control_request(&ctx, frame, ClientControlRequest::SupervisorList {})
11425 .await
11426 .unwrap();
11427
11428 assert!(capture.events().into_iter().all(|event| {
11429 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11430 }));
11431 }
11432
11433 #[tokio::test(flavor = "current_thread")]
11434 async fn supervisor_rescan_preview_does_not_log_request_received() {
11435 let handler = ControlHandler::new(Arc::new(Registry::default()));
11436 let capture = EventCapture::default();
11437 let _subscriber =
11438 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11439 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
11440 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
11441
11442 handler
11443 .handle_client_control_request(
11444 &ctx,
11445 frame,
11446 ClientControlRequest::SupervisorRescan { preview: true },
11447 )
11448 .await
11449 .unwrap();
11450
11451 assert!(capture.events().into_iter().all(|event| {
11452 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11453 }));
11454 }
11455
11456 struct ApproveOperator;
11457 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
11458 fn prompt(
11459 &self,
11460 _: &str,
11461 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
11462 ) -> crate::operator_confirm::ProviderResult {
11463 crate::operator_confirm::ProviderResult::Approved
11464 }
11465 }
11466
11467 #[tokio::test]
11468 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
11469 assert_eq!(
11470 ControlHandler::default().privileged_operator_prompt,
11471 PrivilegedOperatorPrompt::Log
11472 );
11473 for (initial, changed, running) in [
11474 ("log", "enforce", PrivilegedOperatorPrompt::Log),
11475 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
11476 ] {
11477 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
11478 let path = dir.join("subc.jsonc");
11479 std::fs::write(
11480 &path,
11481 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
11482 )
11483 .unwrap();
11484 let config = crate::daemon_config::load(&path).unwrap().unwrap();
11485 let registry = Arc::new(Registry::default());
11486 let handle = SupervisorHandle::new();
11487 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
11488 .with_handle(handle.clone());
11489 let handler = ControlHandler::new(registry)
11490 .with_supervisor(handle)
11491 .with_privileged_operator_prompt(config.privileged_operator_prompt)
11492 .with_supervisor_rescan(supervisor, &path, config.port);
11493 handler.forwarding.operator_confirms().configure(
11494 Arc::new(ApproveOperator),
11495 crate::operator_confirm::OperatorLimits::default(),
11496 );
11497 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
11498
11499 for (disk_mode, needs_restart) in [
11500 (Some(initial), false),
11501 (Some(changed), true),
11502 (Some(changed), true),
11503 (None, running == PrivilegedOperatorPrompt::Enforce),
11504 (Some(initial), false),
11505 ] {
11506 let mut doc = json!({"version":1});
11507 if let Some(mode) = disk_mode {
11508 doc["privileged_operator_prompt"] = json!(mode);
11509 }
11510 std::fs::write(&path, doc.to_string()).unwrap();
11511 for preview in [true, false] {
11512 let request = ClientControlRequest::SupervisorRescan { preview };
11513 let frame = Frame::build(
11514 FrameType::Request,
11515 control_flags(),
11516 0,
11517 0,
11518 1,
11519 serde_json::to_vec(&request).unwrap(),
11520 )
11521 .unwrap();
11522 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
11523 assert_eq!(frames.len(), 1);
11524 let ClientControlResponse::SupervisorRescan { result } =
11525 serde_json::from_slice(&frames[0].body).unwrap()
11526 else {
11527 panic!("expected a rescan response");
11528 };
11529 let expected = if needs_restart {
11530 vec!["privileged_operator_prompt"]
11531 } else {
11532 Vec::new()
11533 };
11534 assert_eq!(result.restart_required, expected);
11535 assert_eq!(handler.privileged_operator_prompt, running);
11536 assert_eq!(
11537 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
11538 running
11539 );
11540 }
11541 }
11542 }
11543 }
11544
11545 #[tokio::test]
11546 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
11547 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
11548 let path = dir.join("subc.jsonc");
11549 let registry = Arc::new(Registry::default());
11550 let handle = SupervisorHandle::new();
11551 let supervisor =
11552 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
11553 let handler = ControlHandler::new(registry)
11554 .with_supervisor(handle)
11555 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
11556 .with_supervisor_rescan(supervisor, &path, None);
11557 handler.forwarding.operator_confirms().configure(
11558 Arc::new(ApproveOperator),
11559 crate::operator_confirm::OperatorLimits::default(),
11560 );
11561 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
11562 for preview in [true, false] {
11563 std::fs::write(
11564 &path,
11565 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
11566 )
11567 .unwrap();
11568 let request = ClientControlRequest::SupervisorRescan { preview };
11569 let frame = Frame::build(
11570 FrameType::Request,
11571 control_flags(),
11572 0,
11573 0,
11574 1,
11575 serde_json::to_vec(&request).unwrap(),
11576 )
11577 .unwrap();
11578 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
11579 assert_eq!(frames.len(), 1);
11580 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
11581 assert_eq!(
11582 handler.privileged_operator_prompt,
11583 PrivilegedOperatorPrompt::Enforce
11584 );
11585 }
11586 }
11587
11588 #[tokio::test]
11593 async fn route_open_refusal_names_the_check_that_refused() {
11594 let handler = ControlHandler::new(Arc::new(Registry::default()));
11595 let capture = EventCapture::default();
11596 let _subscriber =
11597 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11598 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
11599 let response = handler
11600 .handle_control_frame(
11601 &ctx,
11602 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
11603 )
11604 .await
11605 .unwrap();
11606
11607 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11608 let event = capture
11609 .events()
11610 .into_iter()
11611 .find(|event| {
11612 event.target == "control"
11613 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
11614 })
11615 .expect("route.open refusal event");
11616 assert_eq!(
11617 event.fields.get("reason"),
11618 Some(&"\"not_registered\"".to_string())
11619 );
11620 }
11621
11622 #[tokio::test]
11623 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
11624 let registry = Arc::new(Registry::default());
11625 let supervisor_handle = SupervisorHandle::new();
11626 let supervisor =
11627 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11628 .with_handle(supervisor_handle.clone())
11629 .with_connection_file_path(std::env::temp_dir().join(format!(
11630 "subc-route-open-refusal-info-{}",
11631 std::process::id()
11632 )));
11633 let module = supervisor
11634 .supervise_configured(
11635 ModuleSpec {
11636 module_id: "warming".to_string(),
11637 program: fake_aft_stub_path(),
11638 args: Vec::new(),
11639 env: Vec::new(),
11640 reserved: false,
11641 reserved_prefixes: Vec::new(),
11642 protocol: ModuleProtocol::Subc,
11643 overlap: Default::default(),
11644 },
11645 true,
11646 )
11647 .unwrap();
11648 assert_eq!(module.state().unwrap(), ModuleState::Running);
11649
11650 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11651 assert!(handler
11652 .counters()
11653 .snapshot()
11654 .get("route_open_refused_by_code")
11655 .is_none());
11656 let capture = EventCapture::default();
11657 let _subscriber =
11658 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11659 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
11660 let response = handler
11661 .handle_control_frame(
11662 &ctx,
11663 route_open_frame(394, "warming", unique_project_root("refusal-info")),
11664 )
11665 .await
11666 .unwrap();
11667 module.stop().await.unwrap();
11668
11669 assert_eq!(parse_error(&response[0])["code"], "module_warming");
11670 let event = capture
11671 .events()
11672 .into_iter()
11673 .find(|event| {
11674 event.target == "control"
11675 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
11676 })
11677 .expect("route.open refusal event");
11678 assert_eq!(
11679 event.fields.get("module_id"),
11680 Some(&"\"warming\"".to_string())
11681 );
11682 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
11683 assert_eq!(
11684 event.fields.get("reason"),
11685 Some(&"\"supervised_not_registered\"".to_string())
11686 );
11687 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
11688 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
11689 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
11690 assert_eq!(
11691 handler.counters().snapshot()["route_open_refused_by_code"],
11692 json!({ "module_warming": 1 })
11693 );
11694 }
11695
11696 const OUTAGE_START: &str = "route.open refusing module: not serving";
11697 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
11698
11699 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
11700 capture
11701 .events()
11702 .into_iter()
11703 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
11704 .collect()
11705 }
11706
11707 fn supervise_stub(
11708 registry: &Arc<Registry>,
11709 module_id: &str,
11710 enabled: bool,
11711 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
11712 let supervisor_handle = SupervisorHandle::new();
11713 let supervisor =
11714 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
11715 .with_handle(supervisor_handle.clone())
11716 .with_connection_file_path(std::env::temp_dir().join(format!(
11717 "subc-route-outage-{module_id}-{}",
11718 std::process::id()
11719 )));
11720 let module = supervisor
11721 .supervise_configured(
11722 ModuleSpec {
11723 module_id: module_id.to_string(),
11724 program: fake_aft_stub_path(),
11725 args: Vec::new(),
11726 env: Vec::new(),
11727 reserved: false,
11728 reserved_prefixes: Vec::new(),
11729 protocol: ModuleProtocol::Subc,
11730 overlap: Default::default(),
11731 },
11732 enabled,
11733 )
11734 .unwrap();
11735 (supervisor_handle, module)
11736 }
11737
11738 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
11739 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
11740 module_id: module_id.to_string(),
11741 drain_timeout_ms: Some(50),
11742 })
11743 .unwrap();
11744 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11745 }
11746
11747 #[test]
11751 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
11752 let registry = Arc::new(Registry::default());
11753 let forwarding = Arc::new(ForwardingTable::default());
11754 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11755 let second = ControlHandler::with_forwarding(registry, forwarding);
11756 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
11757 }
11758
11759 #[tokio::test(flavor = "current_thread")]
11763 async fn route_open_unknown_module_refusals_add_no_outage_state() {
11764 let handler = ControlHandler::new(Arc::new(Registry::default()));
11765 let capture = EventCapture::default();
11766 let _subscriber =
11767 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11768 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
11769 for corr in 0..8 {
11770 let response = handler
11771 .handle_control_frame(
11772 &ctx,
11773 route_open_frame(
11774 380 + corr,
11775 &format!("nobody-{corr}"),
11776 unique_project_root("outage-unknown"),
11777 ),
11778 )
11779 .await
11780 .unwrap();
11781 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11782 }
11783
11784 assert_eq!(handler.route_outages.tracked_module_count(), 0);
11785 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
11786 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
11787 }
11788
11789 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11794 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
11795 let registry = Arc::new(Registry::default());
11796 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
11797 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11798 let capture = EventCapture::default();
11799 let _subscriber =
11800 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11801 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
11802 handler.route_outages.record_accepted("outage-restart");
11805
11806 let response = handler
11807 .handle_control_frame(
11808 &ctx,
11809 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
11810 )
11811 .await
11812 .unwrap();
11813 assert_eq!(response[0].header.ty, FrameType::Error);
11814 let starts = outage_lines(&capture, OUTAGE_START);
11815 assert_eq!(starts.len(), 1, "{starts:?}");
11816 assert_eq!(starts[0].level, tracing::Level::WARN);
11817 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
11818 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
11819 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
11820 handler.route_outages.record_accepted("outage-restart");
11821 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
11822
11823 let restart = handler
11824 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
11825 .await
11826 .unwrap();
11827 assert_eq!(
11828 restart[0].header.ty,
11829 FrameType::Response,
11830 "{:?}",
11831 parse_error(&restart[0])
11832 );
11833 handler
11834 .handle_control_frame(
11835 &ctx,
11836 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
11837 )
11838 .await
11839 .unwrap();
11840 module.stop().await.unwrap();
11841
11842 let starts = outage_lines(&capture, OUTAGE_START);
11843 assert_eq!(starts.len(), 2, "{starts:?}");
11844 assert_eq!(starts[1].level, tracing::Level::INFO);
11845 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
11846 }
11847
11848 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11852 async fn failed_operator_restart_leaves_no_operator_mark() {
11853 let registry = Arc::new(Registry::default());
11854 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
11855 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11856 let capture = EventCapture::default();
11857 let _subscriber =
11858 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11859 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
11860 handler.route_outages.record_accepted("outage-disabled");
11861
11862 let restart = handler
11863 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
11864 .await
11865 .unwrap();
11866 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
11867 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
11868
11869 handler
11870 .handle_control_frame(
11871 &ctx,
11872 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
11873 )
11874 .await
11875 .unwrap();
11876 let starts = outage_lines(&capture, OUTAGE_START);
11877 assert_eq!(starts.len(), 1, "{starts:?}");
11878 assert_eq!(starts[0].level, tracing::Level::WARN);
11879 }
11880
11881 #[tokio::test(flavor = "current_thread")]
11882 async fn route_open_unknown_module_escapes_target_module_id() {
11883 let handler = ControlHandler::new(Arc::new(Registry::default()));
11884 let capture = EventCapture::default();
11885 let _subscriber =
11886 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11887 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
11888 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
11889 let response = handler
11890 .handle_control_frame(
11891 &ctx,
11892 route_open_frame(
11893 395,
11894 hostile_module_id,
11895 unique_project_root("hostile-target-module-id"),
11896 ),
11897 )
11898 .await
11899 .unwrap();
11900
11901 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11902 let event = capture
11903 .events()
11904 .into_iter()
11905 .find(|event| {
11906 event.target == "control"
11907 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
11908 })
11909 .expect("route.open unknown-module refusal event");
11910 let logged = event.fields.get("module_id").expect("module_id field");
11911 assert!(!logged.bytes().any(|byte| byte < 0x20));
11912 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
11913 }
11914
11915 #[tokio::test(flavor = "current_thread")]
11916 async fn route_open_module_rejection_uses_daemon_counter_key() {
11917 let registry = Arc::new(Registry::default());
11918 let forwarding = Arc::new(ForwardingTable::default());
11919 let handler =
11920 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11921 let module_connection = ConnectionId::new(95);
11922 let (module_ctx, mut module_rx) = route_ctx(module_connection);
11923 hello_via_sink(
11924 &handler,
11925 &module_ctx,
11926 &mut module_rx,
11927 hello_frame("aft", PROTOCOL_VERSION, 395),
11928 )
11929 .await;
11930
11931 let client_connection = ConnectionId::new(96);
11932 let (client_ctx, _client_rx) = route_ctx(client_connection);
11933 let capture = EventCapture::default();
11934 let _subscriber =
11935 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11936 let (route_task, bind) = relay_route_open(
11937 &handler,
11938 client_connection,
11939 &client_ctx.egress,
11940 &mut module_rx,
11941 396,
11942 "aft",
11943 "hostile-module-code",
11944 )
11945 .await;
11946 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
11947 let rejection = Frame::build(
11948 FrameType::Error,
11949 control_flags(),
11950 0,
11951 0,
11952 bind.header.corr,
11953 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
11954 )
11955 .unwrap();
11956 handler
11957 .handle_control_frame(&module_ctx, rejection)
11958 .await
11959 .unwrap();
11960
11961 let response = route_task.await.unwrap();
11962 assert_eq!(parse_error(&response[0])["code"], hostile_code);
11963 let counters = handler.counters().snapshot();
11964 assert_eq!(
11965 counters["route_open_refused_by_code"],
11966 json!({ "module_rejected": 1 })
11967 );
11968 assert!(counters["route_open_refused_by_code"]
11969 .get(hostile_code)
11970 .is_none());
11971
11972 let event = capture
11973 .events()
11974 .into_iter()
11975 .find(|event| {
11976 event.target == "control"
11977 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
11978 })
11979 .expect("route.open module-rejection refusal event");
11980 let logged = event.fields.get("module_code").expect("module_code field");
11981 assert!(!logged.bytes().any(|byte| byte < 0x20));
11982 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
11983 }
11984
11985 #[tokio::test]
11986 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
11987 let registry = Arc::new(Registry::default());
11988 let supervisor_handle = SupervisorHandle::new();
11989 let missing_program = std::env::temp_dir().join(format!(
11990 "subc-route-open-missing-program-{}",
11991 std::process::id()
11992 ));
11993 let supervisor =
11994 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11995 .with_handle(supervisor_handle.clone());
11996 let module = supervisor
11997 .supervise_configured(
11998 ModuleSpec {
11999 module_id: "failed".to_string(),
12000 program: missing_program,
12001 args: Vec::new(),
12002 env: Vec::new(),
12003 reserved: false,
12004 reserved_prefixes: Vec::new(),
12005 protocol: ModuleProtocol::Subc,
12006 overlap: Default::default(),
12007 },
12008 true,
12009 )
12010 .unwrap();
12011 assert_eq!(module.state().unwrap(), ModuleState::Failed);
12012
12013 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12014 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
12015 let response = handler
12016 .handle_control_frame(
12017 &ctx,
12018 route_open_frame(305, "failed", unique_project_root("failed")),
12019 )
12020 .await
12021 .unwrap();
12022
12023 assert_eq!(response[0].header.ty, FrameType::Error);
12024 let error = parse_error(&response[0]);
12025 assert_eq!(error["code"], "target_unavailable");
12026 assert!(error["message"]
12027 .as_str()
12028 .unwrap()
12029 .contains("state=failed, enabled=true, live=false"));
12030 }
12031
12032 #[tokio::test]
12033 async fn route_open_role_mismatch_remains_target_unavailable() {
12034 let registry = Arc::new(Registry::default());
12035 let handler = ControlHandler::new(Arc::clone(®istry));
12036 handler
12037 .handle_control(
12038 ConnectionId::new(41),
12039 non_routable_hello_frame_with_control_ops("health-only", 306, None),
12040 )
12041 .unwrap();
12042
12043 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
12044 let response = handler
12045 .handle_control_frame(
12046 &ctx,
12047 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
12048 )
12049 .await
12050 .unwrap();
12051
12052 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12053 assert!(parse_error(&response[0])["message"]
12054 .as_str()
12055 .unwrap()
12056 .contains("does not provide the requested target"));
12057 }
12058
12059 #[tokio::test]
12060 async fn route_open_inactive_registration_remains_target_unavailable() {
12061 let registry = Arc::new(Registry::default());
12062 let handler = ControlHandler::new(Arc::clone(®istry));
12063 handler
12064 .handle_control(
12065 ConnectionId::new(43),
12066 hello_frame("inactive", PROTOCOL_VERSION, 308),
12067 )
12068 .unwrap();
12069 assert!(registry
12070 .set_module_state_for_test("inactive", ChannelState::Closed)
12071 .unwrap());
12072
12073 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
12074 let response = handler
12075 .handle_control_frame(
12076 &ctx,
12077 route_open_frame(309, "inactive", unique_project_root("inactive")),
12078 )
12079 .await
12080 .unwrap();
12081
12082 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12083 assert!(parse_error(&response[0])["message"]
12084 .as_str()
12085 .unwrap()
12086 .contains("is not active"));
12087 }
12088
12089 #[tokio::test]
12090 async fn late_health_reply_is_recorded_through_the_module_response_path() {
12091 let registry = Arc::new(Registry::default());
12092 let forwarding = Arc::new(ForwardingTable::default());
12093 let supervisor_handle = SupervisorHandle::new();
12094 let supervisor =
12095 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
12096 .with_forwarding(Arc::clone(&forwarding))
12097 .with_handle(supervisor_handle.clone());
12098 let module = supervisor
12099 .supervise_configured(
12100 crate::ModuleSpec {
12101 module_id: "late-health-response".to_string(),
12102 program: PathBuf::from("disabled-module"),
12103 args: Vec::new(),
12104 env: Vec::new(),
12105 reserved: false,
12106 reserved_prefixes: Vec::new(),
12107 protocol: ModuleProtocol::Subc,
12108 overlap: Default::default(),
12109 },
12110 false,
12111 )
12112 .unwrap();
12113 let handler =
12114 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12115 .with_supervisor(supervisor_handle);
12116 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
12117 handler
12118 .handle_control_frame(
12119 &module_ctx,
12120 hello_frame_with_control_ops(
12121 "late-health-response",
12122 PROTOCOL_VERSION,
12123 7,
12124 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12125 ),
12126 )
12127 .await
12128 .unwrap();
12129 let probe_started_at = Instant::now() - Duration::from_millis(80);
12130 let pending = forwarding
12131 .begin_health_probe_rpc_for(
12132 "late-health-response",
12133 MODULE_CONTROL_OP_HEALTH_CHECK,
12134 probe_started_at,
12135 Instant::now() - Duration::from_millis(1),
12136 )
12137 .unwrap();
12138 assert!(forwarding
12139 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
12140 .unwrap());
12141
12142 let responses = handler
12143 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
12144 .await
12145 .unwrap();
12146
12147 assert!(responses.is_empty());
12148 let health = module.status().unwrap().health;
12149 assert_eq!(health.late_answer_count, 1);
12150 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
12151 }
12152
12153 #[tokio::test]
12154 async fn health_probe_timeout_and_module_death_are_typed() {
12155 let registry = Arc::new(Registry::default());
12156 let forwarding = Arc::new(ForwardingTable::default());
12157 let handler =
12158 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12159 .with_health_probe_timeout(Duration::from_millis(50));
12160 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
12161 hello_via_sink(
12162 &handler,
12163 &module_ctx,
12164 &mut module_rx,
12165 hello_frame_with_control_ops(
12166 "aft",
12167 PROTOCOL_VERSION,
12168 7,
12169 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12170 ),
12171 )
12172 .await;
12173
12174 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12175 let responses = handler
12176 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
12177 .await
12178 .unwrap();
12179 assert_eq!(responses[0].header.ty, FrameType::Error);
12180 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
12181 let _ = module_rx.try_recv();
12182
12183 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
12184 let health_handler = handler.clone();
12185 let death_task = tokio::spawn(async move {
12186 health_handler
12187 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
12188 .await
12189 .unwrap()
12190 });
12191 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12192 .await
12193 .unwrap()
12194 .unwrap();
12195 handler
12196 .cleanup_connection(module_ctx.connection_id)
12197 .unwrap();
12198 let responses = death_task.await.unwrap();
12199 assert_eq!(responses[0].header.ty, FrameType::Error);
12200 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
12201 }
12202
12203 #[test]
12204 fn hello_requires_exact_protocol_version() {
12205 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
12206 let registry = Arc::new(Registry::default());
12207 let handler = ControlHandler::new(Arc::clone(®istry));
12208 let responses = handler
12209 .handle_control(
12210 ConnectionId::new(connection),
12211 hello_frame("aft", offered, 9),
12212 )
12213 .unwrap();
12214
12215 assert_eq!(responses.len(), 1);
12216 assert_eq!(responses[0].header.ty, FrameType::Error);
12217 let error = parse_error(&responses[0]);
12218 assert_eq!(error["code"], "version_unsupported");
12219 assert!(registry.get_module("aft").unwrap().is_none());
12220 assert_eq!(registry.active_registration_count().unwrap(), 0);
12221 }
12222 }
12223
12224 #[test]
12225 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
12226 let registry = Arc::new(Registry::default());
12227 let forwarding = Arc::new(ForwardingTable::default());
12228 let handler =
12229 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12230 let module_connection = ConnectionId::new(301);
12231 let registration = registry
12232 .register_with_control_ops(
12233 manifest("aft-push", PROTOCOL_VERSION),
12234 PROTOCOL_VERSION,
12235 module_connection,
12236 module_baseline_control_ops(),
12237 )
12238 .unwrap();
12239 let (module_tx, _module_rx) = mpsc::channel(8);
12240 let endpoint = forwarding
12241 .register_module_connection(
12242 module_connection,
12243 "aft-push".to_string(),
12244 PROTOCOL_VERSION,
12245 manifest_concurrency(®istration.manifest),
12246 FrameSink::new(module_tx),
12247 )
12248 .unwrap();
12249
12250 let unknown = Frame::build(
12252 FrameType::Push,
12253 control_flags(),
12254 0,
12255 0,
12256 5,
12257 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
12258 )
12259 .unwrap();
12260 let out = handler.handle_status_update(endpoint, unknown).unwrap();
12261 assert!(
12262 out.is_empty(),
12263 "unknown push op must be ignored, got {out:?}"
12264 );
12265
12266 let malformed = Frame::build(
12268 FrameType::Push,
12269 control_flags(),
12270 0,
12271 0,
12272 6,
12273 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
12274 )
12275 .unwrap();
12276 let out = handler.handle_status_update(endpoint, malformed).unwrap();
12277 assert_eq!(out.len(), 1);
12278 assert_eq!(out[0].header.ty, FrameType::Error);
12279 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
12280 }
12281
12282 #[test]
12283 fn hello_rejected_when_connection_already_owns_client_routes() {
12284 let registry = Arc::new(Registry::default());
12285 let forwarding = Arc::new(ForwardingTable::default());
12286 let handler =
12287 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12288 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
12290 let client_connection = ConnectionId::new(202);
12291
12292 let responses = handler
12295 .handle_control(
12296 client_connection,
12297 hello_frame("aft-second", PROTOCOL_VERSION, 9),
12298 )
12299 .unwrap();
12300 assert_eq!(responses[0].header.ty, FrameType::Error);
12301 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
12302 assert!(registry.get_module("aft-second").unwrap().is_none());
12303 }
12304
12305 #[tokio::test]
12306 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
12307 let registry = Arc::new(Registry::default());
12308 let forwarding = Arc::new(ForwardingTable::default());
12309 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
12310 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
12311 hello_via_sink(
12312 &handler,
12313 &module_ctx,
12314 &mut module_rx,
12315 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
12316 )
12317 .await;
12318 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
12319 let pending = forwarding
12320 .begin_route_bind_relay_for_test(
12321 client_ctx.connection_id,
12322 client_ctx.egress.clone(),
12323 2,
12324 "alpha",
12325 )
12326 .unwrap();
12327 forwarding
12328 .complete_pending_relay(
12329 module_ctx.connection_id,
12330 pending.corr,
12331 RouteBindRelayOutcome::Accepted,
12332 )
12333 .unwrap();
12334 client_rx.try_recv().unwrap();
12335 for module_id in ["beta", "alpha"] {
12336 let replies = handler
12337 .handle_control_frame(
12338 &module_ctx,
12339 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
12340 )
12341 .await
12342 .unwrap();
12343 assert_eq!(replies.len(), 1, "second HELLO must be refused");
12344 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
12345 }
12346 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
12347 assert!(registry.get_module("beta").unwrap().is_none());
12348 assert!(matches!(
12349 forwarding
12350 .lookup_data_route(
12351 client_ctx.connection_id,
12352 pending.client_channel,
12353 pending.client_epoch,
12354 )
12355 .unwrap(),
12356 DataRoute::Client(DataRouteState::Bound(_))
12357 ));
12358 assert!(handler
12359 .hello_launch_nonces
12360 .lock()
12361 .unwrap()
12362 .presented(module_ctx.connection_id, Some("alpha-nonce")));
12363 assert!(module_rx.try_recv().is_err());
12364 }
12365
12366 #[test]
12367 fn reserved_module_hello_requires_matching_launch_nonce() {
12368 let registry = Arc::new(Registry::default());
12369 let supervisor = SupervisorHandle::new();
12370 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
12373 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12374
12375 let no_nonce = handler
12377 .handle_control(
12378 ConnectionId::new(1),
12379 hello_frame("vault", PROTOCOL_VERSION, 1),
12380 )
12381 .unwrap();
12382 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
12383 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
12384 assert!(registry.get_module("vault").unwrap().is_none());
12385
12386 let wrong = handler
12388 .handle_control(
12389 ConnectionId::new(2),
12390 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
12391 )
12392 .unwrap();
12393 assert_eq!(wrong[0].header.ty, FrameType::Error);
12394 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
12395 assert!(registry.get_module("vault").unwrap().is_none());
12396
12397 let ok = handler
12399 .handle_control(
12400 ConnectionId::new(3),
12401 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
12402 )
12403 .unwrap();
12404 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
12405 assert!(registry.get_module("vault").unwrap().is_some());
12406 }
12407
12408 #[test]
12409 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
12410 let registry = Arc::new(Registry::default());
12411 let supervisor = SupervisorHandle::new();
12412 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
12413 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
12414 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12415
12416 let squat = handler
12417 .handle_control(
12418 ConnectionId::new(1),
12419 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
12420 )
12421 .unwrap();
12422 assert_eq!(squat[0].header.ty, FrameType::Error);
12423 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
12424 assert!(parse_error(&squat[0])["message"]
12425 .as_str()
12426 .unwrap()
12427 .contains("fed:"));
12428
12429 let accepted_peer = handler
12430 .handle_control(
12431 ConnectionId::new(2),
12432 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
12433 )
12434 .unwrap();
12435 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
12436
12437 let accepted_short = handler
12438 .handle_control(
12439 ConnectionId::new(3),
12440 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
12441 )
12442 .unwrap();
12443 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
12444
12445 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
12446 let response = handler
12447 .handle_control(
12448 ConnectionId::new(conn),
12449 hello_frame(module_id, PROTOCOL_VERSION, conn),
12450 )
12451 .unwrap();
12452 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
12453 }
12454 }
12455
12456 #[test]
12457 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
12458 let registry = Arc::new(Registry::default());
12459 let supervisor = SupervisorHandle::new();
12460 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
12461 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
12462 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
12463 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12464
12465 let owner_nonce = handler
12466 .handle_control(
12467 ConnectionId::new(1),
12468 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
12469 )
12470 .unwrap();
12471 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
12472 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
12473 assert!(registry.get_module("fed:special").unwrap().is_none());
12474
12475 let exact_nonce = handler
12476 .handle_control(
12477 ConnectionId::new(2),
12478 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
12479 )
12480 .unwrap();
12481 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
12482 assert!(registry.get_module("fed:special").unwrap().is_some());
12483 }
12484
12485 #[test]
12486 fn non_reserved_module_ignores_launch_nonce() {
12487 let registry = Arc::new(Registry::default());
12488 let handler = ControlHandler::new(Arc::clone(®istry));
12491 let no_nonce = handler
12492 .handle_control(
12493 ConnectionId::new(1),
12494 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
12495 )
12496 .unwrap();
12497 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
12498 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
12499
12500 let echoed_nonce = handler
12501 .handle_control(
12502 ConnectionId::new(2),
12503 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
12504 )
12505 .unwrap();
12506 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
12507 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
12508 }
12509
12510 #[test]
12511 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
12512 let handler = ControlHandler::default();
12513 let conn = ConnectionId::new(1);
12514 let malformed = Frame::build(
12515 FrameType::Hello,
12516 control_flags(),
12517 0,
12518 0,
12519 3,
12520 b"{not json".to_vec(),
12521 )
12522 .unwrap();
12523
12524 let error = handler.handle_control(conn, malformed).unwrap();
12525 assert_eq!(error[0].header.ty, FrameType::Error);
12526 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
12527
12528 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
12529 let pong = handler.handle_control(conn, ping).unwrap();
12530 assert_eq!(pong[0].header.ty, FrameType::Pong);
12531 assert_eq!(pong[0].header.corr, 4);
12532 }
12533
12534 #[test]
12535 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
12536 let registry = Arc::new(Registry::default());
12537 let handler = ControlHandler::new(Arc::clone(®istry));
12538
12539 handler
12540 .handle_control(
12541 ConnectionId::new(1),
12542 hello_frame("aft", PROTOCOL_VERSION, 1),
12543 )
12544 .unwrap();
12545 let duplicate = handler
12546 .handle_control(
12547 ConnectionId::new(2),
12548 hello_frame("aft", PROTOCOL_VERSION, 2),
12549 )
12550 .unwrap();
12551
12552 assert_eq!(duplicate[0].header.ty, FrameType::Error);
12553 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
12554 let registration = registry.get_module("aft").unwrap().unwrap();
12555 assert_eq!(registration.connection_id, ConnectionId::new(1));
12556 }
12557
12558 #[test]
12559 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
12560 let registry = Arc::new(Registry::default());
12561 let forwarding = Arc::new(ForwardingTable::default());
12562 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
12563 let handler =
12564 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12565 .with_process_liveness(process_liveness);
12566 let (ctx, route_channel, route_epoch) =
12567 bind_liveness_route(®istry, &forwarding, "aft-dead");
12568 let responses = handler
12569 .handle_route_poll(
12570 &ctx,
12571 route_poll_frame(41, PollKind::Liveness, route_channel),
12572 route_channel,
12573 route_epoch,
12574 PollKind::Liveness,
12575 )
12576 .unwrap();
12577
12578 assert_eq!(responses.len(), 1);
12579 assert_eq!(responses[0].header.ty, FrameType::Response);
12580 assert_route_poll_liveness(&responses[0], false);
12581 }
12582
12583 #[test]
12584 fn liveness_poll_without_process_source_uses_bound_route() {
12585 let registry = Arc::new(Registry::default());
12586 let forwarding = Arc::new(ForwardingTable::default());
12587 let handler =
12588 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12589 let (ctx, route_channel, route_epoch) =
12590 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
12591 let responses = handler
12592 .handle_route_poll(
12593 &ctx,
12594 route_poll_frame(42, PollKind::Liveness, route_channel),
12595 route_channel,
12596 route_epoch,
12597 PollKind::Liveness,
12598 )
12599 .unwrap();
12600
12601 assert_route_poll_liveness(&responses[0], true);
12602 }
12603
12604 #[test]
12605 fn liveness_poll_untracked_process_source_uses_bound_route() {
12606 let registry = Arc::new(Registry::default());
12607 let forwarding = Arc::new(ForwardingTable::default());
12608 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
12609 let handler =
12610 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12611 .with_process_liveness(process_liveness);
12612 let (ctx, route_channel, route_epoch) =
12613 bind_liveness_route(®istry, &forwarding, "aft-untracked");
12614 let responses = handler
12615 .handle_route_poll(
12616 &ctx,
12617 route_poll_frame(43, PollKind::Liveness, route_channel),
12618 route_channel,
12619 route_epoch,
12620 PollKind::Liveness,
12621 )
12622 .unwrap();
12623
12624 assert_route_poll_liveness(&responses[0], true);
12625 }
12626
12627 #[tokio::test]
12628 async fn unknown_op_returns_unknown_control_op() {
12629 let handler = ControlHandler::default();
12630 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
12631 let request = Frame::build(
12632 FrameType::Request,
12633 control_flags(),
12634 0,
12635 0,
12636 55,
12637 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
12638 )
12639 .unwrap();
12640
12641 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12642
12643 assert_eq!(response.len(), 1);
12644 assert_eq!(response[0].header.ty, FrameType::Error);
12645 assert_eq!(response[0].header.corr, 55);
12646 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
12647 }
12648
12649 #[tokio::test]
12650 async fn supervisor_provenance_rejects_unknown_exact_module() {
12651 let handler = ControlHandler::default();
12652 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
12653 let request = Frame::build(
12654 FrameType::Request,
12655 control_flags(),
12656 0,
12657 0,
12658 57,
12659 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
12660 )
12661 .unwrap();
12662
12663 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12664
12665 assert_eq!(response.len(), 1);
12666 assert_eq!(response[0].header.ty, FrameType::Error);
12667 assert_eq!(response[0].header.corr, 57);
12668 let error = parse_error(&response[0]);
12669 assert_eq!(error["code"], "unknown_module");
12670 assert_eq!(error["message"], "module_id 'missing' is not supervised");
12671 }
12672
12673 #[test]
12674 fn provenance_probe_override_keeps_handler_tests_deterministic() {
12675 let expected = subc_control::RunningImageAgreement::Unavailable {
12676 reason: subc_control::RunningImageUnavailableReason::HashFailed,
12677 };
12678 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
12679 assert_eq!(handler.provenance_probe_override, Some(expected));
12680 }
12681
12682 #[test]
12683 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
12684 let verdict = reload_verdict(
12685 std::path::Path::new("/bin/new"),
12686 Some(std::path::Path::new("/bin/old")),
12687 subc_control::RunningImageAgreement::Unavailable {
12688 reason: subc_control::RunningImageUnavailableReason::HashFailed,
12689 },
12690 );
12691 assert!(matches!(
12692 verdict.path,
12693 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
12694 if configured == std::path::Path::new("/bin/new")
12695 && spawned_from == std::path::Path::new("/bin/old")
12696 ));
12697 }
12698
12699 #[cfg(windows)]
12700 #[test]
12701 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
12702 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
12703 let configured = dir.join("module.exe");
12704 std::fs::write(&configured, b"executable path fixture").unwrap();
12705 let spawned = std::fs::canonicalize(&configured).unwrap();
12706 assert_ne!(
12707 configured, spawned,
12708 "fixture must exercise different path spellings"
12709 );
12710 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
12711 let lower_case = configured.to_string_lossy().to_lowercase();
12712 for spelling in [
12713 &configured,
12714 Path::new(&forward_slashes),
12715 Path::new(&lower_case),
12716 ] {
12717 let verdict = reload_verdict(
12718 spelling,
12719 Some(&spawned),
12720 subc_control::RunningImageAgreement::Unavailable {
12721 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12722 },
12723 );
12724 assert_eq!(
12725 verdict.path,
12726 ReloadPathAgreement::Match,
12727 "configured {spelling:?}, spawned {spawned:?}"
12728 );
12729 }
12730 }
12731
12732 #[cfg(windows)]
12733 #[test]
12734 fn windows_reload_path_keeps_different_existing_program_pending() {
12735 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
12736 let old = dir.join("old.exe");
12737 let new = dir.join("new.exe");
12738 std::fs::write(&old, b"same executable content").unwrap();
12739 std::fs::copy(&old, &new).unwrap();
12740 let spawned = std::fs::canonicalize(&old).unwrap();
12741 let image = subc_control::RunningImageAgreement::Unavailable {
12742 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12743 };
12744 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
12745 assert!(
12746 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
12747 if configured == new && spawned_from == spawned)
12748 );
12749 let missing = dir.join("missing.exe");
12750 assert!(matches!(
12751 reload_verdict(&missing, Some(&spawned), image).path,
12752 ReloadPathAgreement::Mismatch { .. }
12753 ));
12754 }
12755
12756 #[test]
12757 fn reload_verdict_detects_replaced_image_at_same_path() {
12758 let image = subc_control::RunningImageAgreement::Mismatch {
12759 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
12760 digest: "old".into(),
12761 },
12762 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
12763 digest: "new".into(),
12764 },
12765 };
12766 let verdict = reload_verdict(
12767 std::path::Path::new("/bin/same"),
12768 Some(std::path::Path::new("/bin/same")),
12769 image.clone(),
12770 );
12771 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
12772 assert_eq!(verdict.image, image);
12773 }
12774
12775 #[test]
12776 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
12777 let image = subc_control::RunningImageAgreement::Unavailable {
12778 reason: subc_control::RunningImageUnavailableReason::NotRunning,
12779 };
12780 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
12781 assert_eq!(
12782 verdict.path,
12783 subc_control::ReloadPathAgreement::Unavailable {
12784 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
12785 }
12786 );
12787 assert_eq!(verdict.image, image);
12788
12789 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
12790 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12791 };
12792 let verdict = reload_verdict(
12793 std::path::Path::new("/bin/same"),
12794 Some(std::path::Path::new("/bin/same")),
12795 unconfirmed.clone(),
12796 );
12797 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
12798 assert_eq!(verdict.image, unconfirmed);
12799 }
12800
12801 #[test]
12802 fn reload_verdict_preserves_each_image_unavailability_reason() {
12803 use subc_control::RunningImageUnavailableReason as Reason;
12804
12805 for reason in [
12806 Reason::NotRunning,
12807 Reason::UnsupportedPlatform,
12808 Reason::RunningExecutableUnreadable,
12809 Reason::SpawnedPathUnreadable,
12810 Reason::HashFailed,
12811 Reason::ProcessIdentityUnconfirmed,
12812 Reason::Unknown("future_probe_reason".to_string()),
12813 ] {
12814 let image = subc_control::RunningImageAgreement::Unavailable {
12815 reason: reason.clone(),
12816 };
12817 let verdict = reload_verdict(
12818 std::path::Path::new("/bin/same"),
12819 Some(std::path::Path::new("/bin/same")),
12820 image.clone(),
12821 );
12822 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
12823 assert_eq!(verdict.image, image, "{reason:?}");
12824 }
12825 }
12826
12827 #[tokio::test]
12828 async fn malformed_control_bodies_return_invalid_control_body() {
12829 let handler = ControlHandler::default();
12830 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
12831
12832 for (corr, body) in [
12833 (56, br#"{"route_channel":1}"#.as_slice()),
12834 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
12835 (
12836 58,
12837 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
12838 ),
12839 ] {
12840 let request = Frame::build(
12841 FrameType::Request,
12842 control_flags(),
12843 0,
12844 0,
12845 corr,
12846 body.to_vec(),
12847 )
12848 .unwrap();
12849 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12850
12851 assert_eq!(response.len(), 1);
12852 assert_eq!(response[0].header.ty, FrameType::Error);
12853 assert_eq!(response[0].header.corr, corr);
12854 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
12855 }
12856 }
12857
12858 #[tokio::test]
12859 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
12860 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12861 let registry = Arc::new(Registry::default());
12862 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
12863 let router = Router::with_control_handler(Arc::clone(&control));
12864 let connection = router.begin_connection();
12865 let (ctx, mut rx) = route_ctx(connection.id());
12866
12867 router
12868 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
12869 .await
12870 .unwrap();
12871 let response = rx.recv().await.unwrap();
12872 let ack = parse_ack(&response);
12873 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
12874 let channel = 1;
12875
12876 let goodbye =
12877 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
12878 router.route_for_connection(&ctx, goodbye).await.unwrap();
12879 assert!(rx.try_recv().is_err());
12880 assert!(registry.get_module("aft").unwrap().is_none());
12881
12882 router
12883 .route_for_connection(&ctx, channel_request(channel, 13))
12884 .await
12885 .unwrap();
12886 let error_frame = rx.recv().await.unwrap();
12887 assert_eq!(error_frame.header.ty, FrameType::Error);
12888 assert_eq!(error_frame.header.channel, channel);
12889 let captured = crate::router::test_log::captured_logs(&logs);
12890 assert_eq!(
12891 captured
12892 .lines()
12893 .filter(|line| {
12894 line.contains("module_id=aft")
12895 && line.contains("reason=explicit_goodbye")
12896 && line.contains("module registration ended")
12897 })
12898 .count(),
12899 1,
12900 "unexpected GOODBYE registry log: {captured}"
12901 );
12902 }
12903
12904 #[tokio::test]
12905 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
12906 let registry = Arc::new(Registry::default());
12907 let handler = ControlHandler::new(registry).with_capability_config(
12908 [("prov".to_string(), true), ("cons".to_string(), true)],
12909 BTreeMap::new(),
12910 );
12911 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
12912 register_capability_manifest(
12913 &handler,
12914 &provider_ctx,
12915 &mut provider_rx,
12916 capability_manifest("prov", &["thing/v1"], &[]),
12917 1,
12918 )
12919 .await;
12920 let mut consumer = capability_manifest("cons", &[], &[]);
12921 consumer.capabilities.as_mut().unwrap().requires.push(
12922 subc_protocol::manifest::CapabilityRequirement {
12923 capability: "thing/v1".to_string(),
12924 need: subc_protocol::manifest::CapabilityNeed::Required,
12925 },
12926 );
12927 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
12928 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
12929 assert_eq!(
12930 handler.capability_evaluator.verdict("cons", "thing/v1"),
12931 Some(CapabilityVerdict::Provided)
12932 );
12933 let (mut client_rx, _) = open_route_for_capability_test(
12934 &handler,
12935 &provider_ctx,
12936 &mut provider_rx,
12937 703,
12938 3,
12939 "prov",
12940 None,
12941 )
12942 .await;
12943 let goodbye =
12944 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
12945 handler
12946 .handle_control_frame(&provider_ctx, goodbye)
12947 .await
12948 .unwrap();
12949 assert_eq!(
12950 handler.capability_evaluator.verdict("cons", "thing/v1"),
12951 Some(CapabilityVerdict::NeverProvided)
12952 );
12953 let closed = client_rx
12954 .try_recv()
12955 .expect("GOODBYE pushes route.closed before route GOODBYE");
12956 assert!(
12957 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
12958 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
12959 );
12960 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
12961 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
12962 }
12963
12964 struct RouteClosedFixture {
12965 handler: ControlHandler,
12966 #[cfg(unix)]
12967 supervisor: Supervisor,
12968 module_ctx: RouteCtx,
12969 #[cfg(unix)]
12970 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12971 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12972 route: Arc<crate::forwarding::RouteBinding>,
12973 }
12974
12975 async fn route_closed_fixture() -> RouteClosedFixture {
12976 let registry = Arc::new(Registry::default());
12977 let handler = ControlHandler::new(Arc::clone(®istry));
12978 #[cfg(unix)]
12979 let supervisor = Supervisor::new(registry, RestartPolicy::default())
12980 .with_forwarding(Arc::clone(&handler.forwarding));
12981 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
12982 register_capability_manifest(
12983 &handler,
12984 &module_ctx,
12985 &mut module_rx,
12986 capability_manifest("prov", &[], &[]),
12987 1,
12988 )
12989 .await;
12990 let (client_rx, bind) = open_route_for_capability_test(
12991 &handler,
12992 &module_ctx,
12993 &mut module_rx,
12994 702,
12995 2,
12996 "prov",
12997 None,
12998 )
12999 .await;
13000 let ModuleControlRequest::RouteBind {
13001 route_channel: channel,
13002 epoch: route_epoch,
13003 ..
13004 } = bind
13005 else {
13006 panic!("expected route.bind");
13007 };
13008 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
13009 handler
13010 .forwarding
13011 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
13012 .unwrap()
13013 else {
13014 panic!("the acknowledged route must be bound");
13015 };
13016 RouteClosedFixture {
13017 handler,
13018 #[cfg(unix)]
13019 supervisor,
13020 module_ctx,
13021 #[cfg(unix)]
13022 module_rx,
13023 client_rx,
13024 route,
13025 }
13026 }
13027
13028 impl RouteClosedFixture {
13029 #[cfg(unix)]
13030 async fn cut(&mut self) {
13031 let notices = async {
13032 let mut draining = self.module_rx.recv().await.unwrap();
13033 assert!(matches!(
13034 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
13035 &draining.body
13036 )
13037 .unwrap(),
13038 subc_protocol::session::ModuleControlCommand::Draining {
13039 reason: RouteCloseReason::Restart,
13040 ..
13041 }
13042 ));
13043 draining.flushed.take().unwrap().send(()).unwrap();
13044 let mut closing = self.client_rx.recv().await.unwrap();
13045 assert!(matches!(
13046 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
13047 ClientControlPush::RouteClosing {
13048 reason: RouteCloseReason::Restart,
13049 ..
13050 }
13051 ));
13052 closing.flushed.take().unwrap().send(()).unwrap();
13053 };
13054 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
13055 result.unwrap();
13056 }
13057
13058 fn disconnect(&mut self) -> ClientControlPush {
13059 self.handler
13060 .cleanup_connection(self.module_ctx.connection_id)
13061 .unwrap();
13062 let closed = self.client_rx.try_recv().unwrap();
13063 assert_eq!(closed.header.ty, FrameType::Push);
13064 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
13065 assert!(matches!(
13066 &push,
13067 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
13068 if channels == &[self.route.client_channel]
13069 ));
13070 assert_eq!(
13071 self.client_rx.try_recv().unwrap().header.ty,
13072 FrameType::Goodbye
13073 );
13074 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
13075 push
13076 }
13077 }
13078
13079 #[cfg(unix)]
13080 #[tokio::test(start_paused = true)]
13081 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
13082 let mut fixture = route_closed_fixture().await;
13083 fixture.cut().await;
13084 assert!(matches!(
13085 fixture.disconnect(),
13086 ClientControlPush::RouteClosed {
13087 reason: RouteCloseReason::Restart,
13088 drained: true,
13089 ..
13090 }
13091 ));
13092 }
13093
13094 #[cfg(unix)]
13095 #[tokio::test(start_paused = true)]
13096 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
13097 let mut fixture = route_closed_fixture().await;
13098 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
13099 let started = tokio::time::Instant::now();
13100 fixture.cut().await;
13101 assert!(started.elapsed() >= Duration::from_secs(2));
13102 assert_eq!(fixture.route.flow.in_flight(), 1);
13103 assert!(matches!(
13104 fixture.disconnect(),
13105 ClientControlPush::RouteClosed {
13106 reason: RouteCloseReason::Restart,
13107 drained: false,
13108 ..
13109 }
13110 ));
13111 }
13112
13113 #[tokio::test]
13114 async fn module_connection_crash_reports_crash_and_not_drained() {
13115 let mut fixture = route_closed_fixture().await;
13116 assert!(matches!(
13120 fixture.disconnect(),
13121 ClientControlPush::RouteClosed {
13122 reason: RouteCloseReason::Crash,
13123 drained: false,
13124 ..
13125 }
13126 ));
13127 }
13128
13129 #[tokio::test]
13130 async fn dropping_router_connection_releases_registration() {
13131 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13132 let registry = Arc::new(Registry::default());
13133 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
13134 let router = Router::with_control_handler(Arc::clone(&control));
13135 let connection = router.begin_connection();
13136 let connection_id = connection.id();
13137 let (ctx, mut rx) = route_ctx(connection_id);
13138
13139 router
13140 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
13141 .await
13142 .unwrap();
13143 let response = rx.recv().await.unwrap();
13144 let ack = parse_ack(&response);
13145 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13146 assert!(registry.get_module("aft").unwrap().is_some());
13147
13148 drop(connection);
13149
13150 assert!(registry.get_module("aft").unwrap().is_none());
13151 assert_eq!(registry.active_registration_count().unwrap(), 0);
13152
13153 control
13154 .cleanup_connection(ConnectionId::new(u64::MAX))
13155 .unwrap();
13156 let captured = crate::router::test_log::captured_logs(&logs);
13157 println!("captured registry lifecycle logs:\n{captured}");
13158 let events: Vec<_> = captured
13159 .lines()
13160 .filter(|line| {
13161 line.contains("module registered module_id=aft ")
13162 || line.contains("module registration ended")
13163 })
13164 .collect();
13165 assert_eq!(
13166 events.len(),
13167 2,
13168 "unexpected registry lifecycle logs: {captured}"
13169 );
13170 assert!(events[0].contains("module registered module_id=aft "));
13171 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
13172 assert!(events[1].contains(&format!(
13173 "module_id=aft connection_id={}",
13174 connection_id.get()
13175 )));
13176 assert!(events[1].contains("reason=connection_closed"));
13177 assert!(events[1].contains("module registration ended"));
13178 assert_eq!(
13179 captured
13180 .lines()
13181 .filter(|line| line.contains("module registered module_id=aft "))
13182 .count(),
13183 1,
13184 "legacy registration admission line must appear once: {captured}"
13185 );
13186 }
13187
13188 #[tokio::test]
13189 async fn hello_registration_keeps_legacy_module_registered_line_once() {
13190 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13191 let registry = Arc::new(Registry::default());
13192 let control = ControlHandler::new(Arc::clone(®istry));
13193 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
13194 hello_via_sink(
13195 &control,
13196 &ctx,
13197 &mut rx,
13198 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
13199 )
13200 .await;
13201
13202 let captured = crate::router::test_log::captured_logs(&logs);
13203 let admissions: Vec<_> = captured
13204 .lines()
13205 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
13206 .collect();
13207 assert_eq!(
13208 admissions.len(),
13209 1,
13210 "legacy admission line must remain exactly once: {captured}"
13211 );
13212 assert!(admissions[0].contains("routable_provider=true"));
13213 assert!(admissions[0].contains("connection_id=777"));
13214 }
13215
13216 fn capability_manifest(
13217 module_id: &str,
13218 provides: &[&str],
13219 must_never_reach: &[&str],
13220 ) -> ModuleManifest {
13221 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
13222 manifest.capabilities = Some(CapabilityDeclarations {
13223 provides: provides
13224 .iter()
13225 .map(|capability| (*capability).to_string())
13226 .collect(),
13227 requires: Vec::new(),
13228 must_never_reach: must_never_reach
13229 .iter()
13230 .map(|capability| (*capability).to_string())
13231 .collect(),
13232 });
13233 manifest
13234 }
13235
13236 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
13237 Frame::build(
13238 FrameType::Hello,
13239 control_flags(),
13240 0,
13241 0,
13242 corr,
13243 serde_json::to_vec(&ModuleHelloBody {
13244 protocol_ver: manifest.protocol_ver,
13245 manifest,
13246 control_ops: None,
13247 launch_nonce: None,
13248 })
13249 .expect("capability test HELLO serializes"),
13250 )
13251 .expect("capability test HELLO frame builds")
13252 }
13253
13254 fn catalog_update_with_capabilities_frame(
13255 corr: u64,
13256 capabilities: CapabilityDeclarations,
13257 ) -> Frame {
13258 Frame::build(
13259 FrameType::Request,
13260 control_flags(),
13261 0,
13262 0,
13263 corr,
13264 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
13265 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
13266 capabilities: Some(capabilities),
13267 ready: None,
13268 })
13269 .expect("capability catalog.update serializes"),
13270 )
13271 .expect("capability catalog.update frame builds")
13272 }
13273
13274 async fn register_capability_manifest(
13275 handler: &ControlHandler,
13276 ctx: &RouteCtx,
13277 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13278 manifest: ModuleManifest,
13279 corr: u64,
13280 ) {
13281 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
13282 }
13283
13284 async fn open_route_for_capability_test(
13285 handler: &ControlHandler,
13286 target_ctx: &RouteCtx,
13287 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13288 client_connection_id: u64,
13289 corr: u64,
13290 target_module_id: &str,
13291 consumer_identity: Option<ConsumerIdentity>,
13292 ) -> (
13293 mpsc::Receiver<crate::router::OutboundFrame>,
13294 ModuleControlRequest,
13295 ) {
13296 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
13297 let route_handler = handler.clone();
13298 let target_module_id = target_module_id.to_string();
13299 let route_task = tokio::spawn(async move {
13300 route_handler
13301 .handle_control_frame(
13302 &client_ctx,
13303 route_open_frame_with_admission_facts(
13304 corr,
13305 &target_module_id,
13306 unique_project_root("admission-facts"),
13307 consumer_identity,
13308 None,
13309 ),
13310 )
13311 .await
13312 .expect("capability test route.open succeeds")
13313 });
13314 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
13315 .await
13316 .expect("capability test route.open must reach route.bind")
13317 .expect("target control receiver stays open");
13318 let bind_request: ModuleControlRequest =
13319 serde_json::from_slice(&bind.body).expect("route.bind decodes");
13320 handler
13321 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
13322 .await
13323 .expect("capability test route.bind ACK succeeds");
13324 assert!(route_task.await.expect("route.open task joins").is_empty());
13325 let opened = client_rx
13326 .recv()
13327 .await
13328 .expect("successful route.open publishes a response");
13329 assert!(matches!(
13330 serde_json::from_slice::<ClientControlResponse>(&opened.body),
13331 Ok(ClientControlResponse::RouteOpen { .. })
13332 ));
13333 (client_rx, bind_request)
13334 }
13335
13336 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
13337 assert_eq!(frame.header.ty, FrameType::Push);
13338 assert_eq!(frame.header.channel, 0);
13339 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
13340 .expect("route.closed control push decodes");
13341 let ClientControlPush::RouteClosed { channels, .. } = &push else {
13342 panic!("expected route.closed");
13343 };
13344 assert_eq!(channels.len(), 1, "exactly one violating route closed");
13345 let channels = channels.clone();
13346 assert_eq!(
13347 push,
13348 ClientControlPush::RouteClosed {
13349 module_id: target_module_id.to_string(),
13350 channels,
13351 reason: RouteCloseReason::CapabilityDenied,
13352 drained: false,
13353 abandoned: 0,
13354 excluded_subscriptions: 0,
13355 terminal: Some(false),
13356 }
13357 );
13358 }
13359
13360 #[tokio::test]
13361 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
13362 let registry = Arc::new(Registry::default());
13363 let forwarding = Arc::new(ForwardingTable::default());
13364 let supervisor = SupervisorHandle::new();
13365 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13366 let handler =
13367 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13368 .with_supervisor(supervisor);
13369 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
13370 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
13371 register_capability_manifest(
13372 &handler,
13373 &target_ctx,
13374 &mut target_rx,
13375 capability_manifest("target", &["credentials-provider/v1"], &[]),
13376 1,
13377 )
13378 .await;
13379 register_capability_manifest(
13380 &handler,
13381 &opener_ctx,
13382 &mut opener_rx,
13383 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13384 2,
13385 )
13386 .await;
13387
13388 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
13389 let replies = handler
13390 .handle_control_frame(
13391 &client_ctx,
13392 route_open_frame_with_admission_facts(
13393 3,
13394 "target",
13395 unique_project_root("admission-facts"),
13396 Some(ConsumerIdentity {
13397 module_id: "opener".to_string(),
13398 launch_nonce: "opener-nonce".to_string(),
13399 }),
13400 None,
13401 ),
13402 )
13403 .await
13404 .expect("denied route.open returns a typed frame");
13405 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
13406 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13407 assert!(
13408 target_rx.try_recv().is_err(),
13409 "forbidden route.open must not relay route.bind"
13410 );
13411 }
13412
13413 #[tokio::test]
13414 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
13415 let registry = Arc::new(Registry::default());
13416 let forwarding = Arc::new(ForwardingTable::default());
13417 let supervisor = SupervisorHandle::new();
13418 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13419 let handler =
13420 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13421 .with_supervisor(supervisor);
13422 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
13423 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
13424 register_capability_manifest(
13425 &handler,
13426 &target_ctx,
13427 &mut target_rx,
13428 capability_manifest("target", &["credentials-provider/v1"], &[]),
13429 1,
13430 )
13431 .await;
13432 register_capability_manifest(
13433 &handler,
13434 &old_opener_ctx,
13435 &mut old_opener_rx,
13436 capability_manifest("opener", &[], &[]),
13437 2,
13438 )
13439 .await;
13440 let (mut client_rx, _) = open_route_for_capability_test(
13441 &handler,
13442 &target_ctx,
13443 &mut target_rx,
13444 712,
13445 3,
13446 "target",
13447 Some(ConsumerIdentity {
13448 module_id: "opener".to_string(),
13449 launch_nonce: "opener-nonce".to_string(),
13450 }),
13451 )
13452 .await;
13453 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13454
13455 handler
13456 .cleanup_connection(old_opener_ctx.connection_id)
13457 .expect("old opener registration cleans up");
13458 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
13459 register_capability_manifest(
13460 &handler,
13461 &new_opener_ctx,
13462 &mut new_opener_rx,
13463 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13464 4,
13465 )
13466 .await;
13467
13468 assert_capability_denied_push(
13469 client_rx
13470 .try_recv()
13471 .expect("HELLO deny addition must emit route.closed")
13472 .frame,
13473 "target",
13474 );
13475 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13476 assert!(matches!(
13477 target_rx.try_recv(),
13478 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
13479 ));
13480 }
13481
13482 #[tokio::test]
13483 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
13484 let registry = Arc::new(Registry::default());
13485 let forwarding = Arc::new(ForwardingTable::default());
13486 let supervisor = SupervisorHandle::new();
13487 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13488 let handler =
13489 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13490 .with_supervisor(supervisor);
13491 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
13492 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
13493 register_capability_manifest(
13494 &handler,
13495 &target_ctx,
13496 &mut target_rx,
13497 capability_manifest("target", &[], &[]),
13498 1,
13499 )
13500 .await;
13501 register_capability_manifest(
13502 &handler,
13503 &opener_ctx,
13504 &mut opener_rx,
13505 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13506 2,
13507 )
13508 .await;
13509 let (mut client_rx, _) = open_route_for_capability_test(
13510 &handler,
13511 &target_ctx,
13512 &mut target_rx,
13513 722,
13514 3,
13515 "target",
13516 Some(ConsumerIdentity {
13517 module_id: "opener".to_string(),
13518 launch_nonce: "opener-nonce".to_string(),
13519 }),
13520 )
13521 .await;
13522 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13523
13524 let replies = handler
13525 .handle_control_frame(
13526 &target_ctx,
13527 catalog_update_with_capabilities_frame(
13528 4,
13529 CapabilityDeclarations {
13530 provides: vec!["credentials-provider/v1".to_string()],
13531 requires: Vec::new(),
13532 must_never_reach: Vec::new(),
13533 },
13534 ),
13535 )
13536 .await
13537 .expect("claim catalog.update succeeds");
13538 assert!(matches!(
13539 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
13540 Ok(ModuleControlResponseToModule::CatalogUpdate {})
13541 ));
13542 assert_capability_denied_push(
13543 client_rx
13544 .try_recv()
13545 .expect("claim addition must emit route.closed")
13546 .frame,
13547 "target",
13548 );
13549 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13550 assert!(matches!(
13551 target_rx.try_recv(),
13552 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
13553 ));
13554 }
13555
13556 #[tokio::test]
13557 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
13558 let registry = Arc::new(Registry::default());
13559 let forwarding = Arc::new(ForwardingTable::default());
13560 let supervisor = SupervisorHandle::new();
13561 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13562 let handler =
13563 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13564 .with_supervisor(supervisor);
13565 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
13566 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
13567 register_capability_manifest(
13568 &handler,
13569 &target_ctx,
13570 &mut target_rx,
13571 capability_manifest("target", &["credentials-provider/v1"], &[]),
13572 1,
13573 )
13574 .await;
13575 register_capability_manifest(
13576 &handler,
13577 &opener_ctx,
13578 &mut opener_rx,
13579 capability_manifest("opener", &[], &[]),
13580 2,
13581 )
13582 .await;
13583 let (mut client_rx, _) = open_route_for_capability_test(
13584 &handler,
13585 &target_ctx,
13586 &mut target_rx,
13587 732,
13588 3,
13589 "target",
13590 Some(ConsumerIdentity {
13591 module_id: "opener".to_string(),
13592 launch_nonce: "opener-nonce".to_string(),
13593 }),
13594 )
13595 .await;
13596 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13597
13598 handler
13599 .handle_control_frame(
13600 &target_ctx,
13601 catalog_update_with_capabilities_frame(
13602 4,
13603 CapabilityDeclarations {
13604 provides: Vec::new(),
13605 requires: Vec::new(),
13606 must_never_reach: Vec::new(),
13607 },
13608 ),
13609 )
13610 .await
13611 .expect("claim removal catalog.update succeeds");
13612 assert_eq!(
13613 forwarding.active_binding_count().unwrap(),
13614 1,
13615 "removing an attested target claim must leave the route census unchanged"
13616 );
13617 assert!(
13618 client_rx.try_recv().is_err(),
13619 "claim removal must not emit route.closed capability_denied"
13620 );
13621 assert!(
13622 target_rx.try_recv().is_err(),
13623 "claim removal must not send the target a route GOODBYE"
13624 );
13625 }
13626
13627 #[tokio::test]
13630 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
13631 let registry = Arc::new(Registry::default());
13632 let forwarding = Arc::new(ForwardingTable::default());
13633 let supervisor = SupervisorHandle::new();
13634 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13635 let handler =
13636 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13637 .with_supervisor(supervisor);
13638 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
13639 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
13640 register_capability_manifest(
13641 &handler,
13642 &target_ctx,
13643 &mut target_rx,
13644 capability_manifest("target", &["credentials-provider/v1"], &[]),
13645 1,
13646 )
13647 .await;
13648 register_capability_manifest(
13649 &handler,
13650 &opener_ctx,
13651 &mut opener_rx,
13652 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13653 2,
13654 )
13655 .await;
13656
13657 let (_client_rx, bind) = open_route_for_capability_test(
13658 &handler,
13659 &target_ctx,
13660 &mut target_rx,
13661 742,
13662 3,
13663 "target",
13664 None,
13665 )
13666 .await;
13667 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
13668 panic!("direct scope-honesty route must bind");
13669 };
13670 assert_eq!(principal, Some(Principal::Direct));
13671 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13672 }
13673
13674 #[tokio::test]
13677 async fn must_never_reach_self_route_is_capability_forbidden() {
13678 let registry = Arc::new(Registry::default());
13679 let forwarding = Arc::new(ForwardingTable::default());
13680 let supervisor = SupervisorHandle::new();
13681 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
13682 let handler =
13683 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13684 .with_supervisor(supervisor);
13685 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
13686 register_capability_manifest(
13687 &handler,
13688 &self_ctx,
13689 &mut self_rx,
13690 capability_manifest(
13691 "self-provider",
13692 &["credentials-provider/v1"],
13693 &["credentials-provider/v1"],
13694 ),
13695 1,
13696 )
13697 .await;
13698
13699 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
13700 let replies = handler
13701 .handle_control_frame(
13702 &client_ctx,
13703 route_open_frame_with_admission_facts(
13704 2,
13705 "self-provider",
13706 unique_project_root("admission-facts"),
13707 Some(ConsumerIdentity {
13708 module_id: "self-provider".to_string(),
13709 launch_nonce: "self-nonce".to_string(),
13710 }),
13711 None,
13712 ),
13713 )
13714 .await
13715 .expect("self-route refusal returns a typed frame");
13716 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
13717 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13718 assert!(
13719 self_rx.try_recv().is_err(),
13720 "self denial must not relay route.bind"
13721 );
13722 }
13723
13724 #[test]
13725 fn unsupported_channel_zero_frame_returns_error() {
13726 let handler = ControlHandler::default();
13727 let request = Frame::build(
13728 FrameType::Request,
13729 control_flags(),
13730 0,
13731 0,
13732 21,
13733 b"opaque".to_vec(),
13734 )
13735 .unwrap();
13736
13737 let response = handler
13738 .handle_control(ConnectionId::new(1), request)
13739 .unwrap();
13740
13741 assert_eq!(response[0].header.ty, FrameType::Error);
13742 assert_eq!(
13743 parse_error(&response[0])["code"],
13744 "unsupported_control_frame"
13745 );
13746 }
13747
13748 mod swap {
13753 use super::*;
13754
13755 const INCUMBENT: ConnectionId = ConnectionId::new(30);
13756 const CANDIDATE: ConnectionId = ConnectionId::new(40);
13757
13758 struct Swap {
13759 registry: Arc<Registry>,
13760 forwarding: Arc<ForwardingTable>,
13761 handler: ControlHandler,
13762 incumbent_ctx: RouteCtx,
13763 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13764 candidate_ctx: RouteCtx,
13765 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13766 }
13767
13768 async fn swap_with_incumbent() -> Swap {
13769 let registry = Arc::new(Registry::default());
13770 let forwarding = Arc::new(ForwardingTable::default());
13771 let handler =
13772 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13773 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
13774 hello_via_sink(
13775 &handler,
13776 &incumbent_ctx,
13777 &mut incumbent_rx,
13778 hello_frame("aft", PROTOCOL_VERSION, 7),
13779 )
13780 .await;
13781 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
13782 Swap {
13783 registry,
13784 forwarding,
13785 handler,
13786 incumbent_ctx,
13787 incumbent_rx,
13788 candidate_ctx,
13789 candidate_rx,
13790 }
13791 }
13792
13793 fn register_candidate(swap: &Swap, ready: Option<bool>) {
13794 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
13795 candidate_manifest.ready = ready;
13796 let registration = swap
13797 .registry
13798 .register_candidate_with_control_ops(
13799 candidate_manifest,
13800 PROTOCOL_VERSION,
13801 CANDIDATE,
13802 module_baseline_control_ops(),
13803 )
13804 .unwrap();
13805 swap.forwarding
13806 .register_candidate_module_connection(
13807 CANDIDATE,
13808 "aft".to_string(),
13809 PROTOCOL_VERSION,
13810 manifest_concurrency(®istration.manifest),
13811 swap.candidate_ctx.egress.clone(),
13812 )
13813 .unwrap();
13814 }
13815
13816 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
13817 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
13818 swap.registry.promote_candidate("aft").unwrap().unwrap();
13819 cutover.incumbent.unwrap()
13820 }
13821
13822 #[tokio::test]
13831 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
13832 let mut swap = swap_with_incumbent().await;
13833 let hold = Arc::new(std::sync::Barrier::new(2));
13834 let handler = swap
13835 .handler
13836 .clone()
13837 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
13838 handler
13839 .supervisor
13840 .open_swap("aft", "candidate-nonce".to_string());
13841 let hello = {
13842 let handler = handler.clone();
13843 let candidate_ctx = swap.candidate_ctx.clone();
13844 std::thread::spawn(move || {
13845 handler
13846 .handle_hello(
13847 candidate_ctx.connection_id,
13848 Some(candidate_ctx.egress.clone()),
13849 hello_frame_with_nonce(
13850 "aft",
13851 PROTOCOL_VERSION,
13852 8,
13853 Some("candidate-nonce"),
13854 ),
13855 )
13856 .unwrap()
13857 })
13858 };
13859
13860 hold.wait();
13861 assert!(swap
13864 .registry
13865 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
13866 .unwrap()
13867 .is_some());
13868 assert!(swap
13869 .registry
13870 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
13871 .unwrap()
13872 .is_none());
13873 assert!(swap
13874 .forwarding
13875 .module_endpoint_for_connection(CANDIDATE)
13876 .unwrap()
13877 .is_none());
13878
13879 hold.wait();
13880 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
13881 assert_eq!(
13882 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
13883 FrameType::HelloAck
13884 );
13885 let published = swap
13886 .registry
13887 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
13888 .unwrap()
13889 .expect("candidate is published once its HELLO completes");
13890 assert_eq!(published.connection_id, CANDIDATE);
13891 assert!(swap
13892 .forwarding
13893 .module_endpoint_for_connection(CANDIDATE)
13894 .unwrap()
13895 .is_some());
13896 }
13897
13898 fn keyed_total(counters: &Value, key: &str) -> u64 {
13899 counters[key]
13900 .as_object()
13901 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
13902 .unwrap_or(0)
13903 }
13904
13905 #[tokio::test]
13906 async fn replacement_logs_old_end_and_new_admission_once() {
13907 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13908 let mut swap = swap_with_incumbent().await;
13909 swap.handler
13910 .supervisor
13911 .open_swap("aft", "candidate-nonce".to_string());
13912 hello_via_sink(
13913 &swap.handler,
13914 &swap.candidate_ctx,
13915 &mut swap.candidate_rx,
13916 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
13917 )
13918 .await;
13919 cutover(&swap);
13920 swap.handler.cleanup_connection(INCUMBENT).unwrap();
13921
13922 let captured = crate::router::test_log::captured_logs(&logs);
13923 println!("captured replacement registry logs:\n{captured}");
13924 let new_admissions: Vec<_> = captured
13925 .lines()
13926 .filter(|line| {
13927 line.contains("connection_id=40")
13928 && line.contains("swap candidate registered; not routable until cutover")
13929 })
13930 .collect();
13931 assert_eq!(
13932 new_admissions.len(),
13933 1,
13934 "unexpected admission logs: {captured}"
13935 );
13936 assert!(new_admissions[0].contains("module_id=aft"));
13937 assert!(new_admissions[0].contains("connection_id=40"));
13938 assert!(
13939 new_admissions[0].contains("swap candidate registered; not routable until cutover")
13940 );
13941 assert!(new_admissions[0].contains("ready=true"));
13942
13943 let old_admissions: Vec<_> = captured
13944 .lines()
13945 .filter(|line| {
13946 line.contains("module registered module_id=aft ")
13947 && line.contains("connection_id=30")
13948 })
13949 .collect();
13950 assert_eq!(
13951 old_admissions.len(),
13952 1,
13953 "unexpected incumbent admission: {captured}"
13954 );
13955
13956 let old_ends: Vec<_> = captured
13957 .lines()
13958 .filter(|line| {
13959 line.contains("connection_id=30") && line.contains("module registration ended")
13960 })
13961 .collect();
13962 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
13963 assert!(old_ends[0].contains("module_id=aft"));
13964 assert!(old_ends[0].contains("connection_id=30"));
13965 assert!(old_ends[0].contains("reason=replaced"));
13966 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
13967 assert_eq!(
13968 captured
13969 .lines()
13970 .filter(|line| line.contains("module registration promoted"))
13971 .count(),
13972 1,
13973 "unexpected promotion logs: {captured}"
13974 );
13975 }
13976
13977 #[tokio::test]
13983 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
13984 let mut swap = swap_with_incumbent().await;
13985 let handler = swap.handler.clone();
13986
13987 let cotenant = ConnectionId::new(31);
13989 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
13990 let (cotenant_task, cotenant_bind) = relay_route_open(
13991 &handler,
13992 cotenant,
13993 &cotenant_ctx.egress,
13994 &mut swap.incumbent_rx,
13995 100,
13996 "aft",
13997 "swap-cotenant",
13998 )
13999 .await;
14000 handler
14001 .handle_control_frame(
14002 &swap.incumbent_ctx,
14003 route_bind_ack(cotenant_bind.header.corr),
14004 )
14005 .await
14006 .unwrap();
14007 assert!(cotenant_task.await.unwrap().is_empty());
14008 let (cotenant_channel, cotenant_epoch) =
14009 published_route(&cotenant_rx.recv().await.unwrap());
14010
14011 let caller = ConnectionId::new(32);
14013 let (caller_ctx, mut caller_rx) = route_ctx(caller);
14014 let (caller_task, caller_bind) = relay_route_open(
14015 &handler,
14016 caller,
14017 &caller_ctx.egress,
14018 &mut swap.incumbent_rx,
14019 101,
14020 "aft",
14021 "swap-caller",
14022 )
14023 .await;
14024 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
14025
14026 register_candidate(&swap, None);
14027 cutover(&swap);
14028
14029 let ack = handler
14031 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
14032 .await;
14033 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
14034 if module_loop_error.is_some() {
14035 handler.cleanup_connection(INCUMBENT).unwrap();
14038 }
14039
14040 assert!(
14042 cotenant_rx.try_recv().is_err(),
14043 "the co-tenant route on the incumbent was torn down by one late ack: \
14044 {module_loop_error:?}"
14045 );
14046 assert!(matches!(
14047 swap.forwarding
14048 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
14049 .unwrap(),
14050 DataRoute::Client(DataRouteState::Bound(_))
14051 ));
14052 assert_eq!(module_loop_error, None);
14053 assert!(swap
14054 .registry
14055 .get_module_by_connection(INCUMBENT)
14056 .unwrap()
14057 .is_some());
14058
14059 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
14061 .await
14062 .expect("the incumbent is told to drop the abandoned binding")
14063 .unwrap()
14064 .frame;
14065 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14066 assert_eq!(goodbye.header.channel, abandoned_channel);
14067 assert_eq!(goodbye.header.epoch, abandoned_epoch);
14068 assert!(swap.incumbent_rx.try_recv().is_err());
14069
14070 let response = caller_task.await.unwrap();
14072 assert_eq!(response.len(), 1);
14073 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14074 assert!(caller_rx.try_recv().is_err());
14075
14076 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
14080 let counters = handler.counters().snapshot();
14081 assert_eq!(
14082 keyed_total(&counters, "route_open_accepted_by_principal"),
14083 1
14084 );
14085 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
14086 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
14087 }
14088
14089 #[tokio::test]
14093 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
14094 let mut swap = swap_with_incumbent().await;
14095 register_candidate(&swap, None);
14096 let incumbent = cutover(&swap);
14097 swap.forwarding
14098 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
14099 .unwrap()
14100 .expect("the incumbent is still registered");
14101
14102 let client = ConnectionId::new(33);
14103 let (client_ctx, mut client_rx) = route_ctx(client);
14104 let route_handler = swap.handler.clone();
14105 let open_ctx = RouteCtx {
14106 connection_id: client,
14107 egress: client_ctx.egress.clone(),
14108 };
14109 let mut route_task = tokio::spawn(async move {
14110 route_handler
14111 .handle_control_frame(
14112 &open_ctx,
14113 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
14114 )
14115 .await
14116 .unwrap()
14117 });
14118 let bind = tokio::select! {
14119 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
14120 response = &mut route_task => {
14121 let response = response.unwrap();
14122 panic!(
14123 "post-cutover route.open was refused instead of relayed to the candidate: {}",
14124 parse_error(&response[0])["code"]
14125 );
14126 }
14127 };
14128 swap.handler
14129 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
14130 .await
14131 .unwrap();
14132 assert!(route_task.await.unwrap().is_empty());
14133 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
14134 match swap
14135 .forwarding
14136 .lookup_data_route(client, channel, epoch)
14137 .unwrap()
14138 {
14139 DataRoute::Client(DataRouteState::Bound(route)) => {
14140 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
14141 }
14142 other => panic!("expected a bound route on the candidate, got {other:?}"),
14143 }
14144 assert!(swap.incumbent_rx.try_recv().is_err());
14145 }
14146
14147 #[tokio::test]
14152 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
14153 let swap = swap_with_incumbent().await;
14154 register_candidate(&swap, Some(false));
14155 let update = Frame::build(
14156 FrameType::Request,
14157 control_flags(),
14158 0,
14159 0,
14160 55,
14161 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14162 provides: manifest("aft", PROTOCOL_VERSION).provides,
14163 capabilities: None,
14164 ready: Some(true),
14165 })
14166 .unwrap(),
14167 )
14168 .unwrap();
14169
14170 let replies = swap
14171 .handler
14172 .handle_control_frame(&swap.candidate_ctx, update)
14173 .await
14174 .unwrap();
14175
14176 assert_eq!(replies.len(), 1);
14177 assert_eq!(
14178 replies[0].header.ty,
14179 FrameType::Response,
14180 "candidate catalog.update was refused: {:?}",
14181 serde_json::from_slice::<Value>(&replies[0].body).ok()
14182 );
14183 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
14184 assert_eq!(
14185 swap.registry
14186 .get_module("aft")
14187 .unwrap()
14188 .unwrap()
14189 .connection_id,
14190 INCUMBENT
14191 );
14192 }
14193 }
14194
14195 mod swap_admission {
14199 use super::*;
14200
14201 const INCUMBENT_NONCE: &str = "incumbent-nonce";
14202 const CANDIDATE_NONCE: &str = "candidate-nonce";
14203
14204 fn handler_with_incumbent(
14205 module_id: &str,
14206 reserved: bool,
14207 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
14208 let registry = Arc::new(Registry::default());
14209 let supervisor = SupervisorHandle::new();
14210 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
14211 if reserved {
14212 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
14213 }
14214 let handler =
14215 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
14216 let incumbent = handler
14217 .handle_control(
14218 ConnectionId::new(1),
14219 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
14220 )
14221 .unwrap();
14222 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
14223 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
14224 (registry, supervisor, handler)
14225 }
14226
14227 #[test]
14233 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
14234 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
14235
14236 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
14237 let replies = handler
14238 .handle_control(
14239 ConnectionId::new(connection),
14240 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
14241 )
14242 .unwrap();
14243 assert_eq!(replies[0].header.ty, FrameType::Error);
14244 assert_eq!(
14245 parse_error(&replies[0])["code"],
14246 "swap_token_invalid",
14247 "nonce {nonce:?}"
14248 );
14249 }
14250 assert!(registry.get_candidate("aft").unwrap().is_none());
14251 assert_eq!(
14252 registry.get_module("aft").unwrap().unwrap().connection_id,
14253 ConnectionId::new(1)
14254 );
14255
14256 let admitted = handler
14259 .handle_control(
14260 ConnectionId::new(4),
14261 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
14262 )
14263 .unwrap();
14264 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
14265 assert_eq!(
14266 registry
14267 .get_candidate("aft")
14268 .unwrap()
14269 .unwrap()
14270 .connection_id,
14271 ConnectionId::new(4)
14272 );
14273 assert_eq!(
14274 registry.get_module("aft").unwrap().unwrap().connection_id,
14275 ConnectionId::new(1),
14276 "the candidate must not take the active slot"
14277 );
14278 let replayed = handler
14279 .handle_control(
14280 ConnectionId::new(5),
14281 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
14282 )
14283 .unwrap();
14284 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
14285
14286 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
14290 let squatter = handler
14291 .handle_control(
14292 ConnectionId::new(6),
14293 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
14294 )
14295 .unwrap();
14296 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
14297 assert!(
14298 registry.get_module("aft").unwrap().is_none(),
14299 "a squatter took the active slot of an id being swapped"
14300 );
14301 }
14302
14303 #[test]
14308 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
14309 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
14310
14311 let replies = handler
14312 .handle_control(
14313 ConnectionId::new(2),
14314 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14315 )
14316 .unwrap();
14317
14318 assert_eq!(
14319 replies[0].header.ty,
14320 FrameType::HelloAck,
14321 "reserved candidate refused: {:?}",
14322 serde_json::from_slice::<Value>(&replies[0].body).ok()
14323 );
14324 assert_eq!(
14325 registry
14326 .get_candidate("vault")
14327 .unwrap()
14328 .unwrap()
14329 .connection_id,
14330 ConnectionId::new(2)
14331 );
14332 }
14333
14334 #[test]
14337 fn without_an_open_swap_the_ordinary_gates_decide() {
14338 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
14339 supervisor.close_swap("vault");
14340
14341 let candidate = handler
14342 .handle_control(
14343 ConnectionId::new(2),
14344 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14345 )
14346 .unwrap();
14347 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
14348 let duplicate = handler
14349 .handle_control(
14350 ConnectionId::new(3),
14351 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
14352 )
14353 .unwrap();
14354 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14355 assert!(registry.get_candidate("vault").unwrap().is_none());
14356 }
14357 }
14358
14359 mod scopes {
14363 mod activity {
14364 include!("activity_scope_tests.rs");
14365 }
14366 mod delta {
14367 include!("scope_handler_delta_tests.rs");
14368 }
14369 mod call_modes {
14370 include!("scope_call_mode_tests.rs");
14371 }
14372
14373 use subc_protocol::scope::{
14374 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
14375 ScopeStatus,
14376 };
14377
14378 use super::*;
14379
14380 const OWNER: &str = "prefrontal-core";
14381
14382 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
14383 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
14384 }
14385
14386 async fn call(
14387 handler: &ControlHandler,
14388 ctx: &RouteCtx,
14389 request: &ModuleControlRequestFromModule,
14390 ) -> Frame {
14391 let body = serde_json::to_vec(request).unwrap();
14392 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
14393 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
14394 assert_eq!(replies.len(), 1, "{replies:?}");
14395 replies.pop().unwrap()
14396 }
14397
14398 async fn sync(
14399 handler: &ControlHandler,
14400 ctx: &RouteCtx,
14401 generation: u64,
14402 scopes: Vec<ScopeRecord>,
14403 ) -> Result<ModuleControlResponseToModule, String> {
14404 let reply = call(
14405 handler,
14406 ctx,
14407 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
14408 )
14409 .await;
14410 match reply.header.ty {
14411 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
14412 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
14413 }
14414 }
14415
14416 async fn describe(
14417 handler: &ControlHandler,
14418 ctx: &RouteCtx,
14419 owner: &str,
14420 scope_ref: &str,
14421 ) -> ModuleControlResponseToModule {
14422 let reply = call(
14423 handler,
14424 ctx,
14425 &ModuleControlRequestFromModule::ScopeDescribe {
14426 owner: Principal::Reserved {
14427 module_id: owner.to_string(),
14428 },
14429 scope_ref: scope_ref.to_string(),
14430 },
14431 )
14432 .await;
14433 assert_eq!(
14434 reply.header.ty,
14435 FrameType::Response,
14436 "{:?}",
14437 parse_error(&reply)
14438 );
14439 serde_json::from_slice(&reply.body).unwrap()
14440 }
14441
14442 async fn module(
14444 handler: &ControlHandler,
14445 connection: u64,
14446 module_id: &str,
14447 nonce: Option<&str>,
14448 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
14449 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
14450 hello_via_sink(
14451 handler,
14452 &ctx,
14453 &mut rx,
14454 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
14455 )
14456 .await;
14457 (ctx, rx)
14458 }
14459
14460 #[tokio::test]
14463 async fn a_client_connection_cannot_sync_or_describe() {
14464 let handler = ControlHandler::new(Arc::new(Registry::default()));
14465 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
14466 for request in [
14467 ModuleControlRequestFromModule::ScopeSync {
14468 generation: 1,
14469 scopes: vec![head("s", 1)],
14470 },
14471 ModuleControlRequestFromModule::ScopeDescribe {
14472 owner: Principal::Direct,
14473 scope_ref: "s".to_string(),
14474 },
14475 ] {
14476 let reply = call(&handler, &ctx, &request).await;
14477 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
14478 }
14479 assert!(
14480 !handler
14481 .scopes
14482 .read()
14483 .unwrap()
14484 .describe(
14485 &Principal::Reserved {
14486 module_id: OWNER.to_string()
14487 },
14488 "s"
14489 )
14490 .owner_synced
14491 );
14492 }
14493
14494 #[tokio::test]
14497 async fn a_module_without_a_supervised_launch_cannot_sync() {
14498 let handler = ControlHandler::new(Arc::new(Registry::default()));
14499 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
14500 assert_eq!(
14501 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
14502 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14503 );
14504 }
14505
14506 #[tokio::test]
14507 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
14508 let supervisor = SupervisorHandle::new();
14509 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14510 let handler = ControlHandler::new(Arc::new(Registry::default()))
14511 .with_supervisor(supervisor.clone());
14512 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
14513 sync(&handler, &incumbent, 1, vec![head("s", 1)])
14514 .await
14515 .expect("the current launch syncs");
14516
14517 supervisor.open_swap(OWNER, "n2".to_string());
14520 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
14521 assert_eq!(
14522 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
14523 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14524 );
14525 sync(&handler, &incumbent, 2, vec![head("s", 1)])
14526 .await
14527 .expect("the serving owner syncs during the swap");
14528
14529 supervisor.close_swap(OWNER);
14532 assert_eq!(
14533 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
14534 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14535 );
14536 sync(&handler, &incumbent, 3, vec![head("s", 1)])
14537 .await
14538 .expect("the serving owner syncs after the rollback");
14539 handler.cleanup_connection(candidate.connection_id).unwrap();
14540
14541 supervisor.open_swap(OWNER, "n3".to_string());
14546 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
14547 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
14548 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
14549 .await
14550 .expect("the promoted launch takes authority");
14551 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
14552 panic!("unexpected reply {reply:?}");
14553 };
14554 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
14555 assert_eq!(
14556 sync(&handler, &incumbent, 4, Vec::new()).await,
14557 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14558 );
14559 }
14560
14561 #[tokio::test]
14564 async fn closing_the_authority_connection_frees_sync_authority() {
14565 let supervisor = SupervisorHandle::new();
14566 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14567 let handler = ControlHandler::new(Arc::new(Registry::default()))
14568 .with_supervisor(supervisor.clone());
14569 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
14570 sync(&handler, &first, 10, vec![head("s", 1)])
14571 .await
14572 .unwrap();
14573 handler.cleanup_connection(first.connection_id).unwrap();
14574
14575 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
14576 sync(&handler, &second, 1, vec![head("s", 1)])
14577 .await
14578 .expect("the next connection takes the released authority");
14579 }
14580
14581 #[tokio::test]
14582 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
14583 let supervisor = SupervisorHandle::new();
14584 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14585 let handler =
14586 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
14587 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
14588 sync(&handler, &first, 10, vec![head("s", 1)])
14589 .await
14590 .unwrap();
14591 handler
14592 .handle_control_frame(
14593 &first,
14594 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
14595 )
14596 .await
14597 .unwrap();
14598 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
14599 sync(&handler, &second, 1, vec![head("s", 1)])
14600 .await
14601 .expect("GOODBYE releases authority even if the old socket remains open");
14602 }
14603
14604 #[tokio::test]
14605 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
14606 let registry = Arc::new(Registry::default());
14607 let supervisor_handle = SupervisorHandle::new();
14608 let supervisor =
14609 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
14610 .with_handle(supervisor_handle.clone())
14611 .with_daemon_incarnation("incarnation-7".to_string());
14612 supervisor
14615 .supervise_configured(
14616 ModuleSpec {
14617 module_id: OWNER.to_string(),
14618 program: PathBuf::from("/nonexistent/prefrontal-core"),
14619 args: Vec::new(),
14620 env: Vec::new(),
14621 reserved: false,
14622 reserved_prefixes: Vec::new(),
14623 protocol: ModuleProtocol::Subc,
14624 overlap: Default::default(),
14625 },
14626 false,
14627 )
14628 .unwrap();
14629 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
14630 let handler =
14631 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14632 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
14633
14634 let ModuleControlResponseToModule::ScopeDescribe {
14636 status,
14637 daemon_incarnation,
14638 owner_synced,
14639 owner_configured,
14640 scope,
14641 ..
14642 } = describe(&handler, &reader, OWNER, "s").await
14643 else {
14644 panic!("not a describe reply");
14645 };
14646 assert_eq!(status, ScopeStatus::NotLive);
14647 assert_eq!(daemon_incarnation, "incarnation-7");
14648 assert!(!owner_synced);
14649 assert!(owner_configured);
14650 assert!(scope.is_none());
14651
14652 let ModuleControlResponseToModule::ScopeDescribe {
14655 status,
14656 owner_configured,
14657 ..
14658 } = describe(&handler, &reader, "ghost", "s").await
14659 else {
14660 panic!("not a describe reply");
14661 };
14662 assert_eq!(status, ScopeStatus::NotLive);
14663 assert!(!owner_configured);
14664
14665 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
14667 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
14668 let ModuleControlResponseToModule::ScopeDescribe {
14669 status,
14670 scope_epoch,
14671 owner_synced,
14672 scope,
14673 ..
14674 } = describe(&handler, &reader, OWNER, "s").await
14675 else {
14676 panic!("not a describe reply");
14677 };
14678 assert_eq!(status, ScopeStatus::Live);
14679 assert_eq!(scope_epoch, Some(4));
14680 assert!(owner_synced);
14681 let stamp = scope.expect("a live scope carries its stamp");
14682 assert!(
14683 stamp.owner_authorized,
14684 "prefrontal-core is the default authority"
14685 );
14686 assert_eq!(stamp.kind, ScopeKind::Head);
14687 }
14688
14689 #[tokio::test]
14690 async fn scope_authority_owners_decides_owner_authorized() {
14691 let supervisor = SupervisorHandle::new();
14692 supervisor.set_spawn_nonce("broca", "b1".to_string());
14693 let handler = ControlHandler::new(Arc::new(Registry::default()))
14694 .with_supervisor(supervisor)
14695 .with_scope_authority_owners(vec!["broca".to_string()]);
14696 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
14697 let mut gated = head("s", 1);
14698 gated.attributes.agent_id = Some("agent".to_string());
14699 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
14700 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
14701 describe(&handler, &broca, "broca", "s").await
14702 else {
14703 panic!("not a describe reply");
14704 };
14705 assert!(scope.unwrap().owner_authorized);
14706 }
14707
14708 #[tokio::test]
14712 async fn scope_ops_and_the_scopes_capability_are_advertised() {
14713 let handler = ControlHandler::new(Arc::new(Registry::default()));
14714 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
14715 let ack = hello_via_sink(
14716 &handler,
14717 &ctx,
14718 &mut rx,
14719 hello_frame("m", PROTOCOL_VERSION, 1),
14720 )
14721 .await;
14722 let ack = parse_ack(&ack);
14723 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
14724 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
14725 }
14726 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
14727
14728 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
14729 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
14730 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
14731 let reply = handler
14732 .handle_control_frame(&client, frame)
14733 .await
14734 .unwrap()
14735 .pop()
14736 .unwrap();
14737 let ClientControlResponse::ServerDescribe { capabilities, .. } =
14738 serde_json::from_slice(&reply.body).unwrap()
14739 else {
14740 panic!("not a server.describe reply");
14741 };
14742 assert!(
14743 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
14744 "{capabilities:?}"
14745 );
14746 }
14747
14748 const PLEXUS: &str = "plexus";
14751 const OTHER: &str = "other";
14752 const AFT: &str = "aft";
14753 const BROCA: &str = "broca";
14754 const MAGIC: &str = "magic-context";
14755
14756 fn nonce(module_id: &str) -> String {
14757 format!("nonce-{module_id}")
14758 }
14759
14760 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
14761 let (tx, rx) = mpsc::channel(64);
14762 (
14763 RouteCtx {
14764 connection_id: ConnectionId::new(connection),
14765 egress: FrameSink::new(tx),
14766 },
14767 rx,
14768 )
14769 }
14770
14771 struct Rig {
14775 handler: ControlHandler,
14776 forwarding: Arc<ForwardingTable>,
14777 owner: RouteCtx,
14778 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14779 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
14780 generation: u64,
14781 next_connection: u64,
14782 _supervisor: Supervisor,
14783 }
14784
14785 async fn rig() -> Rig {
14786 rig_with_flow_support(true).await
14787 }
14788
14789 async fn rig_with_flow_support(flow_support: bool) -> Rig {
14790 rig_with_scope_capabilities(flow_support, false).await
14791 }
14792
14793 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
14794 let mut provides = Vec::new();
14795 if flow_support {
14796 provides.push("flow-scopes/v1");
14797 }
14798 if run_support {
14799 provides.push("agent-run-scopes/v1");
14800 }
14801 rig_with_target_capabilities(&provides).await
14802 }
14803
14804 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
14805 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
14806 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
14807 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
14810 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
14811 Frame::build(
14812 FrameType::Hello,
14813 control_flags(),
14814 0,
14815 0,
14816 connection,
14817 serde_json::to_vec(&body).unwrap(),
14818 )
14819 .unwrap()
14820 }
14821
14822 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
14823 let registry = Arc::new(Registry::default());
14824 let forwarding = Arc::new(ForwardingTable::default());
14825 let supervisor_handle = SupervisorHandle::new();
14826 let supervisor =
14827 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
14828 .with_handle(supervisor_handle.clone());
14829 supervisor
14830 .supervise_configured(
14831 ModuleSpec {
14832 module_id: OWNER.to_string(),
14833 program: PathBuf::from("/nonexistent/prefrontal-core"),
14834 args: Vec::new(),
14835 env: Vec::new(),
14836 reserved: false,
14837 reserved_prefixes: Vec::new(),
14838 protocol: ModuleProtocol::Subc,
14839 overlap: Default::default(),
14840 },
14841 false,
14842 )
14843 .unwrap();
14844 for module_id in [OWNER, AFT, BROCA, MAGIC] {
14845 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
14846 }
14847 let handler =
14848 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14849 .with_supervisor(supervisor_handle);
14850 let (owner, mut owner_rx) = wide_ctx(1);
14851 hello_via_sink(
14852 &handler,
14853 &owner,
14854 &mut owner_rx,
14855 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
14856 )
14857 .await;
14858 let mut modules = BTreeMap::new();
14859 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
14860 let (ctx, mut rx) = wide_ctx(connection);
14861 let hello = hello_with_capabilities(module_id, connection, provides);
14862 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
14863 modules.insert(module_id.to_string(), (ctx, rx));
14864 }
14865 Rig {
14866 handler,
14867 forwarding,
14868 owner,
14869 _owner_rx: owner_rx,
14870 modules,
14871 generation: 0,
14872 next_connection: 100,
14873 _supervisor: supervisor,
14874 }
14875 }
14876
14877 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
14878 ScopeCarrier::new(Principal::Reserved {
14879 module_id: module_id.to_string(),
14880 })
14881 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
14882 }
14883
14884 fn session(scope_epoch: u64) -> ScopeRecord {
14887 let mut record = head("s", scope_epoch);
14888 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
14889 record.attributes.agent_id = Some("agent-1".to_string());
14890 record.attributes.delegates = true;
14891 record
14892 }
14893
14894 impl Rig {
14895 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
14896 self.generation += 1;
14897 sync(&self.handler, &self.owner, self.generation, scopes)
14898 .await
14899 .expect("the owner's sync is accepted");
14900 }
14901
14902 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
14903 ScopeSelector {
14904 owner: Principal::Reserved {
14905 module_id: OWNER.to_string(),
14906 },
14907 scope_ref: scope_ref.to_string(),
14908 scope_epoch,
14909 }
14910 }
14911
14912 fn open_frame(
14913 &mut self,
14914 opener: Option<&str>,
14915 target: &str,
14916 scope: Option<ScopeSelector>,
14917 ) -> (
14918 RouteCtx,
14919 mpsc::Receiver<crate::router::OutboundFrame>,
14920 Frame,
14921 ) {
14922 self.next_connection += 1;
14923 let (ctx, rx) = wide_ctx(self.next_connection);
14924 let root = unique_project_root("scoped-open");
14925 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
14926 target: RouteTarget::ToolProvider {
14927 module_id: target.to_string(),
14928 },
14929 identity: BindIdentity::new(
14930 root.path().to_path_buf(),
14931 "unit".to_string(),
14932 "session".to_string(),
14933 ),
14934 consumer_identity: opener.map(|module_id| ConsumerIdentity {
14935 module_id: module_id.to_string(),
14936 launch_nonce: nonce(module_id),
14937 }),
14938 consumer_capabilities: None,
14939 role_versions: None,
14940 admission_facts: None,
14941 scope,
14942 })
14943 .unwrap();
14944 let frame = Frame::build(
14945 FrameType::Request,
14946 control_flags(),
14947 0,
14948 0,
14949 self.next_connection,
14950 body,
14951 )
14952 .unwrap();
14953 (ctx, rx, frame)
14954 }
14955
14956 async fn refused(
14958 &mut self,
14959 opener: Option<&str>,
14960 target: &str,
14961 scope: Option<ScopeSelector>,
14962 ) -> String {
14963 self.refusal_body(opener, target, scope).await["code"]
14964 .as_str()
14965 .unwrap()
14966 .to_string()
14967 }
14968
14969 async fn refusal_body(
14970 &mut self,
14971 opener: Option<&str>,
14972 target: &str,
14973 scope: Option<ScopeSelector>,
14974 ) -> Value {
14975 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
14976 let replies = tokio::time::timeout(
14977 Duration::from_secs(2),
14978 self.handler.handle_control_frame(&ctx, frame),
14979 )
14980 .await
14981 .expect("the open must be refused before waiting for a bind ack")
14982 .unwrap();
14983 assert_eq!(replies.len(), 1, "{replies:?}");
14984 assert_eq!(replies[0].header.ty, FrameType::Error);
14985 let (_, module_rx) = self.modules.get_mut(target).unwrap();
14986 assert!(
14987 module_rx.try_recv().is_err(),
14988 "a refused open relays nothing"
14989 );
14990 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
14991 parse_error(&replies[0])
14992 }
14993
14994 async fn relayed(
14996 &mut self,
14997 opener: Option<&str>,
14998 target: &str,
14999 scope: Option<ScopeSelector>,
15000 ) -> Relayed {
15001 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
15002 let handler = self.handler.clone();
15003 let task_ctx = ctx.clone();
15004 let task = tokio::spawn(async move {
15005 handler
15006 .handle_control_frame(&task_ctx, frame)
15007 .await
15008 .unwrap()
15009 });
15010 let (_, module_rx) = self.modules.get_mut(target).unwrap();
15011 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
15012 .await
15013 .expect("the target receives the relayed route.bind")
15014 .unwrap()
15015 .frame;
15016 Relayed {
15017 target: target.to_string(),
15018 client: ctx,
15019 client_rx: rx,
15020 task,
15021 bind,
15022 }
15023 }
15024
15025 async fn ack(&self, relayed: &Relayed) {
15026 let (module, _) = &self.modules[&relayed.target];
15027 self.handler
15028 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
15029 .await
15030 .unwrap();
15031 }
15032
15033 async fn bound(
15035 &mut self,
15036 opener: Option<&str>,
15037 target: &str,
15038 scope: Option<ScopeSelector>,
15039 ) -> Bound {
15040 let relayed = self.relayed(opener, target, scope).await;
15041 self.ack(&relayed).await;
15042 let Relayed {
15043 target,
15044 client,
15045 mut client_rx,
15046 task,
15047 bind,
15048 } = relayed;
15049 assert!(
15050 task.await.unwrap().is_empty(),
15051 "the open is answered by commit"
15052 );
15053 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
15054 Bound {
15055 target,
15056 client,
15057 client_rx,
15058 channel,
15059 epoch,
15060 bind,
15061 }
15062 }
15063
15064 fn live(&self, route: &Bound) -> bool {
15065 matches!(
15066 self.forwarding
15067 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
15068 .unwrap(),
15069 DataRoute::Client(DataRouteState::Bound(_))
15070 )
15071 }
15072 }
15073
15074 struct Relayed {
15075 target: String,
15076 client: RouteCtx,
15077 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15078 task: tokio::task::JoinHandle<Vec<Frame>>,
15079 bind: Frame,
15080 }
15081
15082 struct Bound {
15083 target: String,
15084 client: RouteCtx,
15085 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15086 channel: u16,
15087 epoch: u32,
15088 bind: Frame,
15089 }
15090
15091 impl Bound {
15092 fn closed_reason(&mut self) -> RouteCloseReason {
15095 let mut reason = None;
15096 let mut goodbye = false;
15097 while let Ok(outbound) = self.client_rx.try_recv() {
15098 let frame = outbound.frame;
15099 match frame.header.ty {
15100 FrameType::Goodbye => {
15101 assert_eq!(
15102 (frame.header.channel, frame.header.epoch),
15103 (self.channel, self.epoch)
15104 );
15105 goodbye = true;
15106 }
15107 FrameType::Push => {
15108 let ClientControlPush::RouteClosed {
15109 reason: r,
15110 module_id,
15111 ..
15112 } = serde_json::from_slice(&frame.body).unwrap()
15113 else {
15114 panic!("unexpected push");
15115 };
15116 assert_eq!(module_id, self.target);
15117 reason = Some(r);
15118 }
15119 other => panic!("unexpected frame {other:?}"),
15120 }
15121 }
15122 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
15123 reason.expect("the client is told why the route closed")
15124 }
15125
15126 fn untouched(&mut self) -> bool {
15127 self.client_rx.try_recv().is_err()
15128 }
15129
15130 fn stamp(&self) -> Option<ScopeStamp> {
15131 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
15132 ModuleControlRequest::RouteBind { scope, .. } => scope,
15133 other => panic!("expected a route.bind, got {other:?}"),
15134 }
15135 }
15136 }
15137
15138 #[tokio::test]
15139 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
15140 ) {
15141 let mut rig = rig().await;
15142 let mut record = session(1);
15143 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
15144 record.child_owners = vec![Principal::Reserved {
15145 module_id: MAGIC.to_string(),
15146 }];
15147 rig.sync(vec![record]).await;
15148 let scope = || Some(rig_selector("s", Some(1)));
15149
15150 rig.bound(Some(OWNER), PLEXUS, scope()).await;
15153 rig.bound(Some(AFT), OTHER, scope()).await;
15154 rig.bound(Some(BROCA), PLEXUS, scope()).await;
15155
15156 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
15160 assert_eq!(
15161 rig.refused(opener, target, scope()).await,
15162 error_codes::SCOPE_NOT_CARRIER,
15163 "{opener:?} -> {target}"
15164 );
15165 }
15166 }
15167
15168 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15169 ScopeSelector {
15170 owner: Principal::Reserved {
15171 module_id: OWNER.to_string(),
15172 },
15173 scope_ref: scope_ref.to_string(),
15174 scope_epoch,
15175 }
15176 }
15177
15178 #[tokio::test]
15179 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
15180 let mut rig = rig().await;
15181 rig.sync(vec![session(1)]).await;
15182 for opener in [OWNER, AFT] {
15183 assert_eq!(
15184 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
15185 .await,
15186 error_codes::SCOPE_EPOCH_REQUIRED,
15187 "{opener}"
15188 );
15189 }
15190 }
15191
15192 #[tokio::test]
15193 async fn admission_separates_not_synced_not_live_and_ended() {
15194 let mut rig = rig().await;
15195 let code = rig
15197 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15198 .await;
15199 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
15200 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
15201
15202 let ghost = ScopeSelector {
15204 owner: Principal::Reserved {
15205 module_id: "ghost".to_string(),
15206 },
15207 scope_ref: "s".to_string(),
15208 scope_epoch: Some(1),
15209 };
15210 assert_eq!(
15211 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
15212 error_codes::SCOPE_NOT_LIVE
15213 );
15214
15215 rig.sync(vec![session(2)]).await;
15216 assert_eq!(
15217 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
15218 .await,
15219 error_codes::SCOPE_NOT_LIVE
15220 );
15221 for epoch in [1, 3] {
15222 assert_eq!(
15223 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
15224 .await,
15225 error_codes::SCOPE_ENDED,
15226 "epoch {epoch}"
15227 );
15228 }
15229 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
15231 .await;
15232 }
15233
15234 #[tokio::test]
15235 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
15236 let mut rig = rig().await;
15237 rig.sync(vec![session(1)]).await;
15238 let route = rig
15239 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15240 .await;
15241 let stamp = route.stamp().expect("a scoped bind carries the stamp");
15242 assert_eq!(stamp.scope_ref, "s");
15243 assert_eq!(stamp.scope_epoch, 1);
15244 assert_eq!(stamp.kind, ScopeKind::Head);
15245 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
15246 assert!(stamp.attributes.delegates);
15247 assert!(stamp.owner_authorized);
15248 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15249 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
15250
15251 let (broca, mut broca_rx) = wide_ctx(50);
15254 hello_via_sink(
15255 &rig.handler,
15256 &broca,
15257 &mut broca_rx,
15258 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
15259 )
15260 .await;
15261 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
15262 .await
15263 .unwrap();
15264 let own = ScopeSelector {
15265 owner: Principal::Reserved {
15266 module_id: BROCA.to_string(),
15267 },
15268 scope_ref: "b".to_string(),
15269 scope_epoch: Some(1),
15270 };
15271 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
15272 assert!(!route.stamp().unwrap().owner_authorized);
15273 }
15274
15275 #[tokio::test]
15276 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
15277 let mut rig = rig().await;
15278 let mut record = head("s", 1);
15279 record.carriers = vec![carrier(AFT, None)];
15280 let flow_id = "Flow:run-7/step_2!~";
15281 record.attributes.flow_id = Some(flow_id.to_string());
15282 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
15283 .await
15284 .unwrap();
15285 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
15286 panic!("not a sync reply");
15287 };
15288 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
15289 let route = rig
15290 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15291 .await;
15292 assert!(rig.live(&route), "the stamped bind committed");
15293 let stamp = route.stamp().expect("a flow scope carries a stamp");
15294 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
15295 assert_eq!(stamp.attributes.agent_id, None);
15296 assert!(!stamp.attributes.delegates);
15297 assert!(stamp.owner_authorized);
15298 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15299 assert_eq!(unscoped.stamp(), None);
15300 }
15301
15302 #[tokio::test]
15303 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
15304 let mut rig = rig_with_flow_support(false).await;
15305 let mut record = session(1);
15306 record.attributes.flow_id = Some("flow:7".to_string());
15307 rig.sync(vec![record]).await;
15308 for opener in [OWNER, AFT] {
15309 let body = rig
15310 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
15311 .await;
15312 assert_eq!(body["code"], "target_flow_unsupported");
15313 let message = body["message"].as_str().unwrap();
15314 for required in [PLEXUS, "flow-scopes/v1"] {
15315 assert!(message.contains(required), "{message}");
15316 }
15317 }
15318 }
15319
15320 #[tokio::test]
15321 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
15322 let mut rig = rig_with_flow_support(true).await;
15323 let mut record = session(1);
15324 record.attributes.flow_id = Some("flow:7".to_string());
15325 rig.sync(vec![record]).await;
15326 for opener in [OWNER, AFT] {
15327 let route = rig
15328 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
15329 .await;
15330 assert!(rig.live(&route));
15331 assert_eq!(
15332 route.stamp().unwrap().attributes.flow_id.as_deref(),
15333 Some("flow:7")
15334 );
15335 }
15336 }
15337
15338 #[tokio::test]
15339 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
15340 use std::future::Future;
15341
15342 let mut rig = rig_with_flow_support(true).await;
15343 let mut record = session(1);
15344 record.attributes.flow_id = Some("flow:7".to_string());
15345 rig.sync(vec![record]).await;
15346 let (client, mut client_rx, frame) =
15347 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
15348 for _ in 0..64 {
15351 client.egress.try_send(route_bind_ack(1)).unwrap();
15352 }
15353 let handler = rig.handler.clone();
15354 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15355 std::future::poll_fn(|cx| {
15356 assert!(open.as_mut().poll(cx).is_pending());
15357 std::task::Poll::Ready(())
15358 })
15359 .await;
15360 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15361
15362 let old_connection = rig.modules[PLEXUS].0.connection_id;
15363 rig.handler.cleanup_connection(old_connection).unwrap();
15364 let (replacement, mut replacement_rx) = wide_ctx(200);
15365 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
15366 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
15367 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
15368 let hello = Frame::build(
15369 FrameType::Hello,
15370 control_flags(),
15371 0,
15372 0,
15373 200,
15374 serde_json::to_vec(&body).unwrap(),
15375 )
15376 .unwrap();
15377 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
15378
15379 client_rx.try_recv().unwrap();
15380 let replies = tokio::time::timeout(Duration::from_secs(2), open)
15381 .await
15382 .expect("the replacement is refused without waiting for a bind ack")
15383 .unwrap();
15384 assert_eq!(replies.len(), 1);
15385 let body = parse_error(&replies[0]);
15386 assert_eq!(body["code"], "target_flow_unsupported");
15387 for required in [PLEXUS, "flow-scopes/v1"] {
15388 assert!(body["message"].as_str().unwrap().contains(required));
15389 }
15390 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
15391 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
15392 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15393 assert!(rig
15394 .handler
15395 .registry
15396 .get_module_by_connection(replacement.connection_id)
15397 .unwrap()
15398 .is_some());
15399 }
15400
15401 #[tokio::test]
15402 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
15403 {
15404 let mut rig = rig_with_flow_support(false).await;
15405 rig.sync(vec![session(1)]).await;
15406 let route = rig
15407 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15408 .await;
15409 assert!(rig.live(&route));
15410 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
15411 let mut record = session(1);
15412 record.attributes.flow_id = Some("flow:7".to_string());
15413 rig.sync(vec![record]).await;
15414 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
15415 assert!(rig.live(&unscoped));
15416 assert_eq!(unscoped.stamp(), None);
15417 }
15418
15419 #[tokio::test]
15420 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
15421 let mut rig = rig().await;
15422 let mut record = session(1);
15423 record.attributes.flow_id = Some("flow:7".to_string());
15424 rig.sync(vec![record.clone()]).await;
15425 let before = rig.forwarding.published_scope_tag(OWNER, "s");
15426 let mut owner_route = rig
15427 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
15428 .await;
15429 let mut carrier_route = rig
15430 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
15431 .await;
15432 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15433 rig.sync(vec![record.clone()]).await;
15434 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
15435 assert!(rig.live(&owner_route) && owner_route.untouched());
15436 assert!(rig.live(&carrier_route) && carrier_route.untouched());
15437
15438 record.attributes.flow_id = Some("flow:8".to_string());
15439 rig.sync(vec![record]).await;
15440 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
15441 let before = before.unwrap();
15442 assert_eq!(after.scope_epoch, before.scope_epoch);
15443 assert!(after.version > before.version);
15444 for route in [&mut owner_route, &mut carrier_route] {
15445 assert!(!rig.live(route));
15446 assert_eq!(
15447 route.closed_reason(),
15448 RouteCloseReason::ScopeDelegationChanged
15449 );
15450 }
15451 assert!(rig.live(&unscoped) && unscoped.untouched());
15452 for target in [PLEXUS, OTHER] {
15455 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
15456 let goodbye = module_rx
15457 .try_recv()
15458 .expect("the provider sees the drain")
15459 .frame;
15460 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15461 assert!(module_rx.try_recv().is_err());
15462 }
15463 let rebound = rig
15464 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15465 .await;
15466 assert_eq!(
15467 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
15468 Some("flow:8")
15469 );
15470 }
15471
15472 #[tokio::test]
15477 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
15478 let mut rig = rig().await;
15479 rig.sync(vec![session(1)]).await;
15480 let mut cotenant = rig
15481 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
15482 .await;
15483
15484 let mut changed = session(1);
15485 changed.child_owners.push(Principal::Reserved {
15486 module_id: MAGIC.to_string(),
15487 });
15488 let mut ended = None;
15489 for (code, next) in [
15490 (error_codes::SCOPE_CHANGED, vec![changed]),
15491 (error_codes::SCOPE_ENDED, Vec::new()),
15492 ] {
15493 let relayed = rig
15494 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15495 .await;
15496 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
15497 ended = Some(next.is_empty());
15498 rig.sync(next).await;
15499 rig.ack(&relayed).await;
15500 let replies = relayed.task.await.unwrap();
15501 assert_eq!(replies.len(), 1, "{replies:?}");
15502 assert_eq!(parse_error(&replies[0])["code"], code);
15503 assert_eq!(
15504 subc_protocol::error_codes::is_retryable_route_open(code),
15505 code == error_codes::SCOPE_CHANGED
15506 );
15507 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15508 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15510 let mut goodbyes = Vec::new();
15513 while let Ok(outbound) = plexus_rx.try_recv() {
15514 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
15515 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
15516 }
15517 assert!(
15518 goodbyes.contains(&(bind_channel, bind_epoch)),
15519 "{goodbyes:?}"
15520 );
15521 assert!(rig
15522 .handler
15523 .registry
15524 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
15525 .unwrap()
15526 .is_some());
15527 }
15528 assert_eq!(ended, Some(true));
15529 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
15532 }
15533
15534 #[tokio::test]
15537 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
15538 struct Case {
15539 name: &'static str,
15540 change: fn(&mut ScopeRecord),
15541 closed: [Option<RouteCloseReason>; 4],
15544 }
15545 use RouteCloseReason::*;
15546 let cases = [
15547 Case {
15548 name: "a carrier entry removed",
15549 change: |r| {
15550 r.carriers.retain(|c| {
15551 c.principal
15552 != Principal::Reserved {
15553 module_id: AFT.to_string(),
15554 }
15555 })
15556 },
15557 closed: [None, Some(ScopeCarrierRemoved), None, None],
15558 },
15559 Case {
15560 name: "a target removed from a carrier",
15561 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
15562 closed: [None, None, None, Some(ScopeCarrierRemoved)],
15563 },
15564 Case {
15565 name: "a bare carrier narrowed to targets",
15566 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
15567 closed: [None, Some(ScopeCarrierRemoved), None, None],
15568 },
15569 Case {
15570 name: "delegates turned off",
15571 change: |r| r.attributes.delegates = false,
15572 closed: [Some(ScopeDelegationChanged); 4],
15573 },
15574 Case {
15575 name: "agent_id changed",
15576 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
15577 closed: [Some(ScopeDelegationChanged); 4],
15578 },
15579 Case {
15580 name: "a carrier added, child owners changed, the record re-sent",
15581 change: |r| {
15582 r.carriers.push(carrier(MAGIC, None));
15583 r.child_owners.push(Principal::Reserved {
15584 module_id: MAGIC.to_string(),
15585 });
15586 },
15587 closed: [None; 4],
15588 },
15589 Case {
15590 name: "a target added",
15591 change: |r| {
15592 r.carriers[1]
15593 .targets
15594 .as_mut()
15595 .unwrap()
15596 .push("third".to_string())
15597 },
15598 closed: [None; 4],
15599 },
15600 Case {
15601 name: "delegates turned on",
15602 change: |r| r.attributes.delegates = true,
15603 closed: [None; 4],
15604 },
15605 ];
15606 for case in cases {
15607 let mut rig = rig().await;
15608 rig.sync(vec![session(1)]).await;
15609 let scope = || Some(rig_selector("s", Some(1)));
15610 let mut routes = [
15611 rig.bound(Some(OWNER), PLEXUS, scope()).await,
15612 rig.bound(Some(AFT), PLEXUS, scope()).await,
15613 rig.bound(Some(BROCA), PLEXUS, scope()).await,
15614 rig.bound(Some(BROCA), OTHER, scope()).await,
15615 ];
15616 let mut record = session(1);
15617 (case.change)(&mut record);
15618 rig.sync(vec![record]).await;
15619 for (index, expected) in case.closed.iter().enumerate() {
15620 let route = &mut routes[index];
15621 match expected {
15622 Some(reason) => {
15623 assert!(!rig.live(route), "{}: route {index} still live", case.name);
15624 assert_eq!(
15625 route.closed_reason(),
15626 *reason,
15627 "{}: route {index}",
15628 case.name
15629 );
15630 }
15631 None => {
15632 assert!(rig.live(route), "{}: route {index} closed", case.name);
15633 assert!(
15634 route.untouched(),
15635 "{}: route {index} was told something",
15636 case.name
15637 );
15638 }
15639 }
15640 }
15641 }
15642 }
15643
15644 #[tokio::test]
15645 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
15646 for next in [Vec::new(), vec![session(2)]] {
15648 let mut rig = rig().await;
15649 rig.sync(vec![session(1)]).await;
15650 let mut route = rig
15651 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15652 .await;
15653 rig.sync(next).await;
15654 assert!(!rig.live(&route));
15655 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
15656 }
15657
15658 let mut rig = rig().await;
15661 let mut child = session(1);
15662 child.scope_ref = "child".to_string();
15663 child.kind = ScopeKind::Worker;
15664 child.parent = Some(ScopeParent::new(
15665 Principal::Reserved {
15666 module_id: OWNER.to_string(),
15667 },
15668 "s".to_string(),
15669 1,
15670 ));
15671 rig.sync(vec![session(1), child.clone()]).await;
15672 let mut child_route = rig
15673 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
15674 .await;
15675 assert_eq!(
15676 child_route.stamp().unwrap().parent_state,
15677 Some(ParentState::Linked)
15678 );
15679 rig.sync(vec![child]).await;
15680 assert!(!rig.live(&child_route));
15681 assert_eq!(
15682 child_route.closed_reason(),
15683 RouteCloseReason::ScopeParentEnded
15684 );
15685 }
15686
15687 #[tokio::test]
15688 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
15689 ) {
15690 let mut rig = rig().await;
15691 rig.sync(vec![session(1)]).await;
15692 let mut route = rig
15693 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15694 .await;
15695 let before = rig.forwarding.published_scope_tag(OWNER, "s");
15696 rig.sync(vec![session(1)]).await;
15697 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
15698 assert!(rig.live(&route) && route.untouched());
15699
15700 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
15704 .forwarding
15705 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
15706 .unwrap()
15707 else {
15708 panic!("the route is bound");
15709 };
15710 binding.flow.acquire_tagged(9, false).await.unwrap();
15711 let mut widened = session(1);
15712 widened.carriers.push(carrier(MAGIC, None));
15713 rig.sync(vec![widened]).await;
15714 assert!(rig.live(&route) && route.untouched());
15715 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15716 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
15717 assert_eq!(binding.flow.in_flight(), 1);
15720 binding
15721 .flow
15722 .acquire_tagged(10, false)
15723 .await
15724 .expect("the flow is still open");
15725 }
15726
15727 #[tokio::test]
15730 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
15731 let mut rig = rig().await;
15732 rig.sync(vec![session(1)]).await;
15733 let mut on_incumbent = rig
15734 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15735 .await;
15736
15737 let (candidate, _candidate_rx) = wide_ctx(9);
15740 let registration = rig
15741 .handler
15742 .registry
15743 .register_candidate_with_control_ops(
15744 manifest(PLEXUS, PROTOCOL_VERSION),
15745 PROTOCOL_VERSION,
15746 candidate.connection_id,
15747 module_baseline_control_ops(),
15748 )
15749 .unwrap();
15750 rig.forwarding
15751 .register_candidate_module_connection(
15752 candidate.connection_id,
15753 PLEXUS.to_string(),
15754 PROTOCOL_VERSION,
15755 manifest_concurrency(®istration.manifest),
15756 candidate.egress.clone(),
15757 )
15758 .unwrap();
15759 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
15760 rig.handler
15761 .registry
15762 .promote_candidate(PLEXUS)
15763 .unwrap()
15764 .unwrap();
15765 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
15766
15767 rig.sync(Vec::new()).await;
15768 assert!(!rig.live(&on_incumbent));
15769 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
15770 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15771 let goodbye = incumbent_rx
15772 .try_recv()
15773 .expect("the superseded endpoint is told")
15774 .frame;
15775 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15776 }
15777 }
15778}
15779
15780#[cfg(test)]
15781mod concurrency_default_exposure_tests {
15782 use super::*;
15783
15784 fn hello_body(role_json: &str) -> Vec<u8> {
15785 format!(
15786 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
15787 )
15788 .into_bytes()
15789 }
15790
15791 fn manifest_from(body: &[u8]) -> ModuleManifest {
15792 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
15793 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
15794 .expect("manifest parses")
15795 }
15796
15797 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
15798
15799 #[test]
15800 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
15801 let body = hello_body(&format!(
15802 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
15803 ));
15804 let manifest = manifest_from(&body);
15805 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
15808 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
15809 }
15810
15811 #[test]
15812 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
15813 let body = hello_body(&format!(
15814 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
15815 ));
15816 let manifest = manifest_from(&body);
15817 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
15818 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
15819 }
15820
15821 #[test]
15822 fn non_management_roles_are_never_reported() {
15823 let body = hello_body(
15824 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
15825 );
15826 let manifest = manifest_from(&body);
15827 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
15828 }
15829}