1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_event_declarations,
25 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
26 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
27 },
28 scope::{
29 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
30 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
31 SCOPE_SYNC_OP,
32 },
33 session::{
34 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
35 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
36 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
37 ROLE_VERSIONS_FIELD,
38 },
39 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
40 Priority, RouteTarget, PROTOCOL_VERSION,
41};
42use tokio::time::{timeout_at, Instant};
43use tracing::{debug, info, warn};
44
45use crate::{
46 caller_identity::CallerIdentity,
47 capability_requirements::{
48 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
49 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
50 RuntimeModule,
51 },
52 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
53 forwarding::{
54 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
55 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
56 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
57 },
58 observability::{
59 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
60 },
61 privileged_prompt::{prompt_text, PrivilegedOperation},
62 provenance::{
63 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
64 },
65 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
66 router::{RouteCtx, RouterError},
67 scopes::{HelloLaunchNonces, ScopeTable},
68 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
69 stderr_tail::{CaptureState, TailEntry},
70 supervise::{
71 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
72 SupervisorHandle, SwapHelloAdmission,
73 },
74 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
75};
76
77pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
83
84const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
85const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
86const CAP_PING_PONG: &str = "ping_pong_v1";
87const CAP_SESSION_ATTACH: &str = "session_attach_v1";
88const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
89
90const SUBC_CONTROL_OPS: &[&str] = &[
91 ops::ACTIVITY_RECORD,
92 ops::ACTIVITY_QUERY,
93 ops::ACTIVITY_SUBSCRIBE,
94 ops::SERVER_DESCRIBE,
95 ops::CATALOG_LIST,
96 ops::ROUTE_OPEN,
97 ops::ROUTE_POLL,
98 ops::ROUTE_CLOSING,
99 ops::ROUTE_CLOSED,
100 ops::SUPERVISOR_LIST,
101 ops::SUPERVISOR_RESTART,
102 ops::SUPERVISOR_SWAP,
103 ops::SUPERVISOR_RELOAD,
104 ops::SUPERVISOR_RESCAN,
105 ops::SUPERVISOR_RELEASE_RESERVED,
106 ops::SUPERVISOR_SET_ENABLED,
107 ops::SUPERVISOR_HEALTH_PROBE,
108 ops::SUPERVISOR_HEALTH,
109 ops::SUPERVISOR_STDERR_TAIL,
110 ops::SUPERVISOR_TERMINALS,
111 ops::SUPERVISOR_ROUTES,
112 ops::SUPERVISOR_PROVENANCE,
113 ops::SUPERVISOR_SPAWN_SNAPSHOT,
114 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
115];
116
117const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
118 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
119 "supervisor.live_roots",
120 SCOPE_SYNC_OP,
121 SCOPE_APPLY_OP,
122 SCOPE_DESCRIBE_OP,
123 "operator.confirm",
124];
125
126const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
130
131const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
132
133pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
141
142pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
157
158pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
176
177const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
178const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
179
180fn reload_verdict(
181 configured: &Path,
182 spawned_from: Option<&Path>,
183 image: subc_control::RunningImageAgreement,
184) -> PendingReloadVerdict {
185 let path = match spawned_from {
186 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
187 ReloadPathAgreement::Match
188 }
189 Some(spawned_from) => ReloadPathAgreement::Mismatch {
190 configured: configured.to_path_buf(),
191 spawned_from: spawned_from.to_path_buf(),
192 },
193 None => ReloadPathAgreement::Unavailable {
194 reason: if matches!(
195 image,
196 subc_control::RunningImageAgreement::Unavailable {
197 reason: subc_control::RunningImageUnavailableReason::NotRunning
198 }
199 ) {
200 ReloadPathUnavailableReason::NotRunning
201 } else {
202 ReloadPathUnavailableReason::SpawnedPathUnavailable
203 },
204 },
205 };
206 PendingReloadVerdict { path, image }
207}
208
209fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
210 if configured == spawned_from {
211 return true;
212 }
213 #[cfg(windows)]
214 {
215 match (
221 std::fs::canonicalize(configured),
222 std::fs::canonicalize(spawned_from),
223 ) {
224 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
225 _ => false,
226 }
227 }
228 #[cfg(not(windows))]
229 {
230 false
231 }
232}
233
234#[derive(Clone)]
235struct DaemonProvenanceFacts {
236 build: DaemonBuildProvenance,
237 pid: Option<u32>,
238 started_at_ms: Option<u64>,
239 start_clock: Option<crate::clock::StartClock>,
240 executable_path: Option<PathBuf>,
241 executable_identity: Option<SpawnedFileIdentity>,
242 process_start_time: Option<u64>,
243 probe: ExecutableIdentityProbe,
244}
245
246impl Default for DaemonProvenanceFacts {
247 fn default() -> Self {
248 Self {
249 build: DaemonBuildProvenance {
250 build_git_sha: None,
251 build_lock_digest: None,
252 },
253 pid: None,
254 started_at_ms: None,
255 start_clock: None,
256 executable_path: None,
257 executable_identity: None,
258 process_start_time: None,
259 probe: ExecutableIdentityProbe::default(),
260 }
261 }
262}
263
264#[derive(Debug, Clone)]
265struct SupervisorRescanContext {
266 supervisor: Supervisor,
267 config_path: PathBuf,
268 configured_port: Option<u16>,
269 storage_config: Option<crate::daemon_config::StorageConfig>,
270 admission_facts_carrier_module_id: Option<String>,
271 admission_facts_targets: Option<Vec<String>>,
272 scope_authority_owners: Vec<String>,
273 privileged_operator_prompt: PrivilegedOperatorPrompt,
274}
275
276const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
297 "reloading",
298 "supervisor_not_live",
299 "registration_not_active",
300 "no_forwarding_connection",
301 "relay_send_failed",
302];
303
304#[derive(Clone)]
306pub struct ControlHandler {
307 registry: Arc<Registry>,
308 activity: Option<crate::activity::ActivityLedger>,
309 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
310 forwarding: Arc<ForwardingTable>,
311 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
312 supervisor: SupervisorHandle,
313 subc_capabilities: Arc<[String]>,
314 route_bind_relay_timeout: Duration,
318 route_bind_relay_timeouts: BTreeMap<String, Duration>,
322 route_bind_breakers: RouteBindBreakers,
325 route_bind_concurrency: RouteBindConcurrency,
328 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
333 route_bind_breaker_threshold: u32,
335 route_bind_breaker_cooldown: Duration,
337 health_probe_timeout: Duration,
338 storage_config: Option<crate::daemon_config::StorageConfig>,
341 machine_id: Option<crate::machine_id::MachineId>,
345 admission_facts_carrier_module_id: Option<String>,
346 admission_facts_targets: Option<Vec<String>>,
347 scopes: Arc<RwLock<ScopeTable>>,
351 scope_deadline_added: Arc<tokio::sync::Notify>,
355 scope_authority_owners: Vec<String>,
358 privileged_operator_prompt: PrivilegedOperatorPrompt,
360 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
363 #[cfg(test)]
364 caller_lookup: Option<TestCallerLookup>,
365 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
368 rescan: Option<SupervisorRescanContext>,
369 connected_clients: ConnectedClients,
370 counters: DaemonCounters,
371 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
372 daemon_provenance: DaemonProvenanceFacts,
373 #[cfg(test)]
374 control_dispatch_delay: Option<Duration>,
375 #[cfg(test)]
376 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
377 #[cfg(test)]
381 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
382 #[cfg(test)]
388 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
389}
390
391impl fmt::Debug for ControlHandler {
392 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
393 f.debug_struct("ControlHandler")
394 .field("registry", &self.registry)
395 .field("forwarding", &self.forwarding)
396 .field("process_liveness", &self.process_liveness.is_some())
397 .field("supervisor", &self.supervisor)
398 .field("subc_capabilities", &self.subc_capabilities)
399 .finish()
400 }
401}
402
403struct RouteOpenRequest {
404 target: RouteTarget,
405 identity: BindIdentity,
406 consumer_identity: Option<ConsumerIdentity>,
407 consumer_capabilities: Option<Vec<String>>,
408 role_versions: Option<BTreeMap<String, String>>,
409 admission_facts: Option<serde_json::Value>,
410 scope: Option<ScopeSelector>,
411}
412
413struct RouteBindReservationGuard {
414 forwarding: Arc<ForwardingTable>,
415 endpoint: ModuleEndpointId,
416 relay_corr: u64,
417 armed: bool,
418}
419
420struct ModuleControlRpcGuard {
421 forwarding: Arc<ForwardingTable>,
422 endpoint: ModuleEndpointId,
423 corr: u64,
424 armed: bool,
425}
426
427impl ModuleControlRpcGuard {
428 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
429 Self {
430 forwarding,
431 endpoint,
432 corr,
433 armed: true,
434 }
435 }
436
437 fn disarm(&mut self) {
438 self.armed = false;
439 }
440}
441
442impl Drop for ModuleControlRpcGuard {
443 fn drop(&mut self) {
444 if self.armed {
445 let _ = self
446 .forwarding
447 .cancel_module_control_rpc(self.endpoint, self.corr);
448 }
449 }
450}
451
452impl RouteBindReservationGuard {
453 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
454 Self {
455 forwarding,
456 endpoint,
457 relay_corr,
458 armed: true,
459 }
460 }
461
462 fn release_and_disarm(&mut self) {
463 if !self.armed {
464 return;
465 }
466 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
467 self.endpoint,
468 self.relay_corr,
469 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
470 ) {
471 send_goodbye_target_best_effort(
472 &self.forwarding.counters(),
473 &target,
474 "canceled route.bind",
475 );
476 }
477 self.armed = false;
478 }
479
480 fn disarm(&mut self) {
481 self.armed = false;
482 }
483}
484
485impl Drop for RouteBindReservationGuard {
486 fn drop(&mut self) {
487 self.release_and_disarm();
488 }
489}
490
491#[derive(Debug, Clone, Default)]
521pub(crate) struct RouteBindBreakers {
522 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
523}
524
525#[derive(Debug, Clone, Default)]
526pub(crate) struct RouteBindConcurrency {
527 modules: Arc<Mutex<HashMap<String, usize>>>,
528}
529
530struct RouteBindConcurrencyGuard {
531 concurrency: RouteBindConcurrency,
532 module_id: String,
533}
534
535impl RouteBindConcurrency {
536 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
539 let mut modules = self
540 .modules
541 .lock()
542 .expect("route.bind concurrency mutex poisoned");
543 let in_flight = modules.entry(module_id.to_string()).or_default();
544 if *in_flight >= limit {
545 return Err(*in_flight);
546 }
547 *in_flight += 1;
548 Ok(RouteBindConcurrencyGuard {
549 concurrency: self.clone(),
550 module_id: module_id.to_string(),
551 })
552 }
553}
554
555impl Drop for RouteBindConcurrencyGuard {
556 fn drop(&mut self) {
557 let mut modules = self
558 .concurrency
559 .modules
560 .lock()
561 .expect("route.bind concurrency mutex poisoned");
562 let remove = {
563 let in_flight = modules
564 .get_mut(&self.module_id)
565 .expect("admitted route.bind has a concurrency entry");
566 *in_flight -= 1;
567 *in_flight == 0
568 };
569 if remove {
570 modules.remove(&self.module_id);
571 }
572 }
573}
574
575#[derive(Debug, Default)]
576struct ModuleBreakerState {
577 consecutive_timeouts: u32,
579 cooldown_until: Option<Instant>,
582 probe_in_flight: Option<Arc<()>>,
587}
588
589enum RouteBindAdmission<'a> {
591 Admitted {
592 guard: RouteBindBreakerGuard<'a>,
593 probe: bool,
596 },
597 Refused {
598 consecutive_timeouts: u32,
599 retry_in: Duration,
603 probe_in_flight: bool,
604 },
605}
606
607struct RouteBindBreakerGuard<'a> {
614 breakers: RouteBindBreakers,
615 module_id: &'a str,
616 probe_token: Option<Arc<()>>,
617 settled: bool,
618}
619
620impl RouteBindBreakerGuard<'_> {
621 fn record_accepted(&mut self) -> bool {
625 self.settled = true;
626 self.breakers.record_accepted(self.module_id)
627 }
628
629 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
632 self.settled = true;
633 self.breakers.record_timeout(
634 self.module_id,
635 self.probe_token.as_ref(),
636 threshold,
637 cooldown,
638 )
639 }
640
641 fn record_inconclusive(&mut self) {
650 self.settled = true;
651 self.breakers
652 .record_inconclusive(self.module_id, self.probe_token.as_ref());
653 }
654}
655
656impl Drop for RouteBindBreakerGuard<'_> {
657 fn drop(&mut self) {
658 if !self.settled {
659 self.breakers
660 .record_inconclusive(self.module_id, self.probe_token.as_ref());
661 }
662 }
663}
664
665struct BreakerOpened {
669 consecutive_timeouts: u32,
670 reopened_after_probe: bool,
673}
674
675impl RouteBindBreakers {
676 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
677 self.modules
678 .lock()
679 .expect("route.bind breaker mutex poisoned")
680 }
681
682 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
685 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
686 probe: probe_token.is_some(),
687 guard: RouteBindBreakerGuard {
688 breakers: self.clone(),
689 module_id,
690 probe_token,
691 settled: false,
692 },
693 };
694
695 let mut modules = self.lock();
696 let Some(state) = modules.get_mut(module_id) else {
697 return admitted(None);
698 };
699 let Some(cooldown_until) = state.cooldown_until else {
700 return admitted(None);
701 };
702 if state.probe_in_flight.is_some() {
703 return RouteBindAdmission::Refused {
704 consecutive_timeouts: state.consecutive_timeouts,
705 retry_in: Duration::ZERO,
706 probe_in_flight: true,
707 };
708 }
709 let now = Instant::now();
710 if now < cooldown_until {
711 return RouteBindAdmission::Refused {
712 consecutive_timeouts: state.consecutive_timeouts,
713 retry_in: cooldown_until - now,
714 probe_in_flight: false,
715 };
716 }
717 let token = Arc::new(());
718 state.probe_in_flight = Some(Arc::clone(&token));
719 admitted(Some(token))
720 }
721
722 fn record_accepted(&self, module_id: &str) -> bool {
723 self.lock()
724 .remove(module_id)
725 .is_some_and(|state| state.cooldown_until.is_some())
726 }
727
728 fn record_timeout(
729 &self,
730 module_id: &str,
731 probe_token: Option<&Arc<()>>,
732 threshold: u32,
733 cooldown: Duration,
734 ) -> Option<BreakerOpened> {
735 let mut modules = self.lock();
736 let state = modules.entry(module_id.to_string()).or_default();
737 let was_open = state.cooldown_until.is_some();
738 let was_probe = Self::owns_probe(state, probe_token);
739 if was_probe {
740 state.probe_in_flight = None;
741 }
742 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
743 if state.consecutive_timeouts < threshold {
744 return None;
745 }
746 state.cooldown_until = Some(Instant::now() + cooldown);
747 Some(BreakerOpened {
748 consecutive_timeouts: state.consecutive_timeouts,
749 reopened_after_probe: was_open && was_probe,
750 })
751 }
752
753 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
754 state
758 .probe_in_flight
759 .as_ref()
760 .zip(token)
761 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
762 }
763
764 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
765 if let Some(state) = self.lock().get_mut(module_id) {
766 if Self::owns_probe(state, probe_token) {
767 state.probe_in_flight = None;
768 }
769 }
770 }
771
772 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
787 self.lock()
788 .remove(module_id)
789 .map(|state| state.consecutive_timeouts)
790 .filter(|discarded| *discarded > 0)
791 }
792
793 fn open_snapshot(&self) -> Option<serde_json::Value> {
800 let now = Instant::now();
801 let modules = self.lock();
802 let open = modules
803 .iter()
804 .filter_map(|(module_id, state)| {
805 let cooldown_until = state.cooldown_until?;
806 Some((
807 module_id.clone(),
808 serde_json::json!({
809 "consecutive_timeouts": state.consecutive_timeouts,
810 "cooldown_remaining_ms":
811 cooldown_until.saturating_duration_since(now).as_millis() as u64,
812 "probe_in_flight": state.probe_in_flight.is_some(),
813 }),
814 ))
815 })
816 .collect::<serde_json::Map<String, serde_json::Value>>();
817 (!open.is_empty()).then_some(serde_json::Value::Object(open))
818 }
819}
820
821impl ControlHandler {
822 pub fn new(registry: Arc<Registry>) -> Self {
823 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
824 }
825
826 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
827 let counters = forwarding.counters();
828 let route_bind_breakers = forwarding.route_bind_breakers();
832 let route_bind_concurrency = forwarding.route_bind_concurrency();
833 let route_outages = forwarding.route_outages();
834 Self {
835 registry,
836 activity: None,
837 activity_streams: Arc::new(Mutex::new(HashMap::new())),
838 forwarding,
839 process_liveness: None,
840 supervisor: SupervisorHandle::new(),
841 subc_capabilities: Arc::from([
842 CAP_MANIFEST_REGISTRATION.to_string(),
843 CAP_CHANNEL_LIFECYCLE.to_string(),
844 CAP_PING_PONG.to_string(),
845 CAP_SESSION_ATTACH.to_string(),
846 CAP_ADMISSION_FACTS_RELAY.to_string(),
847 CAP_SCOPES_V1.to_string(),
848 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
849 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
850 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
851 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
852 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
853 ]),
854 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
855 route_bind_relay_timeouts: BTreeMap::new(),
856 route_bind_breakers,
857 route_bind_concurrency,
858 route_outages,
859 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
860 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
861 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
862 storage_config: None,
863 machine_id: None,
864 admission_facts_carrier_module_id: None,
865 admission_facts_targets: None,
866 scopes: Arc::new(RwLock::new(ScopeTable::new(
867 crate::daemon_config::default_scope_authority_owners(),
868 ))),
869 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
870 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
871 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
872 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
873 #[cfg(test)]
874 caller_lookup: None,
875 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
876 rescan: None,
877 connected_clients: ConnectedClients::new(),
878 counters,
879 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
880 daemon_provenance: DaemonProvenanceFacts::default(),
881 #[cfg(test)]
882 control_dispatch_delay: None,
883 #[cfg(test)]
884 provenance_probe_override: None,
885 #[cfg(test)]
886 route_bind_relay_sent_hold: None,
887 #[cfg(test)]
888 hello_registry_to_forwarding_hold: None,
889 }
890 }
891
892 pub fn with_storage_config(
895 mut self,
896 storage_config: Option<crate::daemon_config::StorageConfig>,
897 ) -> Self {
898 self.storage_config = storage_config;
899 self
900 }
901
902 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
905 self.activity = Some(crate::activity::ActivityLedger::open(
906 run_dir.join("activity.jsonl"),
907 incarnation,
908 ));
909 self.supervisor.set_activity(self.activity.clone());
910 self
911 }
912
913 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
916 self.machine_id = machine_id;
917 self
918 }
919
920 pub fn with_admission_facts_config(
924 mut self,
925 carrier_module_id: Option<String>,
926 targets: Option<Vec<String>>,
927 ) -> Self {
928 self.admission_facts_carrier_module_id = carrier_module_id;
929 self.admission_facts_targets = targets;
930 self
931 }
932
933 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
937 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
938 self.scope_authority_owners = owners;
939 self
940 }
941
942 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
944 self.privileged_operator_prompt = mode;
945 self
946 }
947
948 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
951 self.route_bind_relay_timeout = timeout;
952 self
953 }
954
955 pub fn with_route_bind_relay_timeouts(
961 mut self,
962 timeouts: impl IntoIterator<Item = (String, Duration)>,
963 ) -> Self {
964 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
965 self
966 }
967
968 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
974 self.route_bind_relay_timeouts
975 .get(module_id)
976 .copied()
977 .unwrap_or(self.route_bind_relay_timeout)
978 }
979
980 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
988 self.route_bind_breaker_threshold = threshold.max(1);
989 self.route_bind_breaker_cooldown = cooldown;
990 self
991 }
992
993 #[cfg(test)]
994 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
995 self.health_probe_timeout = timeout;
996 self
997 }
998
999 #[cfg(test)]
1000 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1001 self.control_dispatch_delay = Some(delay);
1002 self
1003 }
1004
1005 #[cfg(test)]
1009 pub(crate) fn with_route_bind_relay_sent_hold(
1010 mut self,
1011 hold: Arc<tokio::sync::Barrier>,
1012 ) -> Self {
1013 self.route_bind_relay_sent_hold = Some(hold);
1014 self
1015 }
1016
1017 #[cfg(test)]
1022 pub(crate) fn with_hello_registry_to_forwarding_hold(
1023 mut self,
1024 hold: Arc<std::sync::Barrier>,
1025 ) -> Self {
1026 self.hello_registry_to_forwarding_hold = Some(hold);
1027 self
1028 }
1029
1030 #[cfg(test)]
1031 fn hold_between_registry_and_forwarding(&self) {
1032 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1033 hold.wait();
1034 hold.wait();
1035 }
1036 }
1037
1038 pub fn with_process_liveness(
1039 mut self,
1040 process_liveness: Arc<dyn ModuleProcessLiveness>,
1041 ) -> Self {
1042 self.process_liveness = Some(process_liveness);
1043 self
1044 }
1045
1046 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1047 self.supervisor = supervisor;
1048 self.supervisor.set_activity(self.activity.clone());
1049 self
1050 }
1051
1052 pub fn with_daemon_provenance(
1053 mut self,
1054 pid: u32,
1055 started_at_ms: u64,
1056 executable_path: Option<PathBuf>,
1057 build_git_sha: Option<String>,
1058 build_lock_digest: Option<String>,
1059 ) -> Self {
1060 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1061 let process_start_time = process_start_time(pid);
1062 self.daemon_provenance = DaemonProvenanceFacts {
1063 build: DaemonBuildProvenance {
1064 build_git_sha,
1065 build_lock_digest,
1066 },
1067 pid: Some(pid),
1068 started_at_ms: Some(started_at_ms),
1069 start_clock: None,
1070 executable_path,
1071 executable_identity,
1072 process_start_time,
1073 probe: ExecutableIdentityProbe::default(),
1074 };
1075 self
1076 }
1077
1078 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1079 self.daemon_provenance.start_clock = Some(clock);
1080 self
1081 }
1082
1083 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1085 self.scopes
1086 .write()
1087 .unwrap_or_else(|poisoned| poisoned.into_inner())
1088 .set_wall_clock(Arc::new(clock));
1089 self
1090 }
1091
1092 #[cfg(test)]
1093 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1094 self.provenance_probe_override = Some(result);
1095 self
1096 }
1097
1098 pub fn with_capability_config(
1102 self,
1103 modules: impl IntoIterator<Item = (String, bool)>,
1104 reserved_capabilities: BTreeMap<String, String>,
1105 ) -> Self {
1106 self.capability_evaluator
1107 .configure(modules, reserved_capabilities);
1108 self
1109 }
1110
1111 pub fn with_supervisor_rescan(
1112 mut self,
1113 supervisor: Supervisor,
1114 config_path: impl Into<PathBuf>,
1115 configured_port: Option<u16>,
1116 ) -> Self {
1117 self.rescan = Some(SupervisorRescanContext {
1118 supervisor,
1119 config_path: config_path.into(),
1120 configured_port,
1121 storage_config: self.storage_config.clone(),
1122 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1123 admission_facts_targets: self.admission_facts_targets.clone(),
1124 scope_authority_owners: self.scope_authority_owners.clone(),
1125 privileged_operator_prompt: self.privileged_operator_prompt,
1126 });
1127 self
1128 }
1129
1130 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1131 self.connected_clients = connected_clients;
1132 self
1133 }
1134
1135 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1136 Arc::clone(&self.forwarding)
1137 }
1138
1139 pub(crate) fn counters(&self) -> DaemonCounters {
1140 self.counters.clone()
1141 }
1142
1143 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1146 tokio::spawn(async move {
1147 loop {
1148 self.capability_evaluator
1149 .wait_for_change_or_deadline()
1150 .await;
1151 self.refresh_capability_requirements();
1152 }
1153 });
1154 }
1155
1156 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1162 tokio::spawn(async move {
1163 loop {
1164 let has_deadlines = self
1165 .scopes
1166 .read()
1167 .unwrap_or_else(|poisoned| poisoned.into_inner())
1168 .has_deadlines();
1169 if !has_deadlines {
1170 self.scope_deadline_added.notified().await;
1171 continue;
1172 }
1173 tokio::time::sleep(Duration::from_secs(1)).await;
1174 if let Err(error) = self.sweep_expired_scopes() {
1175 warn!(%error, "scope expiry sweep failed");
1176 }
1177 }
1178 });
1179 }
1180
1181 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1182 let mut table = self
1183 .scopes
1184 .write()
1185 .unwrap_or_else(|poisoned| poisoned.into_inner());
1186 let swept = table.sweep_expired();
1187 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1188 drop(table);
1189 self.close_scope_drained_routes(drained);
1190 Ok(())
1191 }
1192
1193 fn publish_scope_changes(
1194 &self,
1195 changes: &[crate::scopes::ScopeTagChange],
1196 expired: &[crate::scopes::ScopeExpired],
1197 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1198 if changes.is_empty() {
1199 return Ok(Vec::new());
1200 }
1201 let drained = self
1202 .forwarding
1203 .publish_scope_changes(changes)
1204 .map_err(RouterError::Forwarding)?;
1205 let mut counts = HashMap::new();
1206 for route in &drained {
1207 *counts
1208 .entry((
1209 route.scope.owner.as_str(),
1210 route.scope.scope_ref.as_str(),
1211 route.scope.tag.scope_epoch,
1212 ))
1213 .or_insert(0usize) += 1;
1214 }
1215 for scope in expired {
1216 let routes_closed = counts
1217 .get(&(
1218 scope.owner.as_str(),
1219 scope.scope_ref.as_str(),
1220 scope.scope_epoch,
1221 ))
1222 .copied()
1223 .unwrap_or(0);
1224 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1225 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1226 routes_closed, "scope expired");
1227 }
1228 Ok(drained)
1229 }
1230
1231 fn runtime_capability_snapshot(
1232 &self,
1233 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1234 let runtime = self
1235 .supervisor
1236 .list()
1237 .into_iter()
1238 .map(|module| {
1239 let status = module.status().map_err(|err| {
1240 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1241 })?;
1242 Ok(RuntimeModule {
1243 module_id: status.module_id,
1244 state: status.state,
1245 enabled: status.enabled,
1246 })
1247 })
1248 .collect::<Result<Vec<_>, RouterError>>()?;
1249 let registrations = self.registry.list_registered_modules().map_err(|err| {
1255 RouterError::backend(
1256 0,
1257 0,
1258 format!("failed to list capability registrations: {err}"),
1259 )
1260 })?;
1261 let registrations = registrations
1262 .into_iter()
1263 .map(|registration| RegisteredModule {
1264 module_id: registration.manifest.module_id,
1265 module_version: registration.manifest.module_version,
1266 capabilities: registration.manifest.capabilities,
1267 })
1268 .collect();
1269 Ok((runtime, registrations))
1270 }
1271
1272 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1279 let cached_registration = RegisteredModule {
1280 module_id: registration.manifest.module_id.clone(),
1281 module_version: registration.manifest.module_version.clone(),
1282 capabilities: registration.manifest.capabilities.clone(),
1283 };
1284 if self.capability_evaluator.record_hello(&cached_registration) {
1285 warn!(
1286 module_id = %cached_registration.module_id,
1287 "capability claims drifted from the cached manifest"
1288 );
1289 }
1290 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1291 self.enforce_capability_denies();
1292 }
1293 self.refresh_capability_requirements();
1294 }
1295
1296 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1300 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1301 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1302 self.supervisor.set_swap_promotion_observer(observer);
1303 }
1304
1305 pub fn refresh_capability_requirements(&self) {
1306 match self.runtime_capability_snapshot() {
1307 Ok((runtime, registrations)) => {
1308 log_requirement_events(
1309 self.capability_evaluator
1310 .evaluate_now(&runtime, ®istrations),
1311 );
1312 }
1313 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1314 }
1315 }
1316
1317 fn enforce_capability_denies(&self) {
1321 let registrations = match self.registry.list_registered_modules() {
1322 Ok(snapshot) => snapshot,
1323 Err(err) => {
1324 warn!(error = %err, "failed to read registrations for capability deny census");
1325 return;
1326 }
1327 };
1328 let manifests = registrations
1329 .into_iter()
1330 .map(|registration| {
1331 (
1332 registration.manifest.module_id.clone(),
1333 registration.manifest,
1334 )
1335 })
1336 .collect::<BTreeMap<_, _>>();
1337 let census = match self.forwarding.route_census(None) {
1338 Ok(census) => census,
1339 Err(err) => {
1340 warn!(error = %err, "failed to read route census for capability deny enforcement");
1341 return;
1342 }
1343 };
1344
1345 for (target_module_id, routes) in census {
1346 let Some(target_manifest) = manifests.get(&target_module_id) else {
1347 continue;
1348 };
1349 let mut closed_routes = Vec::new();
1350 let mut module_goodbyes = Vec::new();
1351 for route in routes {
1352 let Principal::Reserved {
1353 module_id: opening_module_id,
1354 } = &route.principal
1355 else {
1356 continue;
1357 };
1358 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1359 continue;
1360 };
1361 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1362 continue;
1363 };
1364
1365 match self.forwarding.release_client_route(
1366 route.goodbye_target.connection_id,
1367 route.goodbye_target.channel,
1368 route.goodbye_target.epoch,
1369 ) {
1370 Ok(RouteRelease::Removed(module_goodbye)) => {
1371 warn!(
1372 opening_module_id,
1373 target_module_id,
1374 capability,
1375 "force-closing route because an attested capability deny edge now matches"
1376 );
1377 closed_routes.push(route);
1378 module_goodbyes.push(module_goodbye);
1379 }
1380 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1381 Err(err) => warn!(
1382 opening_module_id,
1383 target_module_id,
1384 capability,
1385 error = %err,
1386 "failed to force-close capability-denied route"
1387 ),
1388 }
1389 }
1390
1391 if closed_routes.is_empty() {
1392 continue;
1393 }
1394 send_route_control_pushes(
1395 &self.forwarding,
1396 closed_routes,
1397 ClientControlPush::RouteClosed {
1398 module_id: target_module_id,
1399 channels: Vec::new(),
1400 reason: RouteCloseReason::CapabilityDenied,
1401 drained: false,
1402 abandoned: 0,
1403 excluded_subscriptions: 0,
1404 terminal: Some(false),
1405 },
1406 );
1407 self.emit_route_goodbyes(module_goodbyes);
1408 }
1409 }
1410
1411 fn not_ready_reason(
1417 &self,
1418 registration: &crate::registry::ModuleRegistration,
1419 ) -> Option<NotReadyReason> {
1420 if !registration.ready {
1421 return Some(NotReadyReason {
1422 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1423 capability: None,
1424 });
1425 }
1426 self.first_unprovided_required_capability(registration)
1427 .map(|capability| NotReadyReason {
1428 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1429 capability: Some(capability),
1430 })
1431 }
1432
1433 fn first_unprovided_required_capability(
1451 &self,
1452 registration: &crate::registry::ModuleRegistration,
1453 ) -> Option<String> {
1454 let required = registration
1455 .manifest
1456 .capabilities
1457 .iter()
1458 .flat_map(|declarations| declarations.requires.iter())
1459 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1460 .map(|requirement| requirement.capability.as_str())
1461 .collect::<BTreeSet<_>>();
1462 if required.is_empty() {
1463 return None;
1464 }
1465 let module_id = registration.manifest.module_id.as_str();
1466 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1467 if required
1468 .iter()
1469 .any(|capability| verdict(capability).is_none())
1470 {
1471 self.refresh_capability_requirements();
1472 }
1473 required
1474 .into_iter()
1475 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1476 .map(str::to_string)
1477 }
1478
1479 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1480 self.capability_evaluator
1481 .statuses()
1482 .into_iter()
1483 .map(capability_requirement_status)
1484 .collect()
1485 }
1486
1487 fn deregister_connection(
1494 &self,
1495 connection_id: ConnectionId,
1496 reason: RegistrationEndReason,
1497 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1498 self.registry
1499 .deregister_connection_with_reason(connection_id, reason)
1500 }
1501
1502 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1503 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1504 return None;
1505 }
1506 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1507 parse_client_control_request(&frame.body)
1508 else {
1509 return None;
1510 };
1511 Some(target_module_id(&target).to_string())
1512 }
1513
1514 pub(crate) fn route_open_capacity_refusal(
1515 &self,
1516 ctx: &RouteCtx,
1517 frame: &Frame,
1518 target_module_id: &str,
1519 in_flight: usize,
1520 limit: usize,
1521 ) -> Result<Frame, RouterError> {
1522 self.route_open_admission_refusal_frame(
1523 ctx,
1524 frame,
1525 target_module_id,
1526 "open_admission_full",
1527 (in_flight, limit),
1528 format!(
1529 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1530 ),
1531 )
1532 }
1533
1534 fn route_open_target_capacity_refusal(
1535 &self,
1536 ctx: &RouteCtx,
1537 frame: &Frame,
1538 target_module_id: &str,
1539 in_flight: usize,
1540 ) -> Result<Frame, RouterError> {
1541 self.route_open_admission_refusal_frame(
1542 ctx,
1543 frame,
1544 target_module_id,
1545 "target_binds_full",
1546 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1547 format!(
1548 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1549 ),
1550 )
1551 }
1552
1553 fn route_open_admission_refusal_frame(
1561 &self,
1562 ctx: &RouteCtx,
1563 frame: &Frame,
1564 target_module_id: &str,
1565 reason: &'static str,
1566 (in_flight, limit): (usize, usize),
1567 message: impl Into<String>,
1568 ) -> Result<Frame, RouterError> {
1569 let code = error_codes::TARGET_UNAVAILABLE;
1570 self.counters.increment_route_open_refused(code);
1571 info!(
1572 target: "control",
1573 code,
1574 reason,
1575 module_id = ?target_module_id,
1576 connection_id = ctx.connection_id.get(),
1577 in_flight,
1578 limit,
1579 "route.open refused"
1580 );
1581 control_error_frame(frame, code, message.into())
1582 }
1583
1584 #[cfg(test)]
1590 pub fn handle_control(
1591 &self,
1592 connection_id: ConnectionId,
1593 frame: Frame,
1594 ) -> Result<Vec<Frame>, RouterError> {
1595 match frame.header.ty {
1596 FrameType::Ping => Ok(vec![pong(&frame)?]),
1597 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1598 FrameType::Goodbye => self.handle_goodbye(connection_id),
1599 ty => Ok(vec![control_error_frame(
1600 &frame,
1601 "unsupported_control_frame",
1602 format!("unsupported channel-0 frame {ty:?}"),
1603 )?]),
1604 }
1605 }
1606
1607 pub async fn handle_control_frame(
1608 &self,
1609 ctx: &RouteCtx,
1610 frame: Frame,
1611 ) -> Result<Vec<Frame>, RouterError> {
1612 self.handle_control_frame_timed(ctx, frame, None).await
1613 }
1614
1615 pub(crate) fn record_caller_endpoints(
1616 &self,
1617 connection: ConnectionId,
1618 endpoints: Option<(SocketAddr, SocketAddr)>,
1619 ) {
1620 let mut callers = self
1621 .caller_endpoints
1622 .lock()
1623 .unwrap_or_else(|p| p.into_inner());
1624 callers.remove(&connection);
1625 if let Some(endpoints) = endpoints {
1626 callers.insert(connection, endpoints);
1627 }
1628 }
1629
1630 #[cfg(test)]
1631 pub(crate) fn with_caller_lookup_for_test(
1632 mut self,
1633 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1634 ) -> Self {
1635 self.caller_lookup = Some(Arc::new(lookup));
1636 self
1637 }
1638
1639 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1640 let endpoints = self
1641 .caller_endpoints
1642 .lock()
1643 .unwrap_or_else(|p| p.into_inner())
1644 .get(&connection)
1645 .copied();
1646 #[cfg(test)]
1647 let lookup = self.caller_lookup.clone();
1648 tokio::task::spawn_blocking(move || {
1649 #[cfg(test)]
1650 if let Some(lookup) = lookup {
1651 return lookup(endpoints);
1652 }
1653 endpoints
1654 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1655 .unwrap_or_default()
1656 })
1657 .await
1658 .unwrap_or_default()
1659 }
1660
1661 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1662 frame.header.channel == 0
1663 && frame.header.ty == FrameType::Request
1664 && self
1665 .forwarding
1666 .module_endpoint_for_connection(ctx.connection_id)
1667 .is_ok_and(|endpoint| endpoint.is_none())
1668 && parse_client_control_request(&frame.body)
1669 .ok()
1670 .is_some_and(|request| privileged_operation(&request).is_some())
1671 }
1672
1673 pub(crate) async fn handle_control_frame_timed(
1674 &self,
1675 ctx: &RouteCtx,
1676 frame: Frame,
1677 dispatch_started_at: Option<StdInstant>,
1678 ) -> Result<Vec<Frame>, RouterError> {
1679 match frame.header.ty {
1680 FrameType::Ping => Ok(vec![pong(&frame)?]),
1681 FrameType::Hello => {
1682 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1683 }
1684 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1685 FrameType::Cancel => {
1686 if self
1689 .forwarding
1690 .operator_confirms()
1691 .cancel(ctx.connection_id, frame.header.corr)
1692 || self
1693 .supervisor
1694 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1695 || self.cancel_activity_subscription(ctx, &frame)?
1696 {
1697 Ok(Vec::new())
1698 } else {
1699 Ok(vec![control_error_frame(
1700 &frame,
1701 "unknown_subscription",
1702 "no subscription has this correlation id",
1703 )?])
1704 }
1705 }
1706 FrameType::Request => {
1707 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1710 if op
1711 .as_ref()
1712 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1713 {
1714 return self.handle_activity_record(ctx, frame).await;
1715 }
1716 if op.as_ref().is_some_and(|probe| {
1717 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1718 }) {
1719 return self.handle_activity_read(ctx, frame).await;
1720 }
1721 if op
1722 .as_ref()
1723 .is_some_and(|probe| probe.op == "operator.confirm")
1724 {
1725 return self.handle_operator_confirm(ctx, frame);
1726 }
1727 if self
1728 .forwarding
1729 .module_endpoint_for_connection(ctx.connection_id)
1730 .map_err(RouterError::Forwarding)?
1731 .is_some()
1732 {
1733 if !is_known_module_request_op(&frame.body) {
1734 return Ok(vec![control_error_frame(
1735 &frame,
1736 "unsupported_control_frame",
1737 "module-originated channel-0 REQUEST is not supported",
1738 )?]);
1739 }
1740 let request = match parse_module_control_request_from_module(&frame.body) {
1741 Ok(request) => request,
1742 Err((err, ControlRequestBodyError::UnknownOp)) => {
1743 return Ok(vec![control_error_frame(
1744 &frame,
1745 "unsupported_control_frame",
1746 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1747 )?])
1748 }
1749 Err((err, ControlRequestBodyError::InvalidBody)) => {
1750 return Ok(vec![control_error_frame(
1751 &frame,
1752 "invalid_control_body",
1753 format!("malformed module control body: {err}"),
1754 )?])
1755 }
1756 };
1757 let op = module_control_request_op(&request);
1758 let corr = frame.header.corr;
1759 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1760 let result =
1761 self.handle_module_control_request(ctx.connection_id, frame, request);
1762 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1763 return result;
1764 }
1765
1766 if is_known_module_request_op(&frame.body) {
1767 return Ok(vec![control_error_frame(
1768 &frame,
1769 "not_registered",
1770 "catalog.update requires an active module registration owned by this connection",
1771 )?]);
1772 }
1773
1774 let request = match parse_client_control_request(&frame.body) {
1775 Ok(request) => request,
1776 Err((err, ControlRequestBodyError::UnknownOp)) => {
1777 return Ok(vec![control_error_frame(
1778 &frame,
1779 "unknown_control_op",
1780 format!("unknown client control op: {err}"),
1781 )?])
1782 }
1783 Err((err, ControlRequestBodyError::InvalidBody)) => {
1784 return Ok(vec![control_error_frame(
1785 &frame,
1786 "invalid_control_body",
1787 format!("malformed client control body: {err}"),
1788 )?])
1789 }
1790 };
1791 let op = client_control_request_op(&request);
1792 let corr = frame.header.corr;
1793 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1794 #[cfg(test)]
1795 if let Some(delay) = self.control_dispatch_delay {
1796 tokio::time::sleep(delay).await;
1797 }
1798 let result = self
1799 .handle_client_control_request(ctx, frame, request)
1800 .await;
1801 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1802 result
1803 }
1804 FrameType::Push => {
1805 let Some(endpoint) = self
1806 .forwarding
1807 .module_endpoint_for_connection(ctx.connection_id)
1808 .map_err(RouterError::Forwarding)?
1809 else {
1810 return Ok(vec![control_error_frame(
1811 &frame,
1812 "unsupported_control_frame",
1813 "client-originated channel-0 PUSH is not supported",
1814 )?]);
1815 };
1816 self.handle_status_update(endpoint, frame)
1817 }
1818 FrameType::Response | FrameType::Error
1819 if self
1820 .forwarding
1821 .module_endpoint_for_connection(ctx.connection_id)
1822 .map_err(RouterError::Forwarding)?
1823 .is_some() =>
1824 {
1825 self.handle_module_relay_response(ctx.connection_id, frame)
1826 }
1827 ty => Ok(vec![control_error_frame(
1828 &frame,
1829 "unsupported_control_frame",
1830 format!("unsupported channel-0 frame {ty:?}"),
1831 )?]),
1832 }
1833 }
1834
1835 pub fn cleanup_connection(
1836 &self,
1837 connection_id: ConnectionId,
1838 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1839 self.caller_endpoints
1840 .lock()
1841 .unwrap_or_else(|p| p.into_inner())
1842 .remove(&connection_id);
1843 self.forwarding
1844 .operator_confirms()
1845 .connection_closed(connection_id);
1846 self.cleanup_connection_with_end_reason(
1847 connection_id,
1848 RegistrationEndReason::ConnectionClosed,
1849 )
1850 }
1851
1852 fn cleanup_connection_with_end_reason(
1853 &self,
1854 connection_id: ConnectionId,
1855 requested_reason: RegistrationEndReason,
1856 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1857 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1858 self.registry
1859 .get_module_by_connection(connection_id)?
1860 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
1861 .and_then(|module| module.registration_end_reason().ok().flatten())
1862 .unwrap_or(requested_reason)
1863 } else {
1864 requested_reason
1865 };
1866 let crash_closed = self
1867 .registry
1868 .get_module_by_connection(connection_id)?
1869 .and_then(|registration| {
1870 self.forwarding
1871 .module_endpoint_for_connection(connection_id)
1872 .ok()
1873 .flatten()
1874 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1875 .map(|routes| (registration.manifest.module_id, routes))
1876 });
1877 let crash_closed = crash_closed.map(|(module_id, routes)| {
1878 let terminal = match self.supervisor.get(&module_id) {
1879 None => false,
1880 Some(module) => match module.will_recover_after_connection_loss() {
1881 Ok(will_recover) => !will_recover,
1882 Err(err) => {
1883 warn!(
1884 %module_id,
1885 error = %err,
1886 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1887 );
1888 false
1889 }
1890 },
1891 };
1892 let (reason, drained) = match self
1896 .forwarding
1897 .daemon_drain_result_for_connection(connection_id)
1898 {
1899 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
1900 Ok(None) => (RouteCloseReason::Crash, false),
1901 Err(err) => {
1902 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1903 (RouteCloseReason::Crash, false)
1904 }
1905 };
1906 (module_id, routes, reason, drained, terminal)
1907 });
1908 let registrations = self.deregister_connection(connection_id, end_reason);
1909 let cleanup = if crash_closed.is_some() {
1910 self.forwarding.cleanup_connection_counted(connection_id)
1911 } else {
1912 self.forwarding
1918 .cleanup_connection(connection_id)
1919 .map(|released| crate::forwarding::ConnectionCleanup {
1920 released,
1921 abandoned_relays: 0,
1922 })
1923 };
1924 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
1929 let abandoned = cleanup
1930 .as_ref()
1931 .map_or(0, |cleanup| cleanup.abandoned_relays);
1932 send_route_control_pushes(
1933 &self.forwarding,
1934 routes,
1935 ClientControlPush::RouteClosed {
1936 module_id,
1937 channels: Vec::new(),
1938 reason,
1939 drained,
1940 abandoned,
1941 excluded_subscriptions: 0,
1942 terminal: Some(terminal),
1943 },
1944 );
1945 }
1946 if let Ok(cleanup) = cleanup {
1947 self.emit_route_goodbyes(cleanup.released);
1948 }
1949 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1953 crate::supervise::notify_registration_release();
1954 self.capability_evaluator.wake_deadline_loop();
1955 self.refresh_capability_requirements();
1956 }
1957 self.supervisor.remove_spawn_subscribers(connection_id);
1958 self.activity_streams
1959 .lock()
1960 .unwrap_or_else(|p| p.into_inner())
1961 .retain(|(id, _), task| {
1962 if *id == connection_id {
1963 task.abort();
1964 false
1965 } else {
1966 true
1967 }
1968 });
1969 self.hello_launch_nonces
1972 .lock()
1973 .unwrap_or_else(|poisoned| poisoned.into_inner())
1974 .forget(connection_id);
1975 self.scopes
1976 .write()
1977 .unwrap_or_else(|poisoned| poisoned.into_inner())
1978 .release_connection(connection_id);
1979 registrations
1980 }
1981
1982 pub(crate) fn handle_route_goodbye(
1983 &self,
1984 connection_id: ConnectionId,
1985 route_channel: u16,
1986 route_epoch: u32,
1987 ) -> Result<bool, RouterError> {
1988 debug!(
1989 connection_id = connection_id.get(),
1990 route_channel, route_epoch, "handling route GOODBYE"
1991 );
1992 let RouteRelease::Removed(released_route) = self
1993 .forwarding
1994 .release_client_route(connection_id, route_channel, route_epoch)
1995 .map_err(RouterError::Forwarding)?
1996 else {
1997 return Ok(false);
1998 };
1999 self.emit_route_goodbyes(vec![released_route]);
2000 Ok(true)
2001 }
2002
2003 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2004 for released in released_routes {
2005 let frame = match Frame::build_with_version(
2006 released.negotiated_ver,
2007 FrameType::Goodbye,
2008 control_flags(),
2009 released.channel,
2010 released.epoch,
2011 0,
2012 Vec::new(),
2013 ) {
2014 Ok(frame) => frame,
2015 Err(err) => {
2016 warn!(
2017 route_channel = released.channel,
2018 error = %err,
2019 "failed to build route GOODBYE frame"
2020 );
2021 continue;
2022 }
2023 };
2024 if !released.close_on_delivery_failure() {
2025 crate::forwarding::send_module_route_goodbye(
2026 &self.counters,
2027 &released.sink,
2028 frame,
2029 released.module_id.as_deref(),
2030 "client route released",
2031 );
2032 continue;
2033 }
2034 if let Err(err) = released.sink.try_send(frame) {
2035 warn!(
2036 target_connection_id = released.connection_id.get(),
2037 route_channel = released.channel,
2038 error = %err,
2039 "route GOODBYE was not delivered to client; closing target connection"
2040 );
2041 if self
2042 .forwarding
2043 .escalate_client_delivery_failure(
2044 released.connection_id,
2045 released.channel,
2046 released.epoch,
2047 CloseReason::new(
2048 "route_goodbye_delivery_failed",
2049 format!(
2050 "failed to enqueue route GOODBYE for channel {}: {err}",
2051 released.channel
2052 ),
2053 ),
2054 crate::forwarding::UndeliveredFrame {
2055 module_id: released.module_id.as_deref(),
2056 sink: &released.sink,
2057 },
2058 )
2059 .unwrap_or(false)
2060 {
2061 self.counters.increment_goodbye_relay_client_failed();
2062 }
2063 }
2064 }
2065 }
2066
2067 fn send_abandoned_route_bind_goodbye(
2079 &self,
2080 module_sink: &crate::FrameSink,
2081 negotiated_ver: u8,
2082 module_channel: u16,
2083 module_epoch: u32,
2084 ) {
2085 let frame = match Frame::build_with_version(
2086 negotiated_ver,
2087 FrameType::Goodbye,
2088 control_flags(),
2089 module_channel,
2090 module_epoch,
2091 0,
2092 Vec::new(),
2093 ) {
2094 Ok(frame) => frame,
2095 Err(err) => {
2096 warn!(
2097 route_channel = module_channel,
2098 error = %err,
2099 "failed to build GOODBYE for abandoned route.bind"
2100 );
2101 return;
2102 }
2103 };
2104 crate::forwarding::send_module_route_goodbye(
2105 &self.counters,
2106 module_sink,
2107 frame,
2108 None,
2109 "abandoned route.bind",
2110 );
2111 }
2112
2113 fn handle_hello(
2114 &self,
2115 connection_id: ConnectionId,
2116 sink: Option<crate::FrameSink>,
2117 frame: Frame,
2118 ) -> Result<Vec<Frame>, RouterError> {
2119 debug!(
2120 connection_id = connection_id.get(),
2121 corr = frame.header.corr,
2122 "handling HELLO"
2123 );
2124 if self
2128 .registry
2129 .get_module_by_connection(connection_id)
2130 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2131 .is_some()
2132 {
2133 return Ok(vec![control_error_frame(
2134 &frame,
2135 "invalid_hello",
2136 "connection is already registered as a module",
2137 )?]);
2138 }
2139 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2140 Ok(value) => value,
2141 Err(err) => {
2142 return Ok(vec![control_error_frame(
2143 &frame,
2144 "invalid_hello",
2145 format!("malformed HELLO body: {err}"),
2146 )?])
2147 }
2148 };
2149 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2150 return Ok(vec![control_error_frame(
2151 &frame,
2152 "invalid_capability_grammar",
2153 err.to_string(),
2154 )?]);
2155 }
2156 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2157 return Ok(vec![control_error_frame(
2158 &frame,
2159 "invalid_manifest",
2160 err.to_string(),
2161 )?]);
2162 }
2163 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2164 return Ok(vec![control_error_frame(
2165 &frame,
2166 "invalid_event_declaration",
2167 err.to_string(),
2168 )?]);
2169 }
2170 if let Some(provenance) = hello_value
2171 .get("manifest")
2172 .and_then(|manifest| manifest.get("provenance"))
2173 {
2174 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2175 return Ok(vec![control_error_frame(
2176 &frame,
2177 "invalid_manifest",
2178 format!("malformed manifest provenance: {err}"),
2179 )?]);
2180 }
2181 }
2182 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2183 Ok(hello) => hello,
2184 Err(err) => {
2185 return Ok(vec![control_error_frame(
2186 &frame,
2187 "invalid_hello",
2188 format!("malformed HELLO body: {err}"),
2189 )?])
2190 }
2191 };
2192
2193 if hello.protocol_ver != hello.manifest.protocol_ver {
2194 return Ok(vec![control_error_frame(
2195 &frame,
2196 "invalid_manifest",
2197 format!(
2198 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2199 hello.protocol_ver, hello.manifest.protocol_ver
2200 ),
2201 )?]);
2202 }
2203
2204 if hello.manifest.module_id.trim().is_empty() {
2205 return Ok(vec![control_error_frame(
2206 &frame,
2207 "invalid_manifest",
2208 "manifest module_id must not be empty",
2209 )?]);
2210 }
2211
2212 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2213 Ok(negotiated_ver) => negotiated_ver,
2214 Err(message) => {
2215 return Ok(vec![control_error_frame(
2216 &frame,
2217 "version_unsupported",
2218 message,
2219 )?])
2220 }
2221 };
2222
2223 let swap_admission = self
2231 .supervisor
2232 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2233 if swap_admission == SwapHelloAdmission::Refused {
2234 warn!(
2235 module_id = %hello.manifest.module_id,
2236 connection_id = connection_id.get(),
2237 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2238 );
2239 return Ok(vec![control_error_frame(
2240 &frame,
2241 "swap_token_invalid",
2242 format!(
2243 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2244 hello.manifest.module_id
2245 ),
2246 )?]);
2247 }
2248 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2249
2250 if let Some(rejection) = (!swap_candidate)
2258 .then(|| {
2259 self.supervisor.reserved_hello_rejection(
2260 &hello.manifest.module_id,
2261 hello.launch_nonce.as_deref(),
2262 )
2263 })
2264 .flatten()
2265 {
2266 let message = match rejection {
2267 ReservedHelloRejection::Exact { module_id } => format!(
2268 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2269 ),
2270 ReservedHelloRejection::Prefix {
2271 prefix,
2272 owner_module_id,
2273 } => format!(
2274 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2275 hello.manifest.module_id
2276 ),
2277 };
2278 return Ok(vec![control_error_frame(
2279 &frame,
2280 "reserved_module",
2281 message,
2282 )?]);
2283 }
2284
2285 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2286 &hello.manifest.module_id,
2287 hello.manifest.capabilities.as_ref(),
2288 );
2289 if let Some(refusal) = reserved_capability_refusals.first() {
2290 let capability = refusal.capability.clone();
2291 let bound_module = refusal.claimants[0].clone();
2292 log_duplicate_claim_events(reserved_capability_refusals);
2293 return Ok(vec![control_error_frame(
2294 &frame,
2295 "reserved_capability",
2296 format!(
2297 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2298 capability, bound_module, hello.manifest.module_id
2299 ),
2300 )?]);
2301 }
2302
2303 if self
2306 .forwarding
2307 .connection_has_client_routes(connection_id)
2308 .map_err(RouterError::Forwarding)?
2309 {
2310 return Ok(vec![control_error_frame(
2311 &frame,
2312 "invalid_hello",
2313 "connection has open client routes and cannot also register as a module",
2314 )?]);
2315 }
2316
2317 self.hello_launch_nonces
2322 .lock()
2323 .unwrap_or_else(|poisoned| poisoned.into_inner())
2324 .record(connection_id, hello.launch_nonce.as_deref());
2325 let control_ops = effective_module_control_ops(hello.control_ops);
2326 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2329 if swap_candidate {
2330 return self.register_swap_candidate(
2331 connection_id,
2332 sink,
2333 &frame,
2334 hello.manifest,
2335 negotiated_ver,
2336 control_ops,
2337 hello_ack,
2338 );
2339 }
2340 let registration = match self.registry.register_unpublished_with_control_ops(
2347 hello.manifest,
2348 negotiated_ver,
2349 connection_id,
2350 control_ops,
2351 ) {
2352 Ok(registration) => registration,
2353 Err(RegistryError::DuplicateModuleId { module_id }) => {
2354 return Ok(vec![control_error_frame(
2355 &frame,
2356 "duplicate_module_id",
2357 format!(
2358 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2359 ),
2360 )?])
2361 }
2362 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2363 return Ok(vec![control_error_frame(
2364 &frame,
2365 "invalid_module_id",
2366 err.to_string(),
2367 )?])
2368 }
2369 Err(err) => {
2370 return Ok(vec![control_error_frame(
2371 &frame,
2372 "registry_error",
2373 err.to_string(),
2374 )?])
2375 }
2376 };
2377 #[cfg(test)]
2378 self.hold_between_registry_and_forwarding();
2379
2380 let reply = if let Some(sink) = sink {
2381 let concurrency = manifest_concurrency(®istration.manifest);
2397 if let Err(err) = self.forwarding.register_module_connection_acked(
2398 connection_id,
2399 registration.manifest.module_id.clone(),
2400 negotiated_ver,
2401 concurrency,
2402 sink,
2403 hello_ack,
2404 ) {
2405 if matches!(
2410 self.deregister_connection(
2411 connection_id,
2412 RegistrationEndReason::RegistrationFailed,
2413 ),
2414 Ok(r) if !r.is_empty()
2415 ) {
2416 crate::supervise::notify_registration_release();
2417 }
2418 return Ok(vec![control_error_frame(
2419 &frame,
2420 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2421 "invalid_hello"
2422 } else {
2423 forwarding_error_code(&err)
2424 },
2425 err.to_string(),
2426 )?]);
2427 }
2428 Vec::new()
2429 } else {
2430 vec![hello_ack]
2433 };
2434
2435 if !self
2440 .registry
2441 .publish_registration(connection_id)
2442 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2443 {
2444 debug!(
2445 module_id = %registration.manifest.module_id,
2446 connection_id = connection_id.get(),
2447 "registration was removed before it could be published"
2448 );
2449 }
2450
2451 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2460 info!(
2461 module_id = %registration.manifest.module_id,
2462 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2463 );
2464 }
2465
2466 self.apply_registration_capabilities(®istration);
2467
2468 self.supervisor
2469 .record_activity_registration(®istration.manifest.module_id);
2470
2471 info!(
2472 module_id = %registration.manifest.module_id,
2473 module_version = %registration.manifest.module_version,
2474 negotiated_ver,
2475 routable_provider = manifest_provides_routable_role(®istration.manifest),
2476 connection_id = connection_id.get(),
2477 "module registered"
2478 );
2479
2480 Ok(reply)
2481 }
2482
2483 #[allow(clippy::too_many_arguments)]
2500 fn register_swap_candidate(
2501 &self,
2502 connection_id: ConnectionId,
2503 sink: Option<crate::FrameSink>,
2504 frame: &Frame,
2505 manifest: ModuleManifest,
2506 negotiated_ver: u8,
2507 control_ops: Vec<String>,
2508 hello_ack: Frame,
2509 ) -> Result<Vec<Frame>, RouterError> {
2510 let module_id = manifest.module_id.clone();
2511 let registration = match self.registry.register_unpublished_candidate_with_control_ops(
2512 manifest,
2513 negotiated_ver,
2514 connection_id,
2515 control_ops,
2516 ) {
2517 Ok(registration) => registration,
2518 Err(RegistryError::DuplicateModuleId { module_id }) => {
2519 return Ok(vec![control_error_frame(
2520 frame,
2521 "duplicate_module_id",
2522 format!(
2523 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2524 ),
2525 )?])
2526 }
2527 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2528 return Ok(vec![control_error_frame(
2529 frame,
2530 "invalid_module_id",
2531 err.to_string(),
2532 )?])
2533 }
2534 Err(err) => {
2535 return Ok(vec![control_error_frame(
2536 frame,
2537 "registry_error",
2538 err.to_string(),
2539 )?])
2540 }
2541 };
2542 #[cfg(test)]
2543 self.hold_between_registry_and_forwarding();
2544 let reply = if let Some(sink) = sink {
2545 let concurrency = manifest_concurrency(®istration.manifest);
2549 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2550 connection_id,
2551 module_id.clone(),
2552 negotiated_ver,
2553 concurrency,
2554 sink,
2555 hello_ack,
2556 ) {
2557 if matches!(
2558 self.deregister_connection(
2559 connection_id,
2560 RegistrationEndReason::RegistrationFailed,
2561 ),
2562 Ok(r) if !r.is_empty()
2563 ) {
2564 crate::supervise::notify_registration_release();
2565 }
2566 return Ok(vec![control_error_frame(
2567 frame,
2568 forwarding_error_code(&err),
2569 err.to_string(),
2570 )?]);
2571 }
2572 Vec::new()
2573 } else {
2574 vec![hello_ack]
2575 };
2576 if !self
2577 .registry
2578 .publish_registration(connection_id)
2579 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2580 {
2581 debug!(
2582 module_id = %module_id,
2583 connection_id = connection_id.get(),
2584 "swap candidate registration was removed before it could be published"
2585 );
2586 }
2587 self.supervisor.mark_swap_candidate_admitted(&module_id);
2588 info!(
2589 module_id = %module_id,
2590 module_version = %registration.manifest.module_version,
2591 negotiated_ver,
2592 ready = registration.ready,
2593 connection_id = connection_id.get(),
2594 "swap candidate registered; not routable until cutover"
2595 );
2596 Ok(reply)
2597 }
2598
2599 fn build_hello_ack(
2600 &self,
2601 frame: &Frame,
2602 negotiated_ver: u8,
2603 module_id: &str,
2604 ) -> Result<Frame, RouterError> {
2605 let ack = ModuleHelloAckBody {
2606 negotiated_ver,
2607 subc_ops: module_subc_ops(),
2608 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2609 storage: self
2610 .storage_config
2611 .as_ref()
2612 .map(|cfg| cfg.descriptor_for(module_id)),
2613 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2614 };
2615 let body = serde_json::to_vec(&ack).map_err(|err| {
2616 RouterError::backend(
2617 0,
2618 frame.header.corr,
2619 format!("failed to encode HELLO_ACK: {err}"),
2620 )
2621 })?;
2622
2623 Frame::build_with_version(
2624 negotiated_ver,
2625 FrameType::HelloAck,
2626 control_flags(),
2627 0,
2628 0,
2629 frame.header.corr,
2630 body,
2631 )
2632 .map_err(RouterError::FrameBuild)
2633 }
2634
2635 async fn handle_client_control_request(
2636 &self,
2637 ctx: &RouteCtx,
2638 frame: Frame,
2639 request: ClientControlRequest,
2640 ) -> Result<Vec<Frame>, RouterError> {
2641 if let Some(operation) = privileged_operation(&request) {
2642 let caller = self.resolve_caller(ctx.connection_id).await;
2643 let op = client_control_request_op(&request);
2644 let text = prompt_text(operation, &caller);
2645 match self.privileged_operator_prompt {
2646 PrivilegedOperatorPrompt::Log => {
2647 info!(target: "subc_daemon::privileged_prompt", operation = op,
2648 target = ?operation, caller = %caller.description(), summary = %text,
2649 "privileged request would prompt the operator");
2650 }
2651 PrivilegedOperatorPrompt::Enforce => {
2652 let key =
2653 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2654 let outcome = self
2655 .forwarding
2656 .operator_confirms()
2657 .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2658 .await;
2659 if outcome.code != "confirmed" {
2660 return Ok(vec![outcome.frame(&frame)]);
2661 }
2662 }
2663 }
2664 }
2665 match request {
2666 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2667 ClientControlRequest::CatalogList { module_id } => {
2668 self.handle_catalog_list(frame, module_id)
2669 }
2670 ClientControlRequest::RouteOpen {
2671 target,
2672 identity,
2673 consumer_identity,
2674 consumer_capabilities,
2675 role_versions,
2676 admission_facts,
2677 scope,
2678 } => {
2679 self.handle_route_open(
2680 ctx,
2681 frame,
2682 RouteOpenRequest {
2683 target,
2684 identity,
2685 consumer_identity,
2686 consumer_capabilities,
2687 role_versions,
2688 admission_facts,
2689 scope,
2690 },
2691 )
2692 .await
2693 }
2694 ClientControlRequest::RoutePoll {
2695 route_channel,
2696 route_epoch,
2697 kind,
2698 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2699 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2700 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2701 self.handle_supervisor_spawn_snapshot(frame)
2702 }
2703 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2704 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2705 }
2706 ClientControlRequest::SupervisorRestart {
2707 module_id,
2708 drain_timeout_ms,
2709 } => {
2710 self.log_supervisor_request_received(
2711 ctx,
2712 frame.header.corr,
2713 ops::SUPERVISOR_RESTART,
2714 Some(&module_id),
2715 None,
2716 )?;
2717 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2718 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2719 .await
2720 }
2721 ClientControlRequest::SupervisorSwap {
2722 module_id,
2723 ready_timeout_ms,
2724 } => {
2725 self.log_supervisor_request_received(
2726 ctx,
2727 frame.header.corr,
2728 ops::SUPERVISOR_SWAP,
2729 Some(&module_id),
2730 None,
2731 )?;
2732 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2733 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2734 .await
2735 }
2736 ClientControlRequest::SupervisorReload { module_id } => {
2737 self.log_supervisor_request_received(
2738 ctx,
2739 frame.header.corr,
2740 ops::SUPERVISOR_RELOAD,
2741 Some(&module_id),
2742 None,
2743 )?;
2744 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2745 self.handle_supervisor_reload(frame, module_id, requester)
2746 .await
2747 }
2748 ClientControlRequest::SupervisorRescan { preview } => {
2749 if !preview {
2750 self.log_supervisor_request_received(
2751 ctx,
2752 frame.header.corr,
2753 ops::SUPERVISOR_RESCAN,
2754 None,
2755 None,
2756 )?;
2757 }
2758 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2759 self.handle_supervisor_rescan(frame, preview, requester)
2760 .await
2761 }
2762 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2763 self.log_supervisor_request_received(
2764 ctx,
2765 frame.header.corr,
2766 ops::SUPERVISOR_RELEASE_RESERVED,
2767 Some(&module_id),
2768 None,
2769 )?;
2770 self.handle_supervisor_release_reserved(frame, module_id)
2771 .await
2772 }
2773 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2774 self.log_supervisor_request_received(
2775 ctx,
2776 frame.header.corr,
2777 ops::SUPERVISOR_SET_ENABLED,
2778 Some(&module_id),
2779 Some(enabled),
2780 )?;
2781 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2782 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2783 .await
2784 }
2785 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2786 self.handle_supervisor_health_probe(frame, module_id).await
2787 }
2788 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2789 ClientControlRequest::SupervisorRoutes { module_id } => {
2790 self.handle_supervisor_routes(frame, module_id)
2791 }
2792 ClientControlRequest::SupervisorProvenance { module_id } => {
2793 self.handle_supervisor_provenance(frame, module_id).await
2794 }
2795 ClientControlRequest::SupervisorStderrTail {
2796 module_id,
2797 max_lines,
2798 max_bytes,
2799 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2800 ClientControlRequest::SupervisorTerminals { module_id } => {
2801 self.handle_supervisor_terminals(frame, module_id).await
2802 }
2803 }
2804 }
2805
2806 fn log_supervisor_request_received(
2807 &self,
2808 ctx: &RouteCtx,
2809 corr: u64,
2810 op: &'static str,
2811 module_id: Option<&str>,
2812 enabled: Option<bool>,
2813 ) -> Result<(), RouterError> {
2814 let caller = self.supervisor_requester(ctx, corr)?;
2815 let caller = principal_label(&caller);
2816 info!(
2819 target: "control",
2820 op,
2821 module_id,
2822 enabled,
2823 connection_id = ctx.connection_id.get(),
2824 caller = %caller,
2825 "supervisor request received"
2826 );
2827 Ok(())
2828 }
2829
2830 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
2831 Ok(self
2832 .registry
2833 .get_module_by_connection(ctx.connection_id)
2834 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2835 .map(|registration| Principal::Reserved {
2836 module_id: registration.manifest.module_id,
2837 })
2838 .unwrap_or(Principal::Direct))
2839 }
2840
2841 fn handle_module_control_request(
2842 &self,
2843 connection_id: ConnectionId,
2844 frame: Frame,
2845 request: ModuleControlRequestFromModule,
2846 ) -> Result<Vec<Frame>, RouterError> {
2847 match request {
2848 ModuleControlRequestFromModule::CatalogUpdate {
2849 provides,
2850 capabilities,
2851 ready,
2852 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2853 ModuleControlRequestFromModule::LiveRoots {} => {
2854 let registered = self
2855 .registry
2856 .get_module_by_connection(connection_id)
2857 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2858 let Some(registration) = registered else {
2859 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2860 };
2861 let response = self
2862 .forwarding
2863 .live_roots(®istration.manifest.module_id)
2864 .map_err(RouterError::Forwarding)?;
2865 Ok(vec![control_response_body_frame(
2866 &frame,
2867 &response,
2868 "ModuleControlResponseToModule::LiveRoots",
2869 )?])
2870 }
2871 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2872 self.handle_scope_sync(connection_id, frame, generation, scopes)
2873 }
2874 ModuleControlRequestFromModule::ScopeApply {
2875 generation,
2876 upsert,
2877 end,
2878 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
2879 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2880 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2881 }
2882 }
2883 }
2884
2885 fn handle_operator_confirm(
2886 &self,
2887 ctx: &RouteCtx,
2888 frame: Frame,
2889 ) -> Result<Vec<Frame>, RouterError> {
2890 use crate::operator_confirm::{audit, Outcome};
2891 let registration = self
2892 .registry
2893 .get_module_by_connection(ctx.connection_id)
2894 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2895 let Some(registration) = registration else {
2896 let outcome = Outcome::refusal("not_registered");
2897 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2898 return Ok(vec![outcome.frame(&frame)]);
2899 };
2900 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
2901 Ok(request) => request,
2902 Err(_) => {
2903 let outcome = Outcome::refusal("invalid_control_body");
2904 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2905 return Ok(vec![outcome.frame(&frame)]);
2906 }
2907 };
2908 let module_id = registration.manifest.module_id;
2909 let nonce = self
2913 .hello_launch_nonces
2914 .lock()
2915 .unwrap_or_else(|p| p.into_inner())
2916 .nonce(ctx.connection_id)
2917 .map(str::to_owned);
2918 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
2919 self.supervisor
2920 .spawned_consumer_authorized(&module_id, nonce)
2921 });
2922 let confirms = self.forwarding.operator_confirms();
2923 self.forwarding
2924 .with_operator_route(
2925 ctx.connection_id,
2926 request.route_channel,
2927 request.route_epoch,
2928 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
2929 )
2930 .map_err(RouterError::Forwarding)
2931 }
2932
2933 fn handle_scope_sync(
2937 &self,
2938 connection_id: ConnectionId,
2939 frame: Frame,
2940 generation: u64,
2941 scopes: Vec<ScopeRecord>,
2942 ) -> Result<Vec<Frame>, RouterError> {
2943 self.handle_scope_change(connection_id, frame, generation, scopes, None)
2944 }
2945
2946 fn handle_scope_change(
2947 &self,
2948 connection_id: ConnectionId,
2949 frame: Frame,
2950 generation: u64,
2951 scopes: Vec<ScopeRecord>,
2952 end: Option<Vec<ScopeEnd>>,
2953 ) -> Result<Vec<Frame>, RouterError> {
2954 let is_apply = end.is_some();
2955 let Some(registration) = self
2956 .registry
2957 .get_module_by_connection(connection_id)
2958 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2959 else {
2960 return Ok(vec![control_error_frame(
2961 &frame,
2962 "not_registered",
2963 "scope.sync and scope.apply require an active module registration owned by this connection",
2964 )?]);
2965 };
2966 let owner = registration.manifest.module_id;
2967 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2968 let is_current_launch = |connection: ConnectionId| {
2969 self.hello_launch_nonces
2970 .lock()
2971 .unwrap_or_else(|poisoned| poisoned.into_inner())
2972 .presented(connection, current_nonce.as_deref())
2973 };
2974 let mut table = self
2979 .scopes
2980 .write()
2981 .unwrap_or_else(|poisoned| poisoned.into_inner());
2982 let outcome = match end {
2983 Some(end) => table.apply(
2984 &owner,
2985 connection_id,
2986 is_current_launch,
2987 generation,
2988 scopes,
2989 end,
2990 ),
2991 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
2992 };
2993 let drained = match &outcome {
2994 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
2995 Err(_) => Vec::new(),
2996 };
2997 let has_deadlines = outcome.is_ok() && table.has_deadlines();
2998 drop(table);
2999 if has_deadlines {
3000 self.scope_deadline_added.notify_one();
3001 }
3002 match outcome {
3003 Ok(applied) => {
3004 let counts = ScopeOutcomeCounts::of(&applied.results);
3005 info!(
3006 owner = %owner,
3007 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3008 generation,
3009 records = applied.results.len(),
3010 created = counts.created,
3011 replaced = counts.replaced,
3012 updated = counts.updated,
3013 unchanged = counts.unchanged,
3014 refused = counts.refused,
3015 ended = applied.ended.len(),
3016 tag_changes = applied.tag_changes.len(),
3017 routes_closed = drained.len(),
3018 "scope change accepted"
3019 );
3020 for refused in applied
3026 .results
3027 .iter()
3028 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3029 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3030 {
3031 warn!(
3032 owner = %owner,
3033 generation,
3034 scope_ref = %refused.scope_ref,
3035 scope_epoch = refused.scope_epoch,
3036 code = refused.code.as_deref().unwrap_or(""),
3037 "scope record refused"
3038 );
3039 }
3040 self.close_scope_drained_routes(drained);
3041 let response = if is_apply {
3042 ModuleControlResponseToModule::ScopeApply {
3043 generation,
3044 results: applied.results,
3045 end_results: applied.end_results,
3046 ended: applied.ended,
3047 }
3048 } else {
3049 ModuleControlResponseToModule::ScopeSync {
3050 generation,
3051 results: applied.results,
3052 ended: applied.ended,
3053 }
3054 };
3055 Ok(vec![control_response_body_frame(
3056 &frame,
3057 &response,
3058 "scope change response",
3059 )?])
3060 }
3061 Err(refusal) => {
3062 info!(
3063 owner = %owner,
3064 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3065 generation,
3066 code = refusal.code,
3067 "scope change refused"
3068 );
3069 Ok(vec![control_error_frame(
3070 &frame,
3071 refusal.code,
3072 refusal.message,
3073 )?])
3074 }
3075 }
3076 }
3077
3078 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3084 if drained.is_empty() {
3085 return;
3086 }
3087 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3088 BTreeMap::new();
3089 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3090 for route in drained {
3091 warn!(
3092 module_id = %route.module_id,
3093 reason = ?route.reason,
3094 client_connection_id = route.client.connection_id.get(),
3095 route_channel = route.client.channel,
3096 "closing route because its scope changed"
3097 );
3098 pushes
3099 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3100 .or_insert_with(|| (route.reason, Vec::new()))
3101 .1
3102 .push(EndpointRoute {
3103 goodbye_target: route.client.clone(),
3104 principal: Principal::Unverified,
3105 bound_at: Instant::now(),
3106 draining: false,
3107 drain_reason: None,
3108 });
3109 goodbyes.push(route.module);
3110 goodbyes.push(route.client);
3111 }
3112 for ((module_id, _), (reason, routes)) in pushes {
3113 send_route_control_pushes(
3114 &self.forwarding,
3115 routes,
3116 ClientControlPush::RouteClosed {
3117 module_id,
3118 channels: Vec::new(),
3119 reason,
3120 drained: false,
3121 abandoned: 0,
3122 excluded_subscriptions: 0,
3123 terminal: Some(false),
3124 },
3125 );
3126 }
3127 self.emit_route_goodbyes(goodbyes);
3128 }
3129
3130 fn handle_scope_describe(
3133 &self,
3134 connection_id: ConnectionId,
3135 frame: Frame,
3136 owner: Principal,
3137 scope_ref: String,
3138 ) -> Result<Vec<Frame>, RouterError> {
3139 let registered = self
3140 .registry
3141 .get_module_by_connection(connection_id)
3142 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3143 if registered.is_none() {
3144 return Ok(vec![control_error_frame(
3145 &frame,
3146 "not_registered",
3147 "scope.describe requires an active module registration owned by this connection",
3148 )?]);
3149 }
3150 let description = self
3151 .scopes
3152 .read()
3153 .unwrap_or_else(|poisoned| poisoned.into_inner())
3154 .describe(&owner, &scope_ref);
3155 let owner_configured = match &owner {
3156 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3161 _ => false,
3162 };
3163 let response = ModuleControlResponseToModule::ScopeDescribe {
3164 status: description.status,
3165 scope_epoch: description.scope_epoch,
3166 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3167 owner_synced: description.owner_synced,
3168 owner_configured,
3169 scope: description.stamp,
3170 };
3171 Ok(vec![control_response_body_frame(
3172 &frame,
3173 &response,
3174 "ModuleControlResponseToModule::ScopeDescribe",
3175 )?])
3176 }
3177
3178 fn handle_catalog_update(
3179 &self,
3180 connection_id: ConnectionId,
3181 frame: Frame,
3182 provides: Vec<ProviderRole>,
3183 capabilities: Option<CapabilityDeclarations>,
3184 ready: Option<bool>,
3185 ) -> Result<Vec<Frame>, RouterError> {
3186 self.refresh_capability_requirements();
3187 let Some(registration) = self
3188 .registry
3189 .get_module_by_connection(connection_id)
3190 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3191 else {
3192 return Ok(vec![control_error_frame(
3193 &frame,
3194 "not_registered",
3195 "catalog.update requires an active module registration owned by this connection",
3196 )?]);
3197 };
3198
3199 if let Some(message) =
3200 catalog_update_frozen_field_message(®istration.manifest, &provides)
3201 {
3202 return Ok(vec![control_error_frame(
3203 &frame,
3204 "catalog_update_frozen_field",
3205 message,
3206 )?]);
3207 }
3208
3209 let mut candidate = registration.manifest.clone();
3210 candidate.provides = provides.clone();
3211 candidate.capabilities = capabilities
3212 .clone()
3213 .or_else(|| registration.manifest.capabilities.clone());
3214 if let Err(err) = candidate.validate_capability_grammar() {
3215 return Ok(vec![control_error_frame(
3216 &frame,
3217 "invalid_capability_grammar",
3218 err.to_string(),
3219 )?]);
3220 }
3221
3222 let mut conflicts = self
3225 .capability_evaluator
3226 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3227 if let Some(conflict) = conflicts.first() {
3228 let message = format!(
3229 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3230 conflict.capability, conflict.claimants[0], candidate.module_id
3231 );
3232 for conflict in &mut conflicts {
3233 conflict.source = DuplicateClaimSource::CatalogUpdate;
3234 }
3235 log_duplicate_claim_events(conflicts);
3236 return Ok(vec![control_error_frame(
3237 &frame,
3238 "reserved_capability",
3239 message,
3240 )?]);
3241 }
3242
3243 let updated = self
3244 .registry
3245 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3246 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3247 if updated.is_none() {
3248 return Ok(vec![control_error_frame(
3249 &frame,
3250 "not_registered",
3251 "catalog.update requires an active module registration owned by this connection",
3252 )?]);
3253 }
3254 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3255 log_duplicate_claim_events(
3256 self.capability_evaluator
3257 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3258 );
3259 }
3260 if capability_census_trigger(
3261 registration.manifest.capabilities.as_ref(),
3262 updated
3263 .as_ref()
3264 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3265 ) {
3266 self.enforce_capability_denies();
3267 }
3268 self.refresh_capability_requirements();
3269
3270 let response = ModuleControlResponseToModule::CatalogUpdate {};
3271 control_response_body_frame(
3272 &frame,
3273 &response,
3274 "ModuleControlResponseToModule::CatalogUpdate",
3275 )
3276 .map(|frame| vec![frame])
3277 }
3278
3279 pub(crate) fn with_activity_ledger_config(
3280 mut self,
3281 run_dir: &Path,
3282 incarnation: String,
3283 config: crate::activity::ActivityConfig,
3284 ) -> Self {
3285 self.activity = Some(crate::activity::ActivityLedger::open_config(
3286 run_dir.join("activity.jsonl"),
3287 incarnation,
3288 config,
3289 ));
3290 self.supervisor.set_activity(self.activity.clone());
3291 self
3292 }
3293
3294 pub(crate) async fn configure_activity(
3295 &self,
3296 config: crate::activity::ActivityConfig,
3297 ) -> Result<(), &'static str> {
3298 match &self.activity {
3299 Some(ledger) => ledger.configure(config).await,
3300 None => Ok(()),
3301 }
3302 }
3303
3304 fn cancel_activity_subscription(
3305 &self,
3306 ctx: &RouteCtx,
3307 frame: &Frame,
3308 ) -> Result<bool, RouterError> {
3309 let task = self
3310 .activity_streams
3311 .lock()
3312 .unwrap_or_else(|p| p.into_inner())
3313 .remove(&(ctx.connection_id, frame.header.corr));
3314 let Some(task) = task else {
3315 return Ok(false);
3316 };
3317 task.abort();
3318 let end = Frame::build_with_version(
3319 response_version(frame),
3320 FrameType::StreamEnd,
3321 control_flags(),
3322 0,
3323 0,
3324 frame.header.corr,
3325 Vec::new(),
3326 )
3327 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3328 let sink = ctx.egress.clone();
3329 tokio::spawn(async move {
3330 let _ = sink.send(end).await;
3331 });
3332 Ok(true)
3333 }
3334
3335 async fn handle_activity_read(
3336 &self,
3337 ctx: &RouteCtx,
3338 frame: Frame,
3339 ) -> Result<Vec<Frame>, RouterError> {
3340 use subc_control::{
3341 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3342 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3343 };
3344 #[derive(Deserialize)]
3345 struct Identity {
3346 #[serde(default)]
3347 consumer_identity: Option<ConsumerIdentity>,
3348 }
3349 let principal = match self
3353 .forwarding
3354 .module_id_for_connection(ctx.connection_id)
3355 .map_err(RouterError::Forwarding)?
3356 {
3357 Some(module_id) => Ok(Principal::Reserved { module_id }),
3358 None => {
3359 let identity: Identity = match serde_json::from_slice(&frame.body) {
3360 Ok(identity) => identity,
3361 Err(_) => {
3362 return Ok(vec![control_error_frame(
3363 &frame,
3364 ACTIVITY_MALFORMED,
3365 "malformed activity read",
3366 )?])
3367 }
3368 };
3369 self.route_open_principal(&frame, identity.consumer_identity)?
3370 }
3371 };
3372 let admitted = matches!(principal, Ok(Principal::Direct))
3373 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3374 if !admitted {
3375 return Ok(vec![control_error_frame(
3376 &frame,
3377 ACTIVITY_READ_FORBIDDEN,
3378 "activity reads require an operator or trusted mirror principal",
3379 )?]);
3380 }
3381 let Some(ledger) = &self.activity else {
3382 return Ok(vec![control_error_frame(
3383 &frame,
3384 ACTIVITY_UNAVAILABLE,
3385 "activity ledger unavailable",
3386 )?]);
3387 };
3388 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3389 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3390 {
3391 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3392 Ok(request) => ledger.query(request).await,
3393 Err(_) => Err(ACTIVITY_MALFORMED),
3394 };
3395 return match result {
3396 Ok(reply) => Ok(vec![control_response_body_frame(
3397 &frame,
3398 &reply,
3399 ops::ACTIVITY_QUERY,
3400 )?]),
3401 Err(code) => Ok(vec![control_error_frame(
3402 &frame,
3403 code,
3404 "activity query refused",
3405 )?]),
3406 };
3407 }
3408 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3409 Ok(request) => request,
3410 Err(_) => {
3411 return Ok(vec![control_error_frame(
3412 &frame,
3413 ACTIVITY_MALFORMED,
3414 "malformed activity subscription",
3415 )?])
3416 }
3417 };
3418 let mut subscription = match ledger.subscribe(request.since).await {
3419 Ok(subscription) => subscription,
3420 Err(code) => {
3421 return Ok(vec![control_error_frame(
3422 &frame,
3423 code,
3424 "activity subscription refused",
3425 )?])
3426 }
3427 };
3428 let key = (ctx.connection_id, frame.header.corr);
3429 let sink = ctx.egress.clone();
3430 let mut streams = self
3431 .activity_streams
3432 .lock()
3433 .unwrap_or_else(|p| p.into_inner());
3434 streams.retain(|_, task| !task.is_finished());
3435 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3436 return Ok(vec![control_error_frame(
3437 &frame,
3438 "duplicate_subscription",
3439 "activity subscription correlation id already in use",
3440 )?]);
3441 }
3442 let task = tokio::spawn(async move {
3443 loop {
3444 let entry = if let Some(entry) = subscription.replay.pop_front() {
3445 Some(entry)
3446 } else {
3447 subscription.receiver.recv().await
3448 };
3449 let Some(entry) = entry else {
3450 break;
3451 };
3452 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3453 return;
3454 };
3455 let Ok(next) = Frame::build_with_version(
3456 response_version(&frame),
3457 FrameType::StreamData,
3458 control_flags(),
3459 0,
3460 0,
3461 frame.header.corr,
3462 body,
3463 ) else {
3464 return;
3465 };
3466 if sink.send(next).await.is_err() {
3467 return;
3468 }
3469 }
3470 if let Some(code) = subscription.terminal_code() {
3473 if let Ok(end) =
3474 control_error_frame(&frame, code, "activity subscriber fell behind")
3475 {
3476 let _ = sink.send(end).await;
3477 }
3478 }
3479 });
3480 streams.insert(key, task.abort_handle());
3481 Ok(Vec::new())
3482 }
3483
3484 async fn handle_activity_record(
3485 &self,
3486 ctx: &RouteCtx,
3487 frame: Frame,
3488 ) -> Result<Vec<Frame>, RouterError> {
3489 use subc_control::{
3490 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3491 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3492 };
3493 let result = async {
3494 let module = self
3495 .forwarding
3496 .module_id_for_connection(ctx.connection_id)
3497 .ok()
3498 .flatten()
3499 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3500 let request: ActivityRecordRequest =
3501 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3502 crate::activity::validate(&module, &request)?;
3503 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3504 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3505 let at_ms = std::time::SystemTime::now()
3506 .duration_since(std::time::UNIX_EPOCH)
3507 .unwrap_or_default()
3508 .as_millis() as u64;
3509 ledger.record(module, request, actor, at_ms).await
3510 }
3511 .await;
3512 match result {
3513 Ok(entry) => Ok(vec![control_response_body_frame(
3514 &frame,
3515 &ActivityRecordResponse { seq: entry.seq },
3516 "activity.record",
3517 )?]),
3518 Err(code) => Ok(vec![control_error_frame(
3519 &frame,
3520 code,
3521 "activity recording refused",
3522 )?]),
3523 }
3524 }
3525
3526 fn activity_actor(
3527 &self,
3528 connection: ConnectionId,
3529 module: &str,
3530 on: Option<&subc_control::ActivityOn>,
3531 ) -> Result<subc_control::ActivityActor, &'static str> {
3532 use crate::forwarding::{DataRoute, DataRouteState};
3533 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3534 let principal = |id: &str| Principal::Reserved {
3535 module_id: id.to_owned(),
3536 };
3537 let mut actor = ActivityActor {
3538 principal: principal(module),
3539 scope_owner: None,
3540 agent_id: None,
3541 run_id: None,
3542 flow_id: None,
3543 project_root: None,
3544 };
3545 let scope = match on {
3546 None => return Ok(actor),
3547 Some(ActivityOn::Scope(scope)) => {
3548 let selector = subc_protocol::scope::ScopeSelector::new(
3549 principal(module),
3550 scope.scope_ref.clone(),
3551 Some(scope.epoch),
3552 );
3553 self.scopes
3554 .read()
3555 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3556 .attribute(&principal(module), module, &selector, true)
3557 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3558 }
3559 Some(ActivityOn::Route(route)) => {
3560 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3561 .forwarding
3562 .lookup_data_route(connection, route.channel, route.epoch)
3563 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3564 else {
3565 return Err(ACTIVITY_ACTOR_UNKNOWN);
3566 };
3567 if binding.module_id != module {
3568 return Err(ACTIVITY_ACTOR_UNKNOWN);
3569 }
3570 actor.principal = binding.principal.clone();
3571 actor.project_root = binding
3572 .project_root
3573 .as_ref()
3574 .map(|root| root.as_path().to_path_buf());
3575 let Some(bound) = &binding.scope else {
3576 return Ok(actor);
3577 };
3578 let owner = principal(&bound.owner);
3579 let selector = subc_protocol::scope::ScopeSelector::new(
3580 owner.clone(),
3581 bound.scope_ref.clone(),
3582 Some(bound.tag.scope_epoch),
3583 );
3584 let admitted = self
3585 .scopes
3586 .read()
3587 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3588 .attribute(&owner, module, &selector, true)
3589 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3590 if admitted.tag != bound.tag {
3591 return Err(ACTIVITY_ACTOR_UNKNOWN);
3592 }
3593 admitted
3594 }
3595 };
3596 actor.scope_owner = Some(scope.stamp.owner);
3597 actor.agent_id = scope.stamp.attributes.agent_id;
3598 actor.run_id = scope.stamp.attributes.run_id;
3599 actor.flow_id = scope.stamp.attributes.flow_id;
3600 Ok(actor)
3601 }
3602
3603 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3604 self.refresh_capability_requirements();
3605 let mut counters = self.counters.snapshot();
3611 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3612 obj.insert(
3613 "activity_unparseable_lines".into(),
3614 ledger.read_errors().into(),
3615 );
3616 obj.insert(
3617 "activity_lifecycle_append_failures".into(),
3618 ledger.lifecycle_failures().into(),
3619 );
3620 obj.insert(
3621 "activity_started_digest_skips".into(),
3622 ledger.lifecycle_digest_skips().into(),
3623 );
3624 }
3625 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3626 self.forwarding.client_route_concentration(),
3627 counters.as_object_mut(),
3628 ) {
3629 obj.insert(
3630 "client_connections_with_routes".into(),
3631 connections_with_routes.into(),
3632 );
3633 obj.insert("max_routes_on_one_connection".into(), max.into());
3634 }
3635 if let (Some(open_breakers), Some(obj)) = (
3641 self.route_bind_breakers.open_snapshot(),
3642 counters.as_object_mut(),
3643 ) {
3644 obj.insert("route_bind_breakers_open".into(), open_breakers);
3645 }
3646 let response = ClientControlResponse::ServerDescribe {
3647 protocol_ver: PROTOCOL_VERSION,
3648 subc_ops: subc_ops(),
3649 capabilities: self.subc_capabilities.as_ref().to_vec(),
3650 connected_clients: self.connected_clients.count(),
3651 counters: Some(counters),
3652 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3653 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3654 capability_requirements: self.capability_requirement_statuses(),
3655 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3656 };
3657 Ok(vec![control_response_body_frame(
3658 &frame,
3659 &response,
3660 "ClientControlResponse::ServerDescribe",
3661 )?])
3662 }
3663
3664 fn handle_catalog_list(
3665 &self,
3666 frame: Frame,
3667 module_id: Option<String>,
3668 ) -> Result<Vec<Frame>, RouterError> {
3669 let (generation, modules) = self.registry.list_modules().map_err(|err| {
3670 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3671 })?;
3672 let entries = modules
3673 .into_iter()
3674 .filter(|registration| {
3675 module_id
3676 .as_deref()
3677 .map(|wanted| registration.manifest.module_id == wanted)
3678 .unwrap_or(true)
3679 })
3680 .map(|registration| {
3681 let not_ready = self.not_ready_reason(®istration);
3682 let roles = registration.manifest.provides;
3683 CatalogEntry::new(
3684 registration.manifest.module_id,
3685 roles,
3686 registration.control_ops,
3687 )
3688 .with_ready(not_ready.is_none())
3689 .with_not_ready(not_ready)
3690 .with_module_version(Some(registration.manifest.module_version))
3691 .with_capabilities(registration.manifest.capabilities)
3692 .with_self_signals(registration.manifest.self_signals)
3693 .with_events(registration.manifest.events)
3694 })
3695 .collect();
3696 let response = ClientControlResponse::CatalogList {
3697 generation,
3698 modules: entries,
3699 subc_ops: subc_ops(),
3700 };
3701 Ok(vec![control_response_body_frame(
3702 &frame,
3703 &response,
3704 "ClientControlResponse::CatalogList",
3705 )?])
3706 }
3707
3708 fn route_open_principal(
3709 &self,
3710 frame: &Frame,
3711 consumer_identity: Option<ConsumerIdentity>,
3712 ) -> Result<Result<Principal, Frame>, RouterError> {
3713 let Some(consumer_identity) = consumer_identity else {
3714 return Ok(Ok(Principal::Direct));
3715 };
3716
3717 if self.supervisor.spawned_consumer_authorized(
3718 &consumer_identity.module_id,
3719 &consumer_identity.launch_nonce,
3720 ) {
3721 return Ok(Ok(Principal::Reserved {
3722 module_id: consumer_identity.module_id,
3723 }));
3724 }
3725
3726 Ok(Err(control_error_frame(
3727 frame,
3728 "bad_consumer_identity",
3729 format!(
3730 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3731 consumer_identity.module_id
3732 ),
3733 )?))
3734 }
3735
3736 fn route_open_refusal_frame(
3743 &self,
3744 ctx: &RouteCtx,
3745 frame: &Frame,
3746 module_id: &str,
3747 reason: &'static str,
3748 code: &'static str,
3749 message: impl Into<String>,
3750 ) -> Result<Frame, RouterError> {
3751 self.observe_route_open_refusal(ctx, module_id, reason, code);
3752 control_error_frame(frame, code, message.into())
3753 }
3754
3755 fn route_open_breaker_refusal_frame(
3773 &self,
3774 ctx: &RouteCtx,
3775 frame: &Frame,
3776 module_id: &str,
3777 consecutive_timeouts: u32,
3778 retry_in: Duration,
3779 probe_in_flight: bool,
3780 ) -> Result<Frame, RouterError> {
3781 self.counters
3782 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
3783 debug!(
3784 target: "control",
3785 code = "module_timeout",
3786 module_id = ?module_id,
3787 connection_id = ctx.connection_id.get(),
3788 consecutive_timeouts,
3789 retry_in_ms = retry_in.as_millis() as u64,
3790 probe_in_flight,
3791 "route.open refused by open bind-relay breaker"
3792 );
3793 let detail = if probe_in_flight {
3800 "one test route is already being tried; retry once it settles".to_string()
3801 } else {
3802 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
3803 };
3804 control_error_frame(
3805 frame,
3806 "module_timeout",
3807 format!(
3808 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
3809 timed out in a row); {detail}; its established routes are unaffected"
3810 ),
3811 )
3812 }
3813
3814 fn observe_route_open_refusal(
3827 &self,
3828 ctx: &RouteCtx,
3829 module_id: &str,
3830 reason: &'static str,
3831 code: &'static str,
3832 ) {
3833 self.counters.increment_route_open_refused(code);
3834 info!(
3835 target: "control",
3836 code,
3837 reason,
3838 module_id = ?module_id,
3839 connection_id = ctx.connection_id.get(),
3840 "route.open refused"
3841 );
3842 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
3843 self.route_outages.record_not_serving(module_id, reason);
3844 }
3845 }
3846
3847 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
3892 self.route_outages.record_accepted(module_id);
3893 self.counters.increment_route_open_accepted(principal);
3894 info!(
3895 target: "control",
3896 principal,
3897 module_id,
3898 connection_id = ctx.connection_id.get(),
3899 "route.open accepted"
3900 );
3901 }
3902
3903 fn supervised_absent_route_open_refusal_frame(
3904 &self,
3905 ctx: &RouteCtx,
3906 frame: &Frame,
3907 module_id: &str,
3908 code: &'static str,
3909 status: &crate::supervise::ModuleStatus,
3910 ) -> Result<Frame, RouterError> {
3911 self.counters.increment_route_open_refused(code);
3912 info!(
3913 target: "control",
3914 code,
3915 reason = "supervised_not_registered",
3916 module_id = ?module_id,
3917 connection_id = ctx.connection_id.get(),
3918 state = %status.state,
3919 enabled = status.enabled,
3920 live = status.live,
3921 "route.open refused"
3922 );
3923 self.route_outages
3927 .record_not_serving(module_id, "supervised_not_registered");
3928 let message = if let Some(pid) = status.waiting_for_previous_pid {
3929 format!("waiting: previous instance (pid {pid}) is still running but hidden; it cannot be identified")
3930 } else {
3931 format!(
3932 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
3933 status.state, status.enabled, status.live
3934 )
3935 };
3936 control_error_frame(frame, code, message)
3937 }
3938
3939 async fn handle_route_open(
3940 &self,
3941 ctx: &RouteCtx,
3942 frame: Frame,
3943 request: RouteOpenRequest,
3944 ) -> Result<Vec<Frame>, RouterError> {
3945 let RouteOpenRequest {
3946 target,
3947 mut identity,
3948 consumer_identity,
3949 consumer_capabilities,
3950 role_versions,
3951 admission_facts,
3952 scope,
3953 } = request;
3954 let target_module_id = target_module_id(&target).to_string();
3955 if self
3956 .registry
3957 .get_module_by_connection(ctx.connection_id)
3958 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3959 .is_some()
3960 {
3961 return Ok(vec![control_error_frame(
3962 &frame,
3963 "invalid_request",
3964 "module connections cannot open client routes",
3965 )?]);
3966 }
3967 debug!(
3968 connection_id = ctx.connection_id.get(),
3969 corr = frame.header.corr,
3970 module_id = %target_module_id,
3971 "handling route.open"
3972 );
3973
3974 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
3980 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
3981 self.observe_route_open_refusal(
3982 ctx,
3983 &target_module_id,
3984 "invalid_role_versions",
3985 error_codes::INVALID_REQUEST,
3986 );
3987 return Ok(vec![control_error_body_frame(
3988 &frame,
3989 ErrorBody {
3990 code: error_codes::INVALID_REQUEST.to_string(),
3991 message: error.to_string(),
3992 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
3993 },
3994 )?]);
3995 }
3996
3997 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
3998 if status.waiting_for_previous_pid.is_some() {
3999 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4000 ctx,
4001 &frame,
4002 &target_module_id,
4003 "module_warming",
4004 &status,
4005 )?]);
4006 }
4007 }
4008
4009 let Some(registration) = self
4028 .registry
4029 .get_module(&target_module_id)
4030 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4031 else {
4032 if let Some((status, warming)) =
4033 self.supervisor_status(&target_module_id, frame.header.corr)?
4034 {
4035 if status.protocol == ModuleProtocol::None {
4042 return Ok(vec![self.route_open_refusal_frame(
4043 ctx,
4044 &frame,
4045 &target_module_id,
4046 "protocol_none",
4047 error_codes::MODULE_NO_PROTOCOL,
4048 format!(
4049 "module_id '{target_module_id}' is declared protocol: none; \
4050 it speaks no subc wire and serves no routes"
4051 ),
4052 )?]);
4053 }
4054 let code = if warming {
4055 "module_warming"
4056 } else {
4057 "target_unavailable"
4058 };
4059 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4060 ctx,
4061 &frame,
4062 &target_module_id,
4063 code,
4064 &status,
4065 )?]);
4066 }
4067 if let Some(removed_ago_ms) =
4068 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4069 {
4070 return Ok(vec![self.route_open_refusal_frame(
4071 ctx,
4072 &frame,
4073 &target_module_id,
4074 "removed",
4075 error_codes::MODULE_REMOVED,
4076 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4077 )?]);
4078 }
4079 return Ok(vec![self.route_open_refusal_frame(
4080 ctx,
4081 &frame,
4082 &target_module_id,
4083 "not_registered",
4084 error_codes::UNKNOWN_MODULE,
4085 format!("module_id '{target_module_id}' is not registered"),
4086 )?]);
4087 };
4088
4089 if !registration.ready {
4093 self.counters
4094 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4095 info!(
4096 target: "control",
4097 code = error_codes::MODULE_WARMING,
4098 module_id = ?target_module_id,
4099 connection_id = ctx.connection_id.get(),
4100 reason = "declared_not_ready",
4101 "route.open refused"
4102 );
4103 self.route_outages
4106 .record_not_serving(&target_module_id, "declared_not_ready");
4107 return Ok(vec![control_error_body_frame(
4108 &frame,
4109 ErrorBody {
4110 code: error_codes::MODULE_WARMING.to_string(),
4111 message: format!(
4112 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4113 ),
4114 detail: Some(serde_json::json!({
4115 "reason": "declared_not_ready"
4116 })),
4117 },
4118 )?]);
4119 }
4120
4121 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4140 self.counters
4141 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4142 info!(
4143 target: "control",
4144 code = error_codes::MODULE_WARMING,
4145 module_id = ?target_module_id,
4146 connection_id = ctx.connection_id.get(),
4147 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4148 capability = %capability,
4149 "route.open refused"
4150 );
4151 return Ok(vec![control_error_body_frame(
4152 &frame,
4153 ErrorBody {
4154 code: error_codes::MODULE_WARMING.to_string(),
4155 message: format!(
4156 "module_id '{target_module_id}' requires capability '{capability}', \
4157 which no registered module provides; retry"
4158 ),
4159 detail: Some(serde_json::json!({
4160 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4161 "capability": capability,
4162 })),
4163 },
4164 )?]);
4165 }
4166
4167 if !target_has_required_role(&target, ®istration.manifest.provides) {
4168 return Ok(vec![self.route_open_refusal_frame(
4169 ctx,
4170 &frame,
4171 &target_module_id,
4172 "role_not_provided",
4173 "target_unavailable",
4174 format!("module_id '{target_module_id}' does not provide the requested target"),
4175 )?]);
4176 }
4177
4178 if registration.state != ChannelState::Active {
4179 return Ok(vec![self.route_open_refusal_frame(
4180 ctx,
4181 &frame,
4182 &target_module_id,
4183 "registration_not_active",
4184 "target_unavailable",
4185 format!("module_id '{target_module_id}' is not active"),
4186 )?]);
4187 }
4188
4189 if self
4190 .forwarding
4191 .module_is_draining(&target_module_id)
4192 .map_err(RouterError::Forwarding)?
4193 {
4194 return Ok(vec![self.route_open_refusal_frame(
4195 ctx,
4196 &frame,
4197 &target_module_id,
4198 "reloading",
4199 "module_reloading",
4200 format!("module_id '{target_module_id}' is reloading"),
4201 )?]);
4202 }
4203
4204 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4205 process_liveness.process_live(&target_module_id) == Some(false)
4206 }) {
4207 if process_liveness.process_replacing(&target_module_id) {
4214 return Ok(vec![self.route_open_refusal_frame(
4215 ctx,
4216 &frame,
4217 &target_module_id,
4218 "reloading",
4219 "module_reloading",
4220 format!("module_id '{target_module_id}' is reloading"),
4221 )?]);
4222 }
4223 return Ok(vec![self.route_open_refusal_frame(
4224 ctx,
4225 &frame,
4226 &target_module_id,
4227 "supervisor_not_live",
4228 "target_unavailable",
4229 format!("module_id '{target_module_id}' is not live"),
4230 )?]);
4231 }
4232
4233 if !self
4234 .forwarding
4235 .has_live_module_connection(&target_module_id)
4236 .map_err(RouterError::Forwarding)?
4237 {
4238 return Ok(vec![self.route_open_refusal_frame(
4239 ctx,
4240 &frame,
4241 &target_module_id,
4242 "no_forwarding_connection",
4243 "target_unavailable",
4244 format!("module_id '{target_module_id}' has no live forwarding connection"),
4245 )?]);
4246 }
4247
4248 if let Some(error) =
4249 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4250 {
4251 self.observe_route_open_refusal(
4252 ctx,
4253 &target_module_id,
4254 "op_not_allowed",
4255 "op_not_allowed",
4256 );
4257 return Ok(vec![error]);
4258 }
4259
4260 let principal = match self.route_open_principal(&frame, consumer_identity)? {
4261 Ok(principal) => principal,
4262 Err(error) => {
4263 self.observe_route_open_refusal(
4264 ctx,
4265 &target_module_id,
4266 "bad_consumer_identity",
4267 "bad_consumer_identity",
4268 );
4269 return Ok(vec![error]);
4270 }
4271 };
4272
4273 if let Principal::Reserved {
4277 module_id: opening_module_id,
4278 } = &principal
4279 {
4280 if let Some(opening_registration) = self
4281 .registry
4282 .get_module(opening_module_id)
4283 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4284 {
4285 if let Some(capability) =
4286 denied_capability(&opening_registration.manifest, ®istration.manifest)
4287 {
4288 warn!(
4289 opening_module_id,
4290 target_module_id,
4291 capability,
4292 "refusing route.open because an attested capability deny edge matches"
4293 );
4294 return Ok(vec![self.route_open_refusal_frame(
4295 ctx,
4296 &frame,
4297 &target_module_id,
4298 "capability_deny_edge",
4299 "capability_forbidden",
4300 format!(
4301 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4302 ),
4303 )?]);
4304 }
4305 }
4306 }
4307
4308 if admission_facts.is_some() {
4309 let carrier_matches = matches!(
4310 &principal,
4311 Principal::Reserved { module_id }
4312 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4313 );
4314 if !carrier_matches {
4315 return Ok(vec![self.route_open_refusal_frame(
4316 ctx,
4317 &frame,
4318 &target_module_id,
4319 "admission_facts_carrier_not_permitted",
4320 "admission_facts_not_permitted",
4321 "admission facts may only be carried by the configured reserved module",
4322 )?]);
4323 }
4324
4325 let target_allowed = self
4326 .admission_facts_targets
4327 .as_ref()
4328 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4329 if !target_allowed {
4330 return Ok(vec![self.route_open_refusal_frame(
4331 ctx,
4332 &frame,
4333 &target_module_id,
4334 "admission_facts_target_not_listed",
4335 "admission_facts_target_not_allowed",
4336 format!(
4337 "admission facts are not permitted for target module_id '{target_module_id}'"
4338 ),
4339 )?]);
4340 }
4341
4342 }
4346
4347 let (bound_scope, mut scope_stamp) = match &scope {
4353 None => (None, None),
4354 Some(selector) => {
4355 let owner_configured = match &selector.owner {
4356 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4362 _ => false,
4363 };
4364 let mut scopes = self
4365 .scopes
4366 .write()
4367 .unwrap_or_else(|poisoned| poisoned.into_inner());
4368 let admitted = scopes
4369 .admit(&principal, &target_module_id, selector, owner_configured)
4370 .and_then(|admission| {
4371 crate::scopes::check_target_flow_support(
4372 &admission.stamp,
4373 &target_module_id,
4374 registration.manifest.capabilities.as_ref(),
4375 )?;
4376 crate::scopes::check_target_agent_run_support(
4377 &admission.stamp,
4378 &target_module_id,
4379 registration.manifest.capabilities.as_ref(),
4380 )?;
4381 let bound = scopes.reserve_bind(
4382 &admission,
4383 ctx.connection_id,
4384 selector.hold_presence,
4385 );
4386 Ok((admission, bound))
4387 });
4388 match admitted {
4389 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4390 Err(refusal) => {
4391 return Ok(vec![self.route_open_refusal_frame(
4392 ctx,
4393 &frame,
4394 &target_module_id,
4395 refusal.code,
4396 refusal.code,
4397 refusal.message,
4398 )?]);
4399 }
4400 }
4401 }
4402 };
4403
4404 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4426 Ok(project_root) => project_root,
4427 Err(err) => {
4428 return Ok(vec![control_error_frame(
4429 &frame,
4430 "invalid_project_root",
4431 err.to_string(),
4432 )?])
4433 }
4434 };
4435 identity.project_root = project_root.as_path().to_path_buf();
4436
4437 let _concurrency_guard = match self
4448 .route_bind_concurrency
4449 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4450 {
4451 Ok(guard) => guard,
4452 Err(in_flight) => {
4453 return Ok(vec![self.route_open_target_capacity_refusal(
4454 ctx,
4455 &frame,
4456 &target_module_id,
4457 in_flight,
4458 )?]);
4459 }
4460 };
4461
4462 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4465 RouteBindAdmission::Admitted { guard, probe } => {
4466 if probe {
4467 info!(
4468 module_id = %target_module_id,
4469 connection_id = ctx.connection_id.get(),
4470 "route.bind breaker half-open: admitting one probe"
4471 );
4472 }
4473 guard
4474 }
4475 RouteBindAdmission::Refused {
4476 consecutive_timeouts,
4477 retry_in,
4478 probe_in_flight,
4479 } => {
4480 return Ok(vec![self.route_open_breaker_refusal_frame(
4481 ctx,
4482 &frame,
4483 &target_module_id,
4484 consecutive_timeouts,
4485 retry_in,
4486 probe_in_flight,
4487 )?]);
4488 }
4489 };
4490
4491 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4496 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4497 let pending = match self
4498 .forwarding
4499 .begin_route_bind_relay_for(
4500 ctx.connection_id,
4501 ctx.egress.clone(),
4502 response_version(&frame),
4503 frame.header.corr,
4504 &target_module_id,
4505 principal.clone(),
4506 bound_scope,
4507 Some(project_root),
4508 relay_deadline,
4509 )
4510 .await
4511 {
4512 Ok(pending) => pending,
4513 Err(err) => {
4514 return Ok(vec![self.route_open_refusal_frame(
4515 ctx,
4516 &frame,
4517 &target_module_id,
4518 "relay_reservation_failed",
4519 forwarding_error_code(&err),
4520 err.to_string(),
4521 )?])
4522 }
4523 };
4524 let crate::forwarding::PendingRouteBindRelay {
4525 endpoint,
4526 module_sink,
4527 negotiated_ver,
4528 client_channel,
4529 client_epoch,
4530 module_channel,
4531 module_epoch,
4532 corr: relay_corr,
4533 receiver,
4534 } = pending;
4535 let mut reservation =
4536 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4537
4538 let relay_registration = self
4543 .registry
4544 .get_module_by_connection(endpoint.connection_id)
4545 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4546 let relay_capabilities = relay_registration
4547 .as_ref()
4548 .and_then(|registration| registration.manifest.capabilities.as_ref());
4549 if let Some(selector) = &scope {
4550 if let Err(refusal) = self
4553 .scopes
4554 .read()
4555 .unwrap_or_else(|poisoned| poisoned.into_inner())
4556 .check_bind_proof(selector)
4557 {
4558 reservation.release_and_disarm();
4559 return Ok(vec![self.route_open_refusal_frame(
4560 ctx,
4561 &frame,
4562 &target_module_id,
4563 refusal.code,
4564 refusal.code,
4565 refusal.message,
4566 )?]);
4567 }
4568 }
4569 if let Some(stamp) = scope_stamp
4570 .as_ref()
4571 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4572 {
4573 if let Err(refusal) = crate::scopes::check_target_flow_support(
4574 stamp,
4575 &target_module_id,
4576 relay_capabilities,
4577 )
4578 .and_then(|()| {
4579 crate::scopes::check_target_agent_run_support(
4580 stamp,
4581 &target_module_id,
4582 relay_capabilities,
4583 )
4584 }) {
4585 reservation.release_and_disarm();
4586 return Ok(vec![self.route_open_refusal_frame(
4587 ctx,
4588 &frame,
4589 &target_module_id,
4590 refusal.code,
4591 refusal.code,
4592 refusal.message,
4593 )?]);
4594 }
4595 }
4596
4597 let call_mode = if relay_capabilities
4600 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4601 {
4602 Some(derive_call_mode(scope_stamp.as_ref()))
4603 } else {
4604 if let Some(stamp) = scope_stamp.as_mut() {
4605 stamp.expires_at_ms = None;
4606 }
4607 None
4608 };
4609
4610 debug!(
4611 connection_id = ctx.connection_id.get(),
4612 client_channel,
4613 client_epoch,
4614 module_channel,
4615 module_epoch,
4616 "reserved route handle pair"
4617 );
4618 let principal_label = principal_label(&principal);
4621 let relay = ModuleControlRequest::RouteBind {
4622 route_channel: module_channel,
4623 epoch: module_epoch,
4624 target,
4625 identity,
4626 principal: Some(principal),
4627 consumer_capabilities,
4628 role_versions,
4629 admission_facts,
4630 scope: scope_stamp,
4631 call_mode,
4632 };
4633 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4634 RouterError::backend(
4635 0,
4636 frame.header.corr,
4637 format!("failed to encode route.bind request: {err}"),
4638 )
4639 })?;
4640 let relay_frame = Frame::build_with_version(
4641 negotiated_ver,
4642 FrameType::Request,
4643 control_flags(),
4644 0,
4645 0,
4646 relay_corr,
4647 relay_body,
4648 )
4649 .map_err(RouterError::FrameBuild)?;
4650
4651 if let Err(err) = module_sink.send(relay_frame).await {
4652 reservation.release_and_disarm();
4653 return Ok(vec![self.route_open_refusal_frame(
4654 ctx,
4655 &frame,
4656 &target_module_id,
4657 "relay_send_failed",
4658 "target_unavailable",
4659 err.to_string(),
4660 )?]);
4661 }
4662
4663 #[cfg(test)]
4664 if let Some(hold) = &self.route_bind_relay_sent_hold {
4665 hold.wait().await;
4669 hold.wait().await;
4670 }
4671
4672 if self
4677 .forwarding
4678 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4679 .map_err(RouterError::Forwarding)?
4680 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4681 {
4682 self.send_abandoned_route_bind_goodbye(
4683 &module_sink,
4684 negotiated_ver,
4685 module_channel,
4686 module_epoch,
4687 );
4688 }
4689
4690 match timeout_at(relay_deadline, receiver).await {
4691 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4692 reservation.disarm();
4693 if breaker.record_accepted() {
4694 info!(
4695 module_id = %target_module_id,
4696 "route.bind breaker closed: the probe was accepted"
4697 );
4698 }
4699 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
4700 Ok(Vec::new())
4701 }
4702 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4703 reservation.release_and_disarm();
4704 breaker.record_inconclusive();
4708 let scope_code = match body.code.as_str() {
4712 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4713 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4714 _ => None,
4715 };
4716 if let Some(code) = scope_code {
4717 self.observe_route_open_refusal(
4718 ctx,
4719 &target_module_id,
4720 "scope_changed_before_commit",
4721 code,
4722 );
4723 return Ok(vec![control_error_body_frame(&frame, body)?]);
4724 }
4725 self.counters
4726 .increment_route_open_refused("module_rejected");
4727 info!(
4728 target: "control",
4729 code = "module_rejected",
4730 module_code = ?body.code,
4731 module_id = ?target_module_id,
4732 connection_id = ctx.connection_id.get(),
4733 "route.open refused"
4734 );
4735 Ok(vec![control_error_body_frame(&frame, body)?])
4736 }
4737 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
4738 reservation.release_and_disarm();
4739 breaker.record_inconclusive();
4740 tracing::warn!(
4747 module_id = %target_module_id,
4748 "route.bind relay abandoned: {message}"
4749 );
4750 Ok(vec![self.route_open_refusal_frame(
4751 ctx,
4752 &frame,
4753 &target_module_id,
4754 "relay_abandoned",
4755 "target_unavailable",
4756 message,
4757 )?])
4758 }
4759 Ok(Err(_)) => {
4760 reservation.release_and_disarm();
4761 breaker.record_inconclusive();
4762 Ok(vec![self.route_open_refusal_frame(
4763 ctx,
4764 &frame,
4765 &target_module_id,
4766 "relay_waiter_canceled",
4767 "target_unavailable",
4768 "route.bind relay waiter was canceled before the module responded",
4769 )?])
4770 }
4771 Err(_) => {
4772 reservation.release_and_disarm();
4773 if let Some(opened) = breaker.record_timeout(
4777 self.route_bind_breaker_threshold,
4778 self.route_bind_breaker_cooldown,
4779 ) {
4780 warn!(
4781 module_id = %target_module_id,
4782 consecutive_timeouts = opened.consecutive_timeouts,
4783 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
4784 reopened_after_probe = opened.reopened_after_probe,
4785 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
4786 );
4787 }
4788 tracing::warn!(
4796 module_id = %target_module_id,
4797 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
4798 "route.bind relay timed out: module did not ack within budget"
4799 );
4800 Ok(vec![self.route_open_refusal_frame(
4801 ctx,
4802 &frame,
4803 &target_module_id,
4804 "relay_timed_out",
4805 "module_timeout",
4806 format!(
4807 "module_id '{target_module_id}' did not answer route.bind within {:?}",
4808 route_bind_relay_timeout
4809 ),
4810 )?])
4811 }
4812 }
4813 }
4814
4815 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4816 let response = ClientControlResponse::SupervisorSpawnSnapshot {
4817 snapshot: self.supervisor.spawn_snapshot(),
4818 };
4819 Ok(vec![control_response_body_frame(
4820 &frame,
4821 &response,
4822 "ClientControlResponse::SupervisorSpawnSnapshot",
4823 )?])
4824 }
4825
4826 fn handle_supervisor_spawn_subscribe(
4827 &self,
4828 ctx: &RouteCtx,
4829 frame: Frame,
4830 since: Option<SpawnCursor>,
4831 ) -> Result<Vec<Frame>, RouterError> {
4832 match self.supervisor.subscribe_spawns(
4833 ctx.connection_id,
4834 frame.header.corr,
4835 response_version(&frame),
4836 since,
4837 ctx.egress.clone(),
4838 ) {
4839 Ok(()) => Ok(Vec::new()),
4840 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
4841 Ok(vec![control_error_body_frame(
4842 &frame,
4843 ErrorBody {
4844 code: "spawn_cursor_incarnation_mismatch".to_string(),
4845 message: "spawn cursor belongs to a different daemon incarnation"
4846 .to_string(),
4847 detail: Some(serde_json::json!({
4848 "current_daemon_incarnation": current
4849 })),
4850 },
4851 )?])
4852 }
4853 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
4854 &frame,
4855 ErrorBody {
4856 code: "spawn_cursor_too_old".to_string(),
4857 message: "spawn cursor predates the retained event ring".to_string(),
4858 detail: Some(serde_json::json!({
4859 "oldest_retained_cursor": oldest
4860 })),
4861 },
4862 )?]),
4863 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
4864 0,
4865 frame.header.corr,
4866 format!("failed to open supervisor spawn subscription: {error}"),
4867 )),
4868 }
4869 }
4870
4871 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4872 let generation = self
4873 .registry
4874 .generation()
4875 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4876 let mut modules = Vec::new();
4877 for module in self.supervisor.list() {
4878 let status = module.status_for_control("list").map_err(|err| {
4879 RouterError::backend(
4880 0,
4881 frame.header.corr,
4882 format!("failed to read supervisor status: {err}"),
4883 )
4884 })?;
4885 let (configured, _) = module.configuration().map_err(|err| {
4886 RouterError::backend(
4887 0,
4888 frame.header.corr,
4889 format!("failed to read module configuration: {err}"),
4890 )
4891 })?;
4892 let image = module.running_image_agreement().await;
4894 let resources = Some(module.child_resource_usage());
4897 let pending_reload = Some(reload_verdict(
4898 &configured.program,
4899 status.spawned_from.as_deref(),
4900 image,
4901 ));
4902 modules.push(
4905 SupervisorEntry::new(
4906 status.module_id,
4907 status.state.to_string(),
4908 status.enabled,
4909 status.live,
4910 status.health.status,
4911 )
4912 .with_launch_nonce_env(Some(!cfg!(unix)))
4913 .with_launch_nonce_source(status.launch_nonce_source)
4914 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
4915 .with_protocol(status.protocol)
4916 .with_pending_reload(pending_reload)
4917 .with_last_probe_ms(status.health.last_probe_ms)
4918 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
4919 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
4920 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
4921 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
4922 .with_restart_count(Some(status.restart_count))
4923 .with_max_restarts(Some(status.max_restarts))
4924 .with_lifetime_restarts(Some(status.lifetime_restarts))
4925 .with_spawn_generation(Some(status.spawn_generation))
4926 .with_restart_window_secs(Some(status.restart_window.as_secs()))
4927 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
4928 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
4929 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
4930 .with_resources(resources),
4931 );
4932 }
4933 let response = ClientControlResponse::SupervisorList {
4934 generation,
4935 modules,
4936 };
4937 Ok(vec![control_response_body_frame(
4938 &frame,
4939 &response,
4940 "ClientControlResponse::SupervisorList",
4941 )?])
4942 }
4943
4944 fn handle_supervisor_stderr_tail(
4945 &self,
4946 frame: Frame,
4947 module_id: String,
4948 max_lines: Option<u32>,
4949 max_bytes: Option<u32>,
4950 ) -> Result<Vec<Frame>, RouterError> {
4951 let Some(module) = self.supervisor.get(&module_id) else {
4952 return Ok(vec![control_error_frame(
4953 &frame,
4954 "unknown_module",
4955 format!("module_id '{module_id}' is not supervised"),
4956 )?]);
4957 };
4958
4959 let snapshot = module.stderr_tail(
4960 max_lines.map(|value| value as usize),
4961 max_bytes.map(|value| value as usize),
4962 );
4963
4964 let response = ClientControlResponse::SupervisorStderrTail {
4965 module_id,
4966 tail: StderrTail {
4967 capture: match snapshot.capture {
4968 CaptureState::Captured => StderrCaptureState::Captured,
4969 CaptureState::Incomplete { reason } => {
4970 StderrCaptureState::Incomplete { reason }
4971 }
4972 CaptureState::NotCaptured { reason } => {
4973 StderrCaptureState::NotCaptured { reason }
4974 }
4975 },
4976 entries: snapshot
4977 .entries
4978 .into_iter()
4979 .map(|entry| match entry {
4980 TailEntry::Line {
4981 text,
4982 truncated,
4983 at_ms,
4984 } => StderrTailEntry::Line {
4985 text,
4986 truncated,
4987 at_ms,
4988 },
4989 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
4990 })
4991 .collect(),
4992 dropped_lines: snapshot.dropped_lines,
4993 },
4994 };
4995 Ok(vec![control_response_body_frame(
4996 &frame,
4997 &response,
4998 "ClientControlResponse::SupervisorStderrTail",
4999 )?])
5000 }
5001
5002 async fn handle_supervisor_terminals(
5003 &self,
5004 frame: Frame,
5005 module_id: String,
5006 ) -> Result<Vec<Frame>, RouterError> {
5007 let Some(module) = self.supervisor.get(&module_id) else {
5008 return Ok(vec![control_error_frame(
5009 &frame,
5010 "unknown_module",
5011 format!("module_id '{module_id}' is not supervised"),
5012 )?]);
5013 };
5014
5015 let terminals = module
5018 .read_durable_terminal_history()
5019 .await
5020 .map_err(|error| {
5021 RouterError::backend(
5022 0,
5023 frame.header.corr,
5024 format!("failed to read terminal history: {error}"),
5025 )
5026 })?;
5027 let response = ClientControlResponse::SupervisorTerminals {
5028 module_id,
5029 terminals,
5030 };
5031 Ok(vec![control_response_body_frame(
5032 &frame,
5033 &response,
5034 "ClientControlResponse::SupervisorTerminals",
5035 )?])
5036 }
5037
5038 fn handle_supervisor_routes(
5039 &self,
5040 frame: Frame,
5041 module_id: Option<String>,
5042 ) -> Result<Vec<Frame>, RouterError> {
5043 let modules = self
5044 .forwarding
5045 .route_census(module_id.as_deref())
5046 .map_err(RouterError::Forwarding)?
5047 .into_iter()
5048 .map(|(module_id, routes)| SupervisorRouteModule {
5049 module_id,
5050 routes: routes
5051 .into_iter()
5052 .map(|route| SupervisorRoute {
5053 consumer: match route.principal {
5054 Principal::Reserved { module_id } => {
5055 SupervisorRouteConsumer::Reserved { module_id }
5056 }
5057 Principal::Direct | Principal::Unverified => {
5058 SupervisorRouteConsumer::Direct {
5059 connection_id: route.goodbye_target.connection_id.get(),
5060 }
5061 }
5062 },
5063 age_ms: Instant::now()
5064 .saturating_duration_since(route.bound_at)
5065 .as_millis()
5066 .try_into()
5067 .unwrap_or(u64::MAX),
5068 draining: route.draining,
5069 drain_reason: route.drain_reason,
5070 })
5071 .collect(),
5072 })
5073 .collect();
5074 let response = ClientControlResponse::SupervisorRoutes { modules };
5075 Ok(vec![control_response_body_frame(
5076 &frame,
5077 &response,
5078 "ClientControlResponse::SupervisorRoutes",
5079 )?])
5080 }
5081
5082 async fn handle_supervisor_provenance(
5083 &self,
5084 frame: Frame,
5085 module_id: Option<String>,
5086 ) -> Result<Vec<Frame>, RouterError> {
5087 let mut selected = if let Some(module_id) = module_id {
5088 let Some(module) = self.supervisor.get(&module_id) else {
5089 return Ok(vec![control_error_frame(
5090 &frame,
5091 "unknown_module",
5092 format!("module_id '{module_id}' is not supervised"),
5093 )?]);
5094 };
5095 vec![module]
5096 } else {
5097 self.supervisor.list()
5098 };
5099
5100 let mut modules = Vec::with_capacity(selected.len());
5101 for module in selected.drain(..) {
5102 let (status, observed_image) = module
5103 .status_and_running_image_agreement()
5104 .await
5105 .map_err(|err| {
5106 RouterError::backend(
5107 0,
5108 frame.header.corr,
5109 format!("failed to read supervisor status: {err}"),
5110 )
5111 })?;
5112 let module_declared = self
5113 .registry
5114 .get_module(&status.module_id)
5115 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5116 .and_then(|registration| registration.manifest.provenance)
5117 .map(|build| ModuleDeclaredProvenance::Reported { build })
5118 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5119 #[cfg(test)]
5120 let running_image = match &self.provenance_probe_override {
5121 Some(result) => result.clone(),
5122 None => observed_image,
5123 };
5124 #[cfg(not(test))]
5125 let running_image = observed_image;
5126 modules.push(SupervisorModuleProvenance {
5127 module_id: status.module_id,
5128 module_declared,
5129 daemon_observed: SupervisorObservedProcess {
5130 launch_nonce_source: status.launch_nonce_source,
5131 pid: status.pid,
5132 spawned_at_ms: status.spawned_at_ms,
5133 spawned_from: status.spawned_from,
5134 running_image,
5135 },
5136 });
5137 }
5138 let daemon = SupervisorDaemonProvenance {
5139 daemon_build: self.daemon_provenance.build.clone(),
5140 daemon_observed: DaemonObservedProcess {
5141 pid: self.daemon_provenance.pid,
5142 started_at_ms: self
5143 .daemon_provenance
5144 .start_clock
5145 .map(|clock| clock.started_at_ms())
5146 .or(self.daemon_provenance.started_at_ms),
5147 running_image: self
5148 .daemon_provenance
5149 .probe
5150 .observe(
5151 self.daemon_provenance.pid,
5152 self.daemon_provenance.executable_path.as_deref(),
5153 self.daemon_provenance.executable_identity.clone(),
5154 self.daemon_provenance.process_start_time,
5155 )
5156 .await,
5157 },
5158 };
5159 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5160 Ok(vec![control_response_body_frame(
5161 &frame,
5162 &response,
5163 "ClientControlResponse::SupervisorProvenance",
5164 )?])
5165 }
5166
5167 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5168 self.refresh_capability_requirements();
5169 let generation = self
5170 .registry
5171 .generation()
5172 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5173 let modules = self
5174 .supervisor
5175 .list()
5176 .into_iter()
5177 .map(|module| {
5178 let status = module.status_for_control("health").map_err(|err| {
5179 RouterError::backend(
5180 0,
5181 frame.header.corr,
5182 format!("failed to read supervisor health: {err}"),
5183 )
5184 })?;
5185 let module_id = status.module_id;
5186 let capability_detail = self
5187 .capability_evaluator
5188 .required_problem_detail(&module_id);
5189 Ok(SupervisorHealthEntry {
5190 module_id,
5191 status: status.health.status,
5192 detail: append_capability_problem_detail(
5193 status.health.detail,
5194 capability_detail,
5195 ),
5196 metrics: status.health.metrics,
5197 consecutive_failures: status.health.consecutive_failures,
5198 late_answer_count: status.health.late_answer_count,
5199 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5200 last_action: status.health.last_action,
5201 last_action_ms: status.health.last_action_ms,
5202 last_probe_ms: status.health.last_probe_ms,
5203 })
5204 })
5205 .collect::<Result<Vec<_>, RouterError>>()?;
5206 let response = ClientControlResponse::SupervisorHealth {
5207 generation,
5208 modules,
5209 };
5210 Ok(vec![control_response_body_frame(
5211 &frame,
5212 &response,
5213 "ClientControlResponse::SupervisorHealth",
5214 )?])
5215 }
5216
5217 async fn handle_supervisor_restart(
5218 &self,
5219 frame: Frame,
5220 module_id: String,
5221 drain_timeout_ms: Option<u64>,
5222 requester: Principal,
5223 ) -> Result<Vec<Frame>, RouterError> {
5224 let operation_lock = self.supervisor.operation_lock();
5225 let _operation_guard = operation_lock.lock().await;
5226 let Some(module) = self.supervisor.get(&module_id) else {
5227 return Ok(vec![control_error_frame(
5228 &frame,
5229 "unknown_module",
5230 format!("module_id '{module_id}' is not supervised"),
5231 )?]);
5232 };
5233
5234 self.route_outages.mark_operator_action(&module_id);
5235 if let Err(err) = module
5236 .restart_requested(drain_timeout_ms, Some(requester))
5237 .await
5238 {
5239 self.route_outages
5240 .operator_action_ended_unrefused(&module_id);
5241 let (code, message) = match err {
5242 crate::supervise::SuperviseError::Disabled { .. } => {
5243 ("module_disabled", err.to_string())
5244 }
5245 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5246 ("swap_in_progress", err.to_string())
5247 }
5248 _ => (
5249 "target_unavailable",
5250 format!("failed to restart module_id '{module_id}': {err}"),
5251 ),
5252 };
5253 return Ok(vec![control_error_frame(&frame, code, message)?]);
5254 }
5255
5256 let response = ClientControlResponse::SupervisorAck {
5257 module_id,
5258 applied: true,
5259 };
5260 Ok(vec![control_response_body_frame(
5261 &frame,
5262 &response,
5263 "ClientControlResponse::SupervisorAck",
5264 )?])
5265 }
5266
5267 async fn handle_supervisor_swap(
5271 &self,
5272 frame: Frame,
5273 module_id: String,
5274 ready_timeout_ms: Option<u64>,
5275 requester: Principal,
5276 ) -> Result<Vec<Frame>, RouterError> {
5277 let module = {
5284 let operation_lock = self.supervisor.operation_lock();
5285 let _operation_guard = operation_lock.lock().await;
5286 self.supervisor.get(&module_id)
5287 };
5288 let Some(module) = module else {
5289 return Ok(vec![control_error_frame(
5290 &frame,
5291 "unknown_module",
5292 format!("module_id '{module_id}' is not supervised"),
5293 )?]);
5294 };
5295
5296 self.route_outages.mark_operator_action(&module_id);
5297 if let Err(err) = module
5298 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5299 .await
5300 {
5301 self.route_outages
5302 .operator_action_ended_unrefused(&module_id);
5303 use crate::supervise::SuperviseError;
5304 let message = err.to_string();
5305 let error = match err {
5306 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5307 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5308 code: "swap_refused".to_string(),
5309 message,
5310 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5311 },
5312 SuperviseError::SwapFailed {
5313 arm,
5314 candidate_exit,
5315 ..
5316 } => ErrorBody {
5317 code: "swap_failed".to_string(),
5318 message,
5319 detail: Some(serde_json::json!({
5320 "arm": arm.as_str(),
5321 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5322 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5323 })),
5324 },
5325 _ => ErrorBody::new(
5326 "target_unavailable",
5327 format!("failed to swap module_id '{module_id}': {message}"),
5328 ),
5329 };
5330 return Ok(vec![control_error_body_frame(&frame, error)?]);
5331 }
5332 self.route_outages
5336 .operator_action_ended_unrefused(&module_id);
5337
5338 let response = ClientControlResponse::SupervisorAck {
5339 module_id,
5340 applied: true,
5341 };
5342 Ok(vec![control_response_body_frame(
5343 &frame,
5344 &response,
5345 "ClientControlResponse::SupervisorAck",
5346 )?])
5347 }
5348
5349 async fn handle_supervisor_reload(
5350 &self,
5351 frame: Frame,
5352 module_id: String,
5353 requester: Principal,
5354 ) -> Result<Vec<Frame>, RouterError> {
5355 let operation_lock = self.supervisor.operation_lock();
5356 let _operation_guard = operation_lock.lock().await;
5357 let Some(module) = self.supervisor.get(&module_id) else {
5358 return Ok(vec![control_error_frame(
5359 &frame,
5360 "unknown_module",
5361 format!("module_id '{module_id}' is not supervised"),
5362 )?]);
5363 };
5364
5365 self.route_outages.mark_operator_action(&module_id);
5366 if let Err(err) = module.reload_requested(Some(requester)).await {
5367 self.route_outages
5368 .operator_action_ended_unrefused(&module_id);
5369 let (code, message) = match err {
5370 crate::supervise::SuperviseError::Disabled { .. } => {
5371 ("module_disabled", err.to_string())
5372 }
5373 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5374 ("swap_in_progress", err.to_string())
5375 }
5376 _ => (
5377 "reload_failed",
5378 format!("failed to reload module_id '{module_id}': {err}"),
5379 ),
5380 };
5381 return Ok(vec![control_error_frame(&frame, code, message)?]);
5382 }
5383
5384 let response = ClientControlResponse::SupervisorAck {
5385 module_id,
5386 applied: true,
5387 };
5388 Ok(vec![control_response_body_frame(
5389 &frame,
5390 &response,
5391 "ClientControlResponse::SupervisorAck",
5392 )?])
5393 }
5394
5395 async fn handle_supervisor_rescan(
5396 &self,
5397 frame: Frame,
5398 preview: bool,
5399 requester: Principal,
5400 ) -> Result<Vec<Frame>, RouterError> {
5401 let Some(context) = self.rescan.clone() else {
5402 return Ok(vec![control_error_frame(
5403 &frame,
5404 "rescan_unavailable",
5405 "the daemon was not started with a reloadable config path".to_string(),
5406 )?]);
5407 };
5408
5409 let operation_lock = self.supervisor.operation_lock();
5410 let _operation_guard = operation_lock.lock().await;
5411 let loaded = match crate::daemon_config::load_with_activity(&context.config_path) {
5412 Ok(config) => config,
5413 Err(err) => {
5414 return Ok(vec![control_error_frame(
5415 &frame,
5416 "invalid_daemon_config",
5417 format!("supervisor rescan rejected daemon config: {err}"),
5418 )?])
5419 }
5420 };
5421 let Some((config, activity_config)) = loaded else {
5429 return Ok(vec![control_error_frame(
5430 &frame,
5431 "invalid_daemon_config",
5432 format!(
5433 "daemon config not found at {}; refusing to rescan (an absent config would \
5434 retire every supervised module)",
5435 context.config_path.display()
5436 ),
5437 )?]);
5438 };
5439 let (
5440 configured_port,
5441 storage_config,
5442 admission_facts_carrier_module_id,
5443 admission_facts_targets,
5444 scope_authority_owners,
5445 privileged_operator_prompt,
5446 modules,
5447 reserved_capabilities,
5448 ) = (
5449 config.port,
5450 config.storage,
5451 config.admission_facts_carrier_module_id,
5452 config.admission_facts_targets,
5453 config.scope_authority_owners,
5454 config.privileged_operator_prompt,
5455 config.modules,
5456 config.reserved_capabilities,
5457 );
5458
5459 let mut restart_required = Vec::new();
5468 for section in RestartRequiredSection::ALL {
5469 let changed = match section {
5470 RestartRequiredSection::Port => configured_port != context.configured_port,
5471 RestartRequiredSection::Storage => storage_config != context.storage_config,
5472 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5473 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5474 }
5475 RestartRequiredSection::AdmissionFactsTargets => {
5476 admission_facts_targets != context.admission_facts_targets
5477 }
5478 RestartRequiredSection::ScopeAuthorityOwners => {
5479 scope_authority_owners != context.scope_authority_owners
5480 }
5481 RestartRequiredSection::PrivilegedOperatorPrompt => {
5482 privileged_operator_prompt != context.privileged_operator_prompt
5483 }
5484 };
5485 if changed {
5486 restart_required.push(section.label().to_string());
5487 }
5488 }
5489 if !restart_required.is_empty() {
5490 warn!(
5491 config_path = %context.config_path.display(),
5492 sections = %restart_required.join(", "),
5493 "daemon config changed outside the modules section; restart the daemon to apply those changes"
5494 );
5495 }
5496
5497 for configured in &modules {
5498 if let Err(err) = validate_spec(&configured.module_spec()) {
5499 return Ok(vec![control_error_frame(
5500 &frame,
5501 "invalid_daemon_config",
5502 format!("supervisor rescan rejected daemon config: {err}"),
5503 )?]);
5504 }
5505 }
5506
5507 let configured_capabilities = modules
5508 .iter()
5509 .map(|module| (module.module_id.clone(), module.enabled))
5510 .collect::<Vec<_>>();
5511 let preview_capability_warnings = if preview {
5512 let (_, registrations) = self.runtime_capability_snapshot()?;
5513 let current_modules = self
5514 .supervisor
5515 .list()
5516 .into_iter()
5517 .map(|module| module.module_id().to_string())
5518 .collect::<BTreeSet<_>>();
5519 let resulting_modules = configured_capabilities.clone();
5520 let removed = current_modules
5521 .into_iter()
5522 .filter(|module_id| {
5523 !resulting_modules
5524 .iter()
5525 .any(|(configured_id, _)| configured_id == module_id)
5526 })
5527 .collect::<Vec<_>>();
5528 self.capability_evaluator.preview_removal_warnings(
5529 resulting_modules,
5530 &removed,
5531 ®istrations,
5532 )
5533 } else {
5534 Vec::new()
5535 };
5536 let result = match self
5537 .reconcile_supervised_modules(&context.supervisor, modules, preview, requester.clone())
5538 .await
5539 {
5540 Ok(result) => result,
5541 Err(message) => {
5542 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5543 }
5544 };
5545 if !preview {
5546 if let Some(ledger) = &self.activity {
5547 ledger.module_rescanned(&result.added, &result.removed, requester);
5548 }
5549 if let Err(code) = self.configure_activity(activity_config).await {
5550 warn!(
5551 code,
5552 "activity retention could not be applied during rescan"
5553 );
5554 }
5555 self.capability_evaluator
5556 .configure(configured_capabilities, reserved_capabilities);
5557 self.capability_evaluator.wake_deadline_loop();
5558 self.refresh_capability_requirements();
5559 }
5560 let mut result = result;
5561 result.restart_required = restart_required;
5562 result.capability_warnings = preview_capability_warnings;
5563 let response = ClientControlResponse::SupervisorRescan { result };
5564 Ok(vec![control_response_body_frame(
5565 &frame,
5566 &response,
5567 "ClientControlResponse::SupervisorRescan",
5568 )?])
5569 }
5570
5571 async fn handle_supervisor_release_reserved(
5572 &self,
5573 frame: Frame,
5574 module_id: String,
5575 ) -> Result<Vec<Frame>, RouterError> {
5576 let Some(context) = self.rescan.clone() else {
5577 return Ok(vec![control_error_frame(
5578 &frame,
5579 "release_unavailable",
5580 "reserved-id release requires a daemon started with a reloadable config path",
5581 )?]);
5582 };
5583 let operation_lock = self.supervisor.operation_lock();
5584 let _operation_guard = operation_lock.lock().await;
5585 let loaded = match crate::daemon_config::load(&context.config_path) {
5586 Ok(Some(config)) => config,
5587 Ok(None) => {
5588 return Ok(vec![control_error_frame(
5589 &frame,
5590 "invalid_daemon_config",
5591 format!(
5592 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5593 context.config_path.display()
5594 ),
5595 )?])
5596 }
5597 Err(err) => {
5598 return Ok(vec![control_error_frame(
5599 &frame,
5600 "invalid_daemon_config",
5601 format!("unable to verify reserved-id release against daemon config: {err}"),
5602 )?])
5603 }
5604 };
5605 if loaded
5606 .modules
5607 .iter()
5608 .any(|configured| configured.module_id == module_id)
5609 {
5610 return Ok(vec![control_error_frame(
5611 &frame,
5612 "reserved_module_configured",
5613 format!(
5614 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5615 ),
5616 )?]);
5617 }
5618 if !self.supervisor.release_retained_reserved_gate(&module_id) {
5619 return Ok(vec![control_error_frame(
5620 &frame,
5621 "reserved_gate_not_retained",
5622 format!(
5623 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
5624 ),
5625 )?]);
5626 }
5627
5628 let response = ClientControlResponse::SupervisorAck {
5629 module_id,
5630 applied: true,
5631 };
5632 Ok(vec![control_response_body_frame(
5633 &frame,
5634 &response,
5635 "ClientControlResponse::SupervisorAck",
5636 )?])
5637 }
5638
5639 async fn reconcile_supervised_modules(
5648 &self,
5649 supervisor: &Supervisor,
5650 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
5651 preview: bool,
5652 requester: Principal,
5653 ) -> Result<SupervisorRescanResult, String> {
5654 let mut current = BTreeMap::new();
5655 for module in self.supervisor.list() {
5656 let (spec, health) = module.configuration().map_err(|err| {
5657 format!(
5658 "failed to read configuration for module_id '{}': {err}",
5659 module.module_id()
5660 )
5661 })?;
5662 let enabled = module
5663 .status()
5664 .map_err(|err| {
5665 format!(
5666 "failed to read status for module_id '{}': {err}",
5667 module.module_id()
5668 )
5669 })?
5670 .enabled;
5671 current.insert(
5672 module.module_id().to_string(),
5673 (module, spec, health, enabled),
5674 );
5675 }
5676 let configured = configured_modules
5677 .into_iter()
5678 .map(|module| (module.module_id.clone(), module))
5679 .collect::<BTreeMap<_, _>>();
5680
5681 let added = configured
5682 .keys()
5683 .filter(|module_id| !current.contains_key(*module_id))
5684 .cloned()
5685 .collect::<Vec<_>>();
5686 let removed = current
5687 .keys()
5688 .filter(|module_id| !configured.contains_key(*module_id))
5689 .cloned()
5690 .collect::<Vec<_>>();
5691 let mut changed_pending_reload = Vec::new();
5692 let mut configuration_changes = BTreeSet::new();
5693 let mut enabled_changes = BTreeSet::new();
5694 let mut unchanged = 0_u32;
5695
5696 for (module_id, configured_module) in &configured {
5697 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5698 else {
5699 continue;
5700 };
5701 let launch_changed = *current_spec != configured_module.module_spec();
5705 let configuration_changed =
5706 launch_changed || *current_health != configured_module.health;
5707 let enabled_changed = *current_enabled != configured_module.enabled;
5708 if configuration_changed {
5709 configuration_changes.insert(module_id.clone());
5710 }
5711 if launch_changed {
5712 changed_pending_reload.push(module_id.clone());
5713 }
5714 if enabled_changed {
5715 enabled_changes.insert(module_id.clone());
5716 }
5717 if !configuration_changed && !enabled_changed {
5718 unchanged = unchanged.saturating_add(1);
5719 }
5720 }
5721
5722 if preview {
5728 return Ok(SupervisorRescanResult {
5729 added,
5730 removed,
5731 changed_pending_reload,
5732 enabled_changes: enabled_changes.iter().cloned().collect(),
5733 unchanged,
5734 preview: true,
5735 restart_required: Vec::new(),
5739 capability_warnings: Vec::new(),
5740 });
5741 }
5742
5743 for module_id in &removed {
5744 let module = ¤t
5745 .get(module_id)
5746 .expect("removed module came from current supervisor state")
5747 .0;
5748 module
5749 .retire_requested(Some(requester.clone()))
5750 .await
5751 .map_err(|err| {
5752 format!("failed to retire module_id '{module_id}' during rescan: {err}")
5753 })?;
5754 self.supervisor.record_rescan_removal(module_id);
5788 self.supervisor.retire(module_id);
5789 self.route_outages.forget(module_id);
5790 }
5791
5792 for module_id in configured.keys() {
5793 let Some((module, _, _, _)) = current.get(module_id) else {
5794 continue;
5795 };
5796 let configured_module = configured
5797 .get(module_id)
5798 .expect("configured module id came from configured map");
5799 if configuration_changes.contains(module_id) {
5800 module
5801 .update_configuration(
5802 configured_module.module_spec(),
5803 configured_module.health.clone(),
5804 configured_module.drain_timeout_ms,
5805 )
5806 .await
5807 .map_err(|err| {
5808 format!(
5809 "failed to update module_id '{module_id}' configuration during rescan: {err}"
5810 )
5811 })?;
5812 }
5813 if enabled_changes.contains(module_id) {
5814 self.route_outages.mark_operator_action(module_id);
5817 module
5818 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
5819 .await
5820 .map_err(|err| {
5821 self.route_outages.operator_action_ended_unrefused(module_id);
5822 format!(
5823 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
5824 configured_module.enabled
5825 )
5826 })?;
5827 }
5828 }
5829
5830 for module_id in &added {
5831 let configured_module = configured
5832 .get(module_id)
5833 .expect("added module id came from configured map");
5834 supervisor
5835 .supervise_configured_requested(
5836 configured_module.module_spec(),
5837 configured_module.enabled,
5838 configured_module.health.clone(),
5839 configured_module.drain_timeout_ms,
5840 configured_module.restart,
5841 Some(requester.clone()),
5842 )
5843 .map_err(|err| {
5844 format!("failed to add module_id '{module_id}' during rescan: {err}")
5845 })?;
5846 }
5847
5848 Ok(SupervisorRescanResult {
5849 added,
5850 removed,
5851 changed_pending_reload,
5852 enabled_changes: enabled_changes.iter().cloned().collect(),
5853 unchanged,
5854 preview: false,
5855 restart_required: Vec::new(),
5858 capability_warnings: Vec::new(),
5859 })
5860 }
5861
5862 async fn handle_supervisor_set_enabled(
5863 &self,
5864 frame: Frame,
5865 module_id: String,
5866 enabled: bool,
5867 requester: Principal,
5868 ) -> Result<Vec<Frame>, RouterError> {
5869 let operation_lock = self.supervisor.operation_lock();
5870 let _operation_guard = operation_lock.lock().await;
5871 let Some(module) = self.supervisor.get(&module_id) else {
5872 return Ok(vec![control_error_frame(
5873 &frame,
5874 "unknown_module",
5875 format!("module_id '{module_id}' is not supervised"),
5876 )?]);
5877 };
5878
5879 self.route_outages.mark_operator_action(&module_id);
5882 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
5883 Ok(applied) => applied,
5884 Err(err) => {
5885 self.route_outages
5886 .operator_action_ended_unrefused(&module_id);
5887 return Ok(vec![control_error_frame(
5888 &frame,
5889 "target_unavailable",
5890 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
5891 )?]);
5892 }
5893 };
5894 if !applied {
5895 self.route_outages
5898 .operator_action_ended_unrefused(&module_id);
5899 }
5900
5901 self.capability_evaluator.wake_deadline_loop();
5902 self.refresh_capability_requirements();
5903 let response = ClientControlResponse::SupervisorAck { module_id, applied };
5904 Ok(vec![control_response_body_frame(
5905 &frame,
5906 &response,
5907 "ClientControlResponse::SupervisorAck",
5908 )?])
5909 }
5910
5911 async fn handle_supervisor_health_probe(
5912 &self,
5913 frame: Frame,
5914 module_id: String,
5915 ) -> Result<Vec<Frame>, RouterError> {
5916 self.refresh_capability_requirements();
5917 let Some(registration) = self
5918 .registry
5919 .get_module(&module_id)
5920 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5921 else {
5922 return Ok(vec![control_error_frame(
5923 &frame,
5924 "unknown_module",
5925 format!("module_id '{module_id}' is not registered"),
5926 )?]);
5927 };
5928
5929 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
5942 {
5943 return Ok(vec![control_error_frame(
5944 &frame,
5945 "health_not_advertised",
5946 format!("module_id '{module_id}' did not advertise health.check"),
5947 )?]);
5948 }
5949
5950 let deadline = Instant::now() + self.health_probe_timeout;
5951 let pending = match self.forwarding.begin_module_control_rpc_for(
5952 &module_id,
5953 MODULE_CONTROL_OP_HEALTH_CHECK,
5954 deadline,
5955 ) {
5956 Ok(pending) => pending,
5957 Err(err) => {
5958 return Ok(vec![control_error_frame(
5959 &frame,
5960 forwarding_error_code(&err),
5961 err.to_string(),
5962 )?])
5963 }
5964 };
5965
5966 let PendingModuleControlRpc {
5967 endpoint,
5968 module_sink,
5969 negotiated_ver,
5970 corr: probe_corr,
5971 receiver,
5972 } = pending;
5973 let mut guard =
5974 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
5975 let probe_body =
5976 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5977 RouterError::backend(
5978 0,
5979 frame.header.corr,
5980 format!("failed to encode health.check request: {err}"),
5981 )
5982 })?;
5983 let probe_frame = Frame::build_with_version(
5984 negotiated_ver,
5985 FrameType::Request,
5986 control_flags(),
5987 0,
5988 0,
5989 probe_corr,
5990 probe_body,
5991 )
5992 .map_err(RouterError::FrameBuild)?;
5993
5994 if let Err(err) = module_sink.send(probe_frame).await {
5995 return Ok(vec![control_error_frame(
5996 &frame,
5997 "target_unavailable",
5998 err.to_string(),
5999 )?]);
6000 }
6001
6002 match timeout_at(deadline, receiver).await {
6003 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6004 guard.disarm();
6005 let Some(report) = response.health_report() else {
6006 return Ok(vec![control_error_frame(
6007 &frame,
6008 "invalid_control_body",
6009 "health.check RPC returned a non-health response",
6010 )?]);
6011 };
6012 let HealthReport {
6017 status,
6018 detail,
6019 metrics,
6020 } = report;
6021 let capability_detail = self
6022 .capability_evaluator
6023 .required_problem_detail(&module_id);
6024 let response = ClientControlResponse::SupervisorHealthProbe {
6025 module_id,
6026 status,
6027 detail: append_capability_problem_detail(detail, capability_detail),
6028 metrics,
6029 };
6030 Ok(vec![control_response_body_frame(
6031 &frame,
6032 &response,
6033 "ClientControlResponse::SupervisorHealthProbe",
6034 )?])
6035 }
6036 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6037 guard.disarm();
6038 Ok(vec![control_error_body_frame(&frame, body)?])
6039 }
6040 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6041 guard.disarm();
6042 Ok(vec![control_error_frame(
6043 &frame,
6044 "target_unavailable",
6045 message,
6046 )?])
6047 }
6048 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6049 guard.disarm();
6050 Ok(vec![control_error_frame(
6051 &frame,
6052 "invalid_control_body",
6053 message,
6054 )?])
6055 }
6056 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6057 guard.disarm();
6058 Ok(vec![control_error_frame(
6059 &frame,
6060 "invalid_control_body",
6061 format!("expected module-control op '{expected}', got '{actual}'"),
6062 )?])
6063 }
6064 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6065 guard.disarm();
6066 Ok(vec![control_error_frame(
6067 &frame,
6068 "module_timeout",
6069 format!(
6070 "module_id '{module_id}' answered health.check after {:?}",
6071 self.health_probe_timeout
6072 ),
6073 )?])
6074 }
6075 Ok(Err(_)) => Ok(vec![control_error_frame(
6076 &frame,
6077 "target_unavailable",
6078 "health.check waiter was canceled before the module responded",
6079 )?]),
6080 Err(_) => Ok(vec![control_error_frame(
6081 &frame,
6082 "module_timeout",
6083 format!(
6084 "module_id '{module_id}' did not answer health.check within {:?}",
6085 self.health_probe_timeout
6086 ),
6087 )?]),
6088 }
6089 }
6090
6091 fn supervisor_status(
6092 &self,
6093 module_id: &str,
6094 corr: u64,
6095 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6096 self.supervisor
6097 .get(module_id)
6098 .map(|module| {
6099 let warming = module.is_warming_for_control("status").map_err(|err| {
6100 RouterError::backend(
6101 0,
6102 corr,
6103 format!(
6104 "failed to read supervisor warming state for module_id '{module_id}': {err}"
6105 ),
6106 )
6107 })?;
6108 module.status_for_control("status").map_err(|err| {
6109 RouterError::backend(
6110 0,
6111 corr,
6112 format!(
6113 "failed to read supervisor status for module_id '{module_id}': {err}"
6114 ),
6115 )
6116 }).map(|status| (status, warming))
6117 })
6118 .transpose()
6119 }
6120
6121 fn guard_module_control_op(
6122 &self,
6123 frame: &Frame,
6124 module_id: &str,
6125 op: &str,
6126 ) -> Result<Option<Frame>, RouterError> {
6127 if self.module_grants_op(module_id, op, frame.header.corr)? {
6128 return Ok(None);
6129 }
6130
6131 Ok(Some(control_error_frame(
6132 frame,
6133 "op_not_allowed",
6134 format!("module_id '{module_id}' did not grant control op '{op}'"),
6135 )?))
6136 }
6137
6138 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6139 let Some(registration) = self
6140 .registry
6141 .get_module(module_id)
6142 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6143 else {
6144 return Ok(false);
6145 };
6146 Ok(module_registration_grants_op(®istration.control_ops, op))
6147 }
6148
6149 fn handle_status_update(
6150 &self,
6151 endpoint: ModuleEndpointId,
6152 frame: Frame,
6153 ) -> Result<Vec<Frame>, RouterError> {
6154 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6155 Ok(update) => update,
6156 Err(err) => {
6157 if is_known_module_push_op(&frame.body) {
6162 return Ok(vec![control_error_frame(
6163 &frame,
6164 "invalid_control_body",
6165 format!("malformed module control push body: {err}"),
6166 )?]);
6167 }
6168 return Ok(Vec::new());
6169 }
6170 };
6171
6172 match update {
6173 ModuleControlPush::RouteStatus {
6174 route_channel,
6175 route_epoch,
6176 status,
6177 } => {
6178 self.forwarding
6179 .cache_status(endpoint, route_channel, route_epoch, status)
6180 .map_err(RouterError::Forwarding)?;
6181 }
6182 }
6183 Ok(Vec::new())
6184 }
6185
6186 fn handle_route_poll(
6187 &self,
6188 ctx: &RouteCtx,
6189 frame: Frame,
6190 route_channel: u16,
6191 route_epoch: u32,
6192 kind: PollKind,
6193 ) -> Result<Vec<Frame>, RouterError> {
6194 let snapshot = self
6195 .forwarding
6196 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6197 .map_err(RouterError::Forwarding)?;
6198 let response = match (kind, snapshot) {
6199 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6200 ClientControlResponse::RoutePoll {
6201 route_channel,
6202 route_epoch,
6203 status,
6204 live: None,
6205 }
6206 }
6207 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6208 route_channel,
6209 route_epoch,
6210 status: None,
6211 live: None,
6212 },
6213 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6214 let live = self
6240 .process_liveness
6241 .as_ref()
6242 .and_then(|source| source.process_live(&module_id))
6243 .unwrap_or(true);
6244 ClientControlResponse::RoutePoll {
6245 route_channel,
6246 route_epoch,
6247 status: None,
6248 live: Some(live),
6249 }
6250 }
6251 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6252 route_channel,
6253 route_epoch,
6254 status: None,
6255 live: Some(false),
6256 },
6257 };
6258
6259 Ok(vec![control_response_body_frame(
6260 &frame,
6261 &response,
6262 "ClientControlResponse::RoutePoll",
6263 )?])
6264 }
6265
6266 pub(crate) fn observe_module_control_completion(
6267 &self,
6268 completion: ModuleControlRpcCompletion,
6269 ) -> bool {
6270 match completion {
6271 ModuleControlRpcCompletion::Unknown => false,
6272 ModuleControlRpcCompletion::Settled => true,
6273 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6274 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6275 info!(
6276 module_id = %module_id,
6277 latency_ms,
6278 "late health.check answer proves the module is alive"
6279 );
6280 match self
6281 .supervisor
6282 .record_late_health_answer(&module_id, latency_ms)
6283 {
6284 Ok(true) => {}
6285 Ok(false) => debug!(
6286 module_id = %module_id,
6287 latency_ms,
6288 "late health.check answer has no active supervisor snapshot"
6289 ),
6290 Err(err) => warn!(
6291 module_id = %module_id,
6292 latency_ms,
6293 error = %err,
6294 "failed to record late health.check answer"
6295 ),
6296 }
6297 true
6298 }
6299 }
6300 }
6301
6302 fn refuse_to_end_module_connection_for_a_client(
6323 &self,
6324 module_connection_id: ConnectionId,
6325 corr: u64,
6326 err: ForwardingError,
6327 ) -> Result<(), RouterError> {
6328 if let ForwardingError::ConnectionClosing { connection_id } = err {
6329 if connection_id != module_connection_id {
6330 warn!(
6331 module_connection_id = module_connection_id.get(),
6332 client_connection_id = connection_id.get(),
6333 corr,
6334 "dropping a route.bind response for a closing client; the module connection keeps serving"
6335 );
6336 return Ok(());
6337 }
6338 }
6339 Err(RouterError::Forwarding(err))
6340 }
6341
6342 fn handle_module_relay_response(
6343 &self,
6344 connection_id: ConnectionId,
6345 frame: Frame,
6346 ) -> Result<Vec<Frame>, RouterError> {
6347 let mut secondary_error = None;
6348 let outcome = match frame.header.ty {
6349 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6350 Ok(probe) if probe.op == "route.bind" => {
6351 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6352 Ok(ModuleControlResponse::RouteBindAck {}) => {
6353 RouteBindRelayOutcome::Accepted
6354 }
6355 Ok(other) => {
6356 let message =
6357 format!("route.bind response carried unexpected body: {other:?}");
6358 secondary_error = Some(control_error_frame(
6359 &frame,
6360 "invalid_control_body",
6361 message.clone(),
6362 )?);
6363 RouteBindRelayOutcome::ModuleGone(message)
6364 }
6365 Err(err) => {
6366 let message = format!("malformed route.bind response body: {err}");
6367 secondary_error = Some(control_error_frame(
6368 &frame,
6369 "invalid_control_body",
6370 message.clone(),
6371 )?);
6372 RouteBindRelayOutcome::ModuleGone(message)
6373 }
6374 }
6375 }
6376 Ok(probe) => {
6377 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6378 {
6379 Ok(response) => ModuleControlRpcOutcome::Response(response),
6380 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6381 "malformed {} response body: {err}",
6382 probe.op
6383 )),
6384 };
6385 let completion = self
6386 .forwarding
6387 .complete_module_control_rpc(
6388 connection_id,
6389 frame.header.corr,
6390 Some(&probe.op),
6391 outcome,
6392 )
6393 .map_err(RouterError::Forwarding)?;
6394 if !self.observe_module_control_completion(completion) {
6395 debug!(
6396 connection_id = connection_id.get(),
6397 corr = frame.header.corr,
6398 op = %probe.op,
6399 "dropping late or unknown module-control RPC response"
6400 );
6401 }
6402 return Ok(Vec::new());
6403 }
6404 Err(err) => {
6405 if let Some(expected_op) = self
6406 .forwarding
6407 .pending_module_control_op(connection_id, frame.header.corr)
6408 .map_err(RouterError::Forwarding)?
6409 {
6410 let completion = self
6411 .forwarding
6412 .complete_module_control_rpc(
6413 connection_id,
6414 frame.header.corr,
6415 None,
6416 ModuleControlRpcOutcome::MalformedResponse(format!(
6417 "malformed {expected_op} response body: {err}"
6418 )),
6419 )
6420 .map_err(RouterError::Forwarding)?;
6421 if !self.observe_module_control_completion(completion) {
6422 debug!(
6423 connection_id = connection_id.get(),
6424 corr = frame.header.corr,
6425 "dropping late malformed module-control RPC response"
6426 );
6427 }
6428 return Ok(Vec::new());
6429 }
6430 let message = format!("malformed route.bind response body: {err}");
6431 secondary_error = Some(control_error_frame(
6432 &frame,
6433 "invalid_control_body",
6434 message.clone(),
6435 )?);
6436 RouteBindRelayOutcome::ModuleGone(message)
6437 }
6438 },
6439 FrameType::Error => {
6440 if self
6441 .forwarding
6442 .pending_module_control_op(connection_id, frame.header.corr)
6443 .map_err(RouterError::Forwarding)?
6444 .is_some()
6445 {
6446 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6447 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
6448 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6449 "malformed module-control ERROR body: {err}"
6450 )),
6451 };
6452 let completion = self
6453 .forwarding
6454 .complete_module_control_rpc(
6455 connection_id,
6456 frame.header.corr,
6457 None,
6458 outcome,
6459 )
6460 .map_err(RouterError::Forwarding)?;
6461 if !self.observe_module_control_completion(completion) {
6462 debug!(
6463 connection_id = connection_id.get(),
6464 corr = frame.header.corr,
6465 "dropping late or unknown module-control RPC error"
6466 );
6467 }
6468 return Ok(Vec::new());
6469 }
6470 match serde_json::from_slice::<ErrorBody>(&frame.body) {
6471 Ok(body) => RouteBindRelayOutcome::Rejected(body),
6472 Err(err) => {
6473 let message = format!("malformed route.bind ERROR body: {err}");
6474 secondary_error = Some(control_error_frame(
6475 &frame,
6476 "invalid_control_body",
6477 message.clone(),
6478 )?);
6479 RouteBindRelayOutcome::ModuleGone(message)
6480 }
6481 }
6482 }
6483 ty => {
6484 return Ok(vec![control_error_frame(
6485 &frame,
6486 "unsupported_control_frame",
6487 format!("unsupported module channel-0 frame {ty:?}"),
6488 )?])
6489 }
6490 };
6491
6492 let settled =
6493 self.forwarding
6494 .complete_pending_relay(connection_id, frame.header.corr, outcome);
6495 let completion = match settled {
6496 Ok(completion) => completion,
6497 Err(err) => {
6498 self.refuse_to_end_module_connection_for_a_client(
6499 connection_id,
6500 frame.header.corr,
6501 err,
6502 )?;
6503 return Ok(secondary_error.into_iter().collect());
6504 }
6505 };
6506 if let Some(target) = completion.abandoned.as_ref() {
6507 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6508 }
6509 if !completion.settled {
6510 debug!(
6511 connection_id = connection_id.get(),
6512 corr = frame.header.corr,
6513 frame_type = ?frame.header.ty,
6514 "dropping late or unknown route.bind relay response"
6515 );
6516 }
6517 Ok(secondary_error.into_iter().collect())
6518 }
6519
6520 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6521 debug!(connection_id = connection_id.get(), "handling GOODBYE");
6522 self.cleanup_connection_with_end_reason(
6526 connection_id,
6527 RegistrationEndReason::ExplicitGoodbye,
6528 )
6529 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6530 Ok(Vec::new())
6531 }
6532}
6533
6534impl Default for ControlHandler {
6535 fn default() -> Self {
6536 Self::new(Arc::new(Registry::default()))
6537 }
6538}
6539
6540impl crate::supervise::SwapPromotionObserver for ControlHandler {
6541 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6542 self.apply_registration_capabilities(registration);
6543 }
6544}
6545
6546fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6547 CapabilityRequirementStatus {
6548 consumer: status.consumer,
6549 capability: status.capability,
6550 need: match status.need {
6551 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6552 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6553 },
6554 verdict: status.verdict.as_str().to_string(),
6555 episode_seq: status.episode_seq,
6556 config_satisfiable: status.config_satisfiable,
6557 runtime_available: status.runtime_available,
6558 detail: status.detail,
6559 }
6560}
6561
6562fn append_capability_problem_detail(
6563 detail: Option<String>,
6564 capability_detail: Option<String>,
6565) -> Option<String> {
6566 match (detail, capability_detail) {
6567 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
6568 (Some(detail), None) => Some(detail),
6569 (None, Some(capability_detail)) => Some(capability_detail),
6570 (None, None) => None,
6571 }
6572}
6573
6574fn subc_ops() -> Vec<String> {
6575 SUBC_CONTROL_OPS
6576 .iter()
6577 .map(|op| (*op).to_string())
6578 .collect()
6579}
6580
6581fn module_subc_ops() -> Vec<String> {
6582 SUBC_CONTROL_OPS
6583 .iter()
6584 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
6585 .map(|op| (*op).to_string())
6586 .collect()
6587}
6588
6589#[cfg(test)]
6590fn module_baseline_control_ops() -> Vec<String> {
6591 MODULE_BASELINE_CONTROL_OPS
6592 .iter()
6593 .map(|op| (*op).to_string())
6594 .collect()
6595}
6596
6597fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
6598 let mut seen = HashSet::new();
6599 let mut effective = Vec::new();
6600 for op in MODULE_BASELINE_CONTROL_OPS {
6601 if seen.insert((*op).to_string()) {
6602 effective.push((*op).to_string());
6603 }
6604 }
6605 for op in declared.unwrap_or_default() {
6606 if seen.insert(op.clone()) {
6607 effective.push(op);
6608 }
6609 }
6610 effective
6611}
6612
6613fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
6614 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
6615}
6616
6617fn target_module_id(target: &RouteTarget) -> &str {
6618 match target {
6619 RouteTarget::ToolProvider { module_id }
6620 | RouteTarget::ManagementSurface { module_id }
6621 | RouteTarget::InternalService { module_id, .. } => module_id,
6622 }
6623}
6624
6625fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
6626 roles.iter().any(|role| match (target, role) {
6627 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
6628 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
6629 (
6630 RouteTarget::InternalService { service_id, .. },
6631 ProviderRole::InternalService {
6632 service_id: provided,
6633 ..
6634 },
6635 ) => service_id == provided,
6636 _ => false,
6637 })
6638}
6639
6640fn is_routable_role(role: &ProviderRole) -> bool {
6641 matches!(
6642 role,
6643 ProviderRole::ToolProvider { .. }
6644 | ProviderRole::ManagementSurface { .. }
6645 | ProviderRole::InternalService { .. }
6646 )
6647}
6648
6649#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6650enum ControlRequestBodyError {
6651 UnknownOp,
6652 InvalidBody,
6653}
6654
6655#[derive(Debug, Deserialize)]
6656struct ControlOpProbe {
6657 op: String,
6658}
6659
6660const MODULE_PUSH_OPS: &[&str] = &["route.status"];
6663
6664fn is_known_module_push_op(body: &[u8]) -> bool {
6665 serde_json::from_slice::<ControlOpProbe>(body)
6666 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
6667 .unwrap_or(false)
6668}
6669
6670fn is_known_module_request_op(body: &[u8]) -> bool {
6671 serde_json::from_slice::<ControlOpProbe>(body)
6672 .map(|probe| is_module_to_subc_op(&probe.op))
6673 .unwrap_or(false)
6674}
6675
6676fn is_module_to_subc_op(op: &str) -> bool {
6677 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
6678}
6679
6680fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
6681 debug!(
6682 op = %op,
6683 connection_id = connection_id.get(),
6684 corr,
6685 "control dispatch"
6686 );
6687}
6688
6689fn log_slow_control_dispatch(
6690 dispatch_started_at: Option<StdInstant>,
6691 op: &'static str,
6692 connection_id: ConnectionId,
6693 corr: u64,
6694) {
6695 let Some(dispatch_started_at) = dispatch_started_at else {
6696 return;
6697 };
6698 let elapsed = dispatch_started_at.elapsed();
6699 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
6700 warn!(
6701 op = %op,
6702 connection_id = connection_id.get(),
6703 corr,
6704 elapsed_ms = elapsed.as_millis() as u64,
6705 "slow control dispatch"
6706 );
6707 }
6708}
6709
6710#[cfg(test)]
6712type TestCallerLookup =
6713 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
6714
6715fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
6717 match request {
6718 ClientControlRequest::SupervisorRestart { module_id, .. } => {
6719 Some(PrivilegedOperation::Restart { module_id })
6720 }
6721 ClientControlRequest::SupervisorSwap { module_id, .. } => {
6722 Some(PrivilegedOperation::Swap { module_id })
6723 }
6724 ClientControlRequest::SupervisorReload { module_id } => {
6725 Some(PrivilegedOperation::Reload { module_id })
6726 }
6727 ClientControlRequest::SupervisorRescan { preview } => {
6728 if *preview {
6729 None
6730 } else {
6731 Some(PrivilegedOperation::Rescan)
6732 }
6733 }
6734 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
6735 Some(PrivilegedOperation::SetEnabled {
6736 module_id,
6737 enabled: *enabled,
6738 })
6739 }
6740 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
6741 Some(PrivilegedOperation::ReleaseReserved { module_id })
6742 }
6743 ClientControlRequest::ServerDescribe {}
6744 | ClientControlRequest::CatalogList { .. }
6745 | ClientControlRequest::RouteOpen { .. }
6746 | ClientControlRequest::RoutePoll { .. }
6747 | ClientControlRequest::SupervisorList {}
6748 | ClientControlRequest::SupervisorHealth {}
6749 | ClientControlRequest::SupervisorHealthProbe { .. }
6750 | ClientControlRequest::SupervisorRoutes { .. }
6751 | ClientControlRequest::SupervisorProvenance { .. }
6752 | ClientControlRequest::SupervisorStderrTail { .. }
6753 | ClientControlRequest::SupervisorTerminals { .. }
6754 | ClientControlRequest::SupervisorSpawnSnapshot {}
6755 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
6756 }
6757}
6758
6759fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
6760 match request {
6761 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
6762 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
6763 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
6764 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
6765 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
6766 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
6767 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
6768 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
6769 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
6770 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
6771 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
6772 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
6773 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
6774 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
6775 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
6776 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
6777 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
6778 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
6779 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
6780 }
6781}
6782
6783fn principal_label(principal: &Principal) -> String {
6784 match principal {
6785 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
6786 Principal::Direct => "direct".to_string(),
6787 other => format!("{other:?}"),
6788 }
6789}
6790
6791fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
6792 match request {
6793 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
6794 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
6795 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
6796 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
6797 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
6798 }
6799}
6800
6801fn parse_client_control_request(
6802 body: &[u8],
6803) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
6804 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
6805 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6806 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
6807 ControlRequestBodyError::InvalidBody
6808 }
6809 Ok(_) => ControlRequestBodyError::UnknownOp,
6810 Err(_) => ControlRequestBodyError::InvalidBody,
6811 };
6812 (err, classification)
6813 })
6814}
6815
6816fn parse_module_control_request_from_module(
6817 body: &[u8],
6818) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
6819 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
6820 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6821 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
6822 Ok(_) => ControlRequestBodyError::UnknownOp,
6823 Err(_) => ControlRequestBodyError::InvalidBody,
6824 };
6825 (err, classification)
6826 })
6827}
6828
6829#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
6830enum ProviderRoleKind {
6831 ToolProvider,
6832 PipelineStage,
6833 ManagementSurface,
6834 InternalService,
6835}
6836
6837fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
6838 match role {
6839 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
6840 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
6841 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
6842 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
6843 }
6844}
6845
6846fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
6847 roles.iter().map(provider_role_kind).collect()
6848}
6849
6850const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
6853
6854#[derive(Debug, Default, PartialEq, Eq)]
6856struct ScopeOutcomeCounts {
6857 created: usize,
6858 replaced: usize,
6859 updated: usize,
6860 unchanged: usize,
6861 refused: usize,
6862}
6863
6864impl ScopeOutcomeCounts {
6865 fn of(results: &[ScopeRecordResult]) -> Self {
6866 let mut counts = Self::default();
6867 for result in results {
6868 let slot = match result.outcome {
6869 ScopeRecordOutcome::Created => &mut counts.created,
6870 ScopeRecordOutcome::Replaced => &mut counts.replaced,
6871 ScopeRecordOutcome::Updated => &mut counts.updated,
6872 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
6873 ScopeRecordOutcome::Refused => &mut counts.refused,
6874 };
6875 *slot += 1;
6876 }
6877 counts
6878 }
6879}
6880
6881#[cfg(test)]
6882mod scope_outcome_count_tests {
6883 use super::*;
6884
6885 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
6886 ScopeRecordResult {
6887 scope_ref: "r".to_string(),
6888 scope_epoch: 1,
6889 outcome,
6890 code: None,
6891 message: None,
6892 version: None,
6893 parent_state: None,
6894 }
6895 }
6896
6897 #[test]
6900 fn every_outcome_is_counted_in_its_own_field() {
6901 let results = [
6902 result(ScopeRecordOutcome::Created),
6903 result(ScopeRecordOutcome::Created),
6904 result(ScopeRecordOutcome::Replaced),
6905 result(ScopeRecordOutcome::Updated),
6906 result(ScopeRecordOutcome::Unchanged),
6907 result(ScopeRecordOutcome::Refused),
6908 result(ScopeRecordOutcome::Refused),
6909 result(ScopeRecordOutcome::Refused),
6910 ];
6911 assert_eq!(
6912 ScopeOutcomeCounts::of(&results),
6913 ScopeOutcomeCounts {
6914 created: 2,
6915 replaced: 1,
6916 updated: 1,
6917 unchanged: 1,
6918 refused: 3,
6919 }
6920 );
6921 }
6922}
6923
6924fn capability_census_trigger(
6928 old: Option<&CapabilityDeclarations>,
6929 new: Option<&CapabilityDeclarations>,
6930) -> bool {
6931 let old_provides = old
6932 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
6933 .unwrap_or_default();
6934 let old_denies = old
6935 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
6936 .unwrap_or_default();
6937 let new = new.cloned().unwrap_or(CapabilityDeclarations {
6938 provides: Vec::new(),
6939 requires: Vec::new(),
6940 must_never_reach: Vec::new(),
6941 });
6942
6943 new.provides
6944 .iter()
6945 .any(|capability| !old_provides.contains(capability))
6946 || new
6947 .must_never_reach
6948 .iter()
6949 .any(|capability| !old_denies.contains(capability))
6950}
6951
6952fn denied_capability<'a>(
6955 opening_manifest: &'a ModuleManifest,
6956 target_manifest: &ModuleManifest,
6957) -> Option<&'a str> {
6958 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
6959 let target_capabilities = target_manifest.capabilities.as_ref()?;
6960 opening_capabilities
6961 .must_never_reach
6962 .iter()
6963 .find(|denied| {
6964 target_capabilities
6965 .provides
6966 .iter()
6967 .any(|provided| provided == *denied)
6968 })
6969 .map(String::as_str)
6970}
6971
6972fn catalog_update_frozen_field_message(
6973 registered: &ModuleManifest,
6974 provides: &[ProviderRole],
6975) -> Option<String> {
6976 let old_has_provides = !registered.provides.is_empty();
6977 let new_has_provides = !provides.is_empty();
6978 if old_has_provides != new_has_provides {
6979 return Some(format!(
6980 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
6981 registered.module_id
6982 ));
6983 }
6984
6985 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
6986 return Some(format!(
6987 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
6988 registered.module_id
6989 ));
6990 }
6991
6992 let registered_concurrency = manifest_concurrency(registered);
6993 let mut candidate = registered.clone();
6994 candidate.provides = provides.to_vec();
6995 let candidate_concurrency = manifest_concurrency(&candidate);
6996 if candidate_concurrency != registered_concurrency {
6997 return Some(format!(
6998 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
6999 registered.module_id, registered_concurrency, candidate_concurrency
7000 ));
7001 }
7002
7003 None
7006}
7007
7008fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7009 manifest.provides.iter().any(is_routable_role)
7010}
7011
7012fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7018 manifest
7019 .provides
7020 .iter()
7021 .find_map(|provider| match provider {
7022 ProviderRole::ToolProvider { concurrency, .. }
7023 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7024 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7025 })
7026 .unwrap_or(Concurrency::ModuleManaged)
7027}
7028
7029fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7036 let has_management_surface = manifest
7037 .provides
7038 .iter()
7039 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7040 if !has_management_surface {
7041 return false;
7042 }
7043 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7044 return false;
7045 };
7046 let Some(provides) = raw
7047 .get("manifest")
7048 .and_then(|manifest| manifest.get("provides"))
7049 .and_then(serde_json::Value::as_array)
7050 else {
7051 return false;
7052 };
7053 provides.iter().any(|role| {
7058 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7059 && role.get("concurrency").is_none()
7060 })
7061}
7062
7063fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7064 if peer_version != PROTOCOL_VERSION {
7065 return Err(format!(
7066 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7067 ));
7068 }
7069 Ok(PROTOCOL_VERSION)
7070}
7071
7072fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7073 Frame::build_with_version(
7074 response_version(frame),
7075 FrameType::Pong,
7076 frame.header.flags,
7077 0,
7078 0,
7079 frame.header.corr,
7080 Vec::new(),
7081 )
7082 .map_err(RouterError::FrameBuild)
7083}
7084
7085pub(crate) fn control_error_frame(
7086 frame: &Frame,
7087 code: &'static str,
7088 message: impl Into<String>,
7089) -> Result<Frame, RouterError> {
7090 control_error_body_frame(
7091 frame,
7092 ErrorBody {
7093 code: code.to_string(),
7094 message: message.into(),
7095 detail: None,
7096 },
7097 )
7098}
7099
7100fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7101 let body = serde_json::to_vec(&error).map_err(|err| {
7102 RouterError::backend(
7103 0,
7104 frame.header.corr,
7105 format!("failed to encode control ERROR: {err}"),
7106 )
7107 })?;
7108
7109 Frame::build_with_version(
7110 response_version(frame),
7111 FrameType::Error,
7112 control_flags(),
7113 0,
7114 0,
7115 frame.header.corr,
7116 body,
7117 )
7118 .map_err(RouterError::FrameBuild)
7119}
7120
7121fn control_response_body_frame<T: Serialize>(
7122 frame: &Frame,
7123 reply: &T,
7124 label: &'static str,
7125) -> Result<Frame, RouterError> {
7126 let body = serde_json::to_vec(reply).map_err(|err| {
7127 RouterError::backend(
7128 0,
7129 frame.header.corr,
7130 format!("failed to encode {label}: {err}"),
7131 )
7132 })?;
7133
7134 Frame::build_with_version(
7135 response_version(frame),
7136 FrameType::Response,
7137 control_flags(),
7138 0,
7139 0,
7140 frame.header.corr,
7141 body,
7142 )
7143 .map_err(RouterError::FrameBuild)
7144}
7145
7146fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7176 match err {
7177 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7178 ForwardingError::NoModuleConnection => "target_unavailable",
7179 ForwardingError::ModuleReloading { .. } => "module_reloading",
7180 ForwardingError::ClientRouteChannelExhausted { .. }
7181 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7182 ForwardingError::StaleModuleEndpoint
7183 | ForwardingError::UnknownReservation { .. }
7184 | ForwardingError::ConnectionClosing { .. }
7185 | ForwardingError::ClientEgressClosed { .. }
7186 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7187 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7190 ForwardingError::RelayCorrelationExhausted
7191 | ForwardingError::RouteOpenBuild(_)
7192 | ForwardingError::Poisoned => "forwarding_error",
7193 }
7194}
7195
7196fn response_version(frame: &Frame) -> u8 {
7197 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7198 frame.header.ver
7199 } else {
7200 PROTOCOL_VERSION
7201 }
7202}
7203
7204fn control_flags() -> Flags {
7205 Flags::new(false, Priority::Passive, false)
7206}
7207
7208fn send_goodbye_target_best_effort(
7213 counters: &DaemonCounters,
7214 target: &GoodbyeTarget,
7215 context: &'static str,
7216) {
7217 let Ok(frame) = Frame::build_with_version(
7218 target.negotiated_ver,
7219 FrameType::Goodbye,
7220 control_flags(),
7221 target.channel,
7222 target.epoch,
7223 0,
7224 Vec::new(),
7225 ) else {
7226 return;
7227 };
7228 crate::forwarding::send_module_route_goodbye(
7229 counters,
7230 &target.sink,
7231 frame,
7232 target.module_id.as_deref(),
7233 context,
7234 );
7235}
7236
7237pub(crate) fn send_route_control_pushes(
7238 forwarding: &ForwardingTable,
7239 routes: Vec<EndpointRoute>,
7240 push: ClientControlPush,
7241) {
7242 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7243 for route in routes {
7244 let target = route.goodbye_target;
7245 if let Some((existing, channels)) = targets
7246 .iter_mut()
7247 .find(|(existing, _)| existing.connection_id == target.connection_id)
7248 {
7249 debug_assert_eq!(
7250 existing.negotiated_ver, target.negotiated_ver,
7251 "one connection cannot negotiate multiple frame versions"
7252 );
7253 if !channels.contains(&target.channel) {
7254 channels.push(target.channel);
7255 }
7256 continue;
7257 }
7258 let channel = target.channel;
7259 targets.push((target, vec![channel]));
7260 }
7261 for (target, mut channels) in targets {
7262 channels.sort_unstable();
7263 let mut push = push.clone();
7264 match &mut push {
7265 ClientControlPush::RouteClosing {
7266 channels: covered, ..
7267 }
7268 | ClientControlPush::RouteClosed {
7269 channels: covered, ..
7270 } => *covered = channels,
7271 }
7272 let body = match serde_json::to_vec(&push) {
7273 Ok(body) => body,
7274 Err(err) => {
7275 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7276 continue;
7277 }
7278 };
7279 let frame = match Frame::build_with_version(
7280 target.negotiated_ver,
7281 FrameType::Push,
7282 control_flags(),
7283 0,
7284 0,
7285 0,
7286 body.clone(),
7287 ) {
7288 Ok(frame) => frame,
7289 Err(err) => {
7290 warn!(
7291 route_channel = target.channel,
7292 error = %err,
7293 "failed to build route lifecycle control PUSH frame"
7294 );
7295 continue;
7296 }
7297 };
7298 if let Err(err) = target.sink.try_send(frame) {
7299 if target.close_on_delivery_failure() {
7300 warn!(
7301 target_connection_id = target.connection_id.get(),
7302 route_channel = target.channel,
7303 error = %err,
7304 "route lifecycle control PUSH was not delivered to client; closing target connection"
7305 );
7306 let _ = forwarding.escalate_client_delivery_failure(
7307 target.connection_id,
7308 target.channel,
7309 target.epoch,
7310 CloseReason::new(
7311 "route_lifecycle_push_delivery_failed",
7312 format!(
7313 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7314 target.channel
7315 ),
7316 ),
7317 crate::forwarding::UndeliveredFrame {
7318 module_id: target.module_id.as_deref(),
7319 sink: &target.sink,
7320 },
7321 );
7322 }
7323 }
7324 }
7325}
7326
7327#[cfg(test)]
7328mod tests {
7329 mod activity_read {
7330 include!("activity_read_tests.rs");
7331 }
7332
7333 mod lifecycle_activity {
7334 include!("control_lifecycle_tests.rs");
7335 }
7336 #[cfg(unix)]
7337 #[tokio::test]
7338 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
7339 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
7340 let path = dir.join("subc.jsonc");
7341 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
7342 "program":"/bin/sleep","args":["60"],"protocol":"none",
7343 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
7344 }}}).to_string()).unwrap();
7345 let mut configured = crate::daemon_config::load(&path)
7346 .unwrap()
7347 .unwrap()
7348 .modules
7349 .pop()
7350 .unwrap();
7351 let registry = std::sync::Arc::new(crate::Registry::default());
7352 let handle = crate::SupervisorHandle::new();
7353 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
7354 .with_handle(handle.clone());
7355 let module = supervisor
7356 .supervise_configured_with_health(
7357 configured.module_spec(),
7358 true,
7359 configured.health.clone(),
7360 None,
7361 configured.restart,
7362 )
7363 .unwrap();
7364 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
7365 let before = module.status().unwrap().pid;
7366 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
7367 configured.health.cadence = std::time::Duration::from_secs(3600);
7368 let health_only = handler
7369 .reconcile_supervised_modules(
7370 &supervisor,
7371 vec![configured.clone()],
7372 false,
7373 Principal::Direct,
7374 )
7375 .await
7376 .unwrap();
7377 assert!(
7378 health_only.changed_pending_reload.is_empty(),
7379 "health policy is already applied live"
7380 );
7381 assert_eq!(module.status().unwrap().pid, before);
7382 assert_eq!(
7383 module.configuration().unwrap().1.http,
7384 configured.health.http
7385 );
7386 configured.args = vec!["61".into()];
7387 let launch = handler
7388 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
7389 .await
7390 .unwrap();
7391 assert_eq!(launch.changed_pending_reload, ["stock"]);
7392 assert_eq!(
7393 module.status().unwrap().pid,
7394 before,
7395 "a launch edit is stored until reload"
7396 );
7397 module.drain().await.unwrap();
7398 }
7399 use cortexkit_test_support::ScratchDir;
7400 use std::{
7401 collections::BTreeMap,
7402 fmt,
7403 path::PathBuf,
7404 sync::{Arc, Mutex},
7405 time::Duration,
7406 };
7407
7408 use serde_json::{json, Value};
7409 use subc_protocol::{
7410 manifest::{
7411 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
7412 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
7413 },
7414 session::HealthStatus,
7415 FrameType,
7416 };
7417
7418 use super::*;
7419 use crate::{
7420 forwarding::{DataRoute, DataRouteState},
7421 registry::ChannelState,
7422 router::FrameSink,
7423 stderr_tail::DEFAULT_MAX_LINE_BYTES,
7424 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
7425 RouteCtx, Router,
7426 };
7427 use tokio::{
7428 sync::mpsc,
7429 time::{sleep, Instant},
7430 };
7431 use tracing::{
7432 field::{Field, Visit},
7433 Event, Subscriber,
7434 };
7435 use tracing_subscriber::{layer::Context, prelude::*, Layer};
7436
7437 fn fake_aft_stub_path() -> PathBuf {
7456 let mut path = std::env::current_exe().expect("current_exe available in tests");
7457 path.pop(); path.pop(); path.push(if cfg!(windows) {
7460 "fake-aft-stub.exe"
7461 } else {
7462 "fake-aft-stub"
7463 });
7464 assert!(
7465 path.exists(),
7466 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
7467 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
7468 path.display()
7469 );
7470 path
7471 }
7472
7473 fn client_retries(code: &str) -> bool {
7478 subc_protocol::error_codes::is_retryable_route_open(code)
7479 }
7480
7481 #[test]
7492 fn retryability_of_forwarding_codes_matches_the_failure() {
7493 let transient = [
7496 ForwardingError::NoModuleConnection,
7497 ForwardingError::ModuleReloading {
7498 module_id: "m".into(),
7499 },
7500 ForwardingError::StaleModuleEndpoint,
7501 ForwardingError::UnknownReservation {
7502 client_channel: 1,
7503 module_channel: 1,
7504 },
7505 ForwardingError::ConnectionClosing {
7506 connection_id: ConnectionId::new(1),
7507 },
7508 ForwardingError::ClientEgressClosed {
7509 connection_id: ConnectionId::new(1),
7510 },
7511 ForwardingError::ModuleEgressUnavailable {
7512 connection_id: ConnectionId::new(1),
7513 },
7514 ];
7515 for err in transient {
7516 let code = forwarding_error_code(&err);
7517 assert!(
7518 client_retries(code),
7519 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
7520 );
7521 }
7522
7523 let permanent = [
7528 ForwardingError::ConnectionRoleConflict {
7529 connection_id: ConnectionId::new(1),
7530 },
7531 ForwardingError::ClientRouteChannelExhausted {
7532 connection_id: ConnectionId::new(1),
7533 },
7534 ForwardingError::ModuleRouteChannelExhausted {
7535 endpoint: ModuleEndpointId {
7536 connection_id: ConnectionId::new(1),
7537 generation: 1,
7538 },
7539 },
7540 ForwardingError::RelayCorrelationExhausted,
7541 ForwardingError::RouteOpenBuild("x".into()),
7542 ForwardingError::Poisoned,
7543 ];
7544 for err in permanent {
7545 let code = forwarding_error_code(&err);
7546 assert!(
7547 !client_retries(code),
7548 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
7549 );
7550 }
7551 }
7552
7553 #[tokio::test]
7581 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
7582 let handler = ControlHandler::default();
7583 let frame =
7584 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
7585
7586 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
7590 assert_eq!(
7591 stamped,
7592 Principal::Direct,
7593 "a caller that proved nothing must not be stamped as a supervised module"
7594 );
7595
7596 let forged = handler
7601 .route_open_principal(
7602 &frame,
7603 Some(ConsumerIdentity {
7604 module_id: "aft".to_string(),
7605 launch_nonce: "not-a-real-nonce".to_string(),
7606 }),
7607 )
7608 .unwrap();
7609 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
7610 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
7611 }
7612
7613 #[test]
7630 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
7631 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
7632 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
7633 let ClientControlRequest::RouteOpen {
7634 consumer_identity, ..
7635 } = parsed
7636 else {
7637 panic!("route.open body must parse as RouteOpen");
7638 };
7639 assert_eq!(
7640 consumer_identity,
7641 Some(ConsumerIdentity {
7642 module_id: "aft".to_string(),
7643 launch_nonce: "n".to_string(),
7644 }),
7645 "the wire field name must reach the value route_open_principal reads"
7646 );
7647 }
7648
7649 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
7650 ModuleManifest::builder(module_id, "0.1.0")
7651 .protocol_ver(protocol_ver)
7652 .provides(vec![ProviderRole::ToolProvider {
7653 tools: vec![Tool {
7654 name: "read".to_string(),
7655 description: None,
7656 execution_mode: ExecutionMode::Pure,
7657 schema: json!({"type": "object"}),
7658 }],
7659 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
7660 concurrency: Concurrency::ModuleManaged,
7661 emits_push: true,
7662 sub_supervises: true,
7663 }])
7664 .build()
7665 }
7666
7667 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
7668 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
7669 }
7670
7671 fn hello_frame_with_control_ops(
7672 module_id: &str,
7673 protocol_ver: u8,
7674 corr: u64,
7675 control_ops: Option<Vec<String>>,
7676 ) -> Frame {
7677 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
7678 }
7679
7680 fn hello_frame_with_nonce(
7681 module_id: &str,
7682 protocol_ver: u8,
7683 corr: u64,
7684 launch_nonce: Option<&str>,
7685 ) -> Frame {
7686 hello_frame_full(
7687 module_id,
7688 protocol_ver,
7689 corr,
7690 None,
7691 launch_nonce.map(ToOwned::to_owned),
7692 )
7693 }
7694
7695 fn hello_frame_full(
7696 module_id: &str,
7697 protocol_ver: u8,
7698 corr: u64,
7699 control_ops: Option<Vec<String>>,
7700 launch_nonce: Option<String>,
7701 ) -> Frame {
7702 let body = serde_json::to_vec(&ModuleHelloBody {
7703 manifest: manifest(module_id, protocol_ver),
7704 protocol_ver,
7705 control_ops,
7706 launch_nonce,
7707 })
7708 .unwrap();
7709 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7710 }
7711
7712 fn non_routable_hello_frame_with_control_ops(
7713 module_id: &str,
7714 corr: u64,
7715 control_ops: Option<Vec<String>>,
7716 ) -> Frame {
7717 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
7718 manifest.provides.clear();
7719 let body = serde_json::to_vec(&ModuleHelloBody {
7720 manifest,
7721 protocol_ver: PROTOCOL_VERSION,
7722 control_ops,
7723 launch_nonce: None,
7724 })
7725 .unwrap();
7726 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7727 }
7728
7729 fn capability_grammar_hello_frame(
7730 capabilities: Value,
7731 runtime_computed: Option<Value>,
7732 corr: u64,
7733 ) -> Frame {
7734 let mut body = serde_json::to_value(ModuleHelloBody {
7735 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
7736 protocol_ver: PROTOCOL_VERSION,
7737 control_ops: None,
7738 launch_nonce: None,
7739 })
7740 .expect("HELLO body serializes");
7741 body["manifest"]["capabilities"] = capabilities;
7742 if let Some(runtime_computed) = runtime_computed {
7743 body["runtime_computed"] = runtime_computed;
7744 }
7745 Frame::build(
7746 FrameType::Hello,
7747 control_flags(),
7748 0,
7749 0,
7750 corr,
7751 serde_json::to_vec(&body).expect("HELLO body reserializes"),
7752 )
7753 .expect("HELLO frame builds")
7754 }
7755
7756 fn channel_request(channel: u16, corr: u64) -> Frame {
7757 Frame::build(
7758 FrameType::Request,
7759 Flags::new(true, Priority::Interactive, false),
7760 channel,
7761 0,
7762 corr,
7763 b"opaque".to_vec(),
7764 )
7765 .unwrap()
7766 }
7767
7768 fn route_ctx(
7769 connection_id: ConnectionId,
7770 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
7771 let (tx, rx) = mpsc::channel(8);
7772 (
7773 RouteCtx {
7774 connection_id,
7775 egress: FrameSink::new(tx),
7776 },
7777 rx,
7778 )
7779 }
7780
7781 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
7782 serde_json::from_slice(&frame.body).unwrap()
7783 }
7784
7785 async fn hello_via_sink(
7790 handler: &ControlHandler,
7791 ctx: &RouteCtx,
7792 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7793 hello: Frame,
7794 ) -> Frame {
7795 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
7796 assert!(
7797 replies.is_empty(),
7798 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
7799 );
7800 let ack = rx
7801 .try_recv()
7802 .expect("HELLO_ACK is queued on the module sink")
7803 .frame;
7804 assert_eq!(ack.header.ty, FrameType::HelloAck);
7805 ack
7806 }
7807
7808 fn parse_error(frame: &Frame) -> Value {
7809 serde_json::from_slice(&frame.body).unwrap()
7810 }
7811
7812 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
7813 serde_json::from_slice(&frame.body).unwrap()
7814 }
7815
7816 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
7817 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
7818 route_channel,
7819 route_epoch: 0,
7820 kind,
7821 })
7822 .unwrap();
7823 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7824 }
7825
7826 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
7827 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
7828 module_id: module_id.to_string(),
7829 })
7830 .unwrap();
7831 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7832 }
7833
7834 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
7835 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
7836 }
7837
7838 fn route_open_frame_with_consumer_capabilities(
7839 corr: u64,
7840 module_id: &str,
7841 project_root: ScratchDir,
7842 consumer_capabilities: Option<Vec<String>>,
7843 ) -> Frame {
7844 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7845 target: RouteTarget::ToolProvider {
7846 module_id: module_id.to_string(),
7847 },
7848 identity: BindIdentity::new(
7849 project_root.path().to_path_buf(),
7850 "unit".to_string(),
7851 "session".to_string(),
7852 ),
7853 consumer_identity: None,
7854 consumer_capabilities,
7855 role_versions: None,
7856 admission_facts: None,
7857 scope: None,
7858 })
7859 .unwrap();
7860 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7861 }
7862
7863 fn route_open_frame_with_role_versions(
7864 corr: u64,
7865 module_id: &str,
7866 project_root: ScratchDir,
7867 role_versions: Option<BTreeMap<String, String>>,
7868 ) -> Frame {
7869 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7870 target: RouteTarget::ToolProvider {
7871 module_id: module_id.to_string(),
7872 },
7873 identity: BindIdentity::new(
7874 project_root.path().to_path_buf(),
7875 "unit".to_string(),
7876 format!("session-{corr}"),
7877 ),
7878 consumer_identity: None,
7879 consumer_capabilities: None,
7880 role_versions,
7881 admission_facts: None,
7882 scope: None,
7883 })
7884 .unwrap();
7885 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7886 }
7887
7888 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
7889 entries
7890 .iter()
7891 .map(|(role, version)| (role.to_string(), version.to_string()))
7892 .collect()
7893 }
7894
7895 fn route_open_frame_with_admission_facts(
7896 corr: u64,
7897 module_id: &str,
7898 project_root: ScratchDir,
7899 consumer_identity: Option<subc_control::ConsumerIdentity>,
7900 facts: Option<Value>,
7901 ) -> Frame {
7902 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7903 target: RouteTarget::ToolProvider {
7904 module_id: module_id.to_string(),
7905 },
7906 identity: BindIdentity::new(
7907 project_root.path().to_path_buf(),
7908 "unit".to_string(),
7909 format!("session-{corr}"),
7910 ),
7911 consumer_identity,
7912 consumer_capabilities: None,
7913 role_versions: None,
7914 admission_facts: facts,
7915 scope: None,
7916 })
7917 .unwrap();
7918 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7919 }
7920
7921 #[derive(Clone, Default)]
7922 struct EventCapture {
7923 events: Arc<Mutex<Vec<CapturedEvent>>>,
7924 }
7925
7926 #[derive(Clone, Debug)]
7927 struct CapturedEvent {
7928 target: String,
7929 level: tracing::Level,
7930 fields: BTreeMap<String, String>,
7931 }
7932
7933 impl EventCapture {
7934 fn events(&self) -> Vec<CapturedEvent> {
7935 self.events.lock().unwrap().clone()
7936 }
7937 }
7938
7939 impl<S> Layer<S> for EventCapture
7940 where
7941 S: Subscriber,
7942 {
7943 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
7944 let mut visitor = EventFieldVisitor::default();
7945 event.record(&mut visitor);
7946 self.events.lock().unwrap().push(CapturedEvent {
7947 target: event.metadata().target().to_string(),
7948 level: *event.metadata().level(),
7949 fields: visitor.fields,
7950 });
7951 }
7952 }
7953
7954 #[derive(Default)]
7955 struct EventFieldVisitor {
7956 fields: BTreeMap<String, String>,
7957 }
7958
7959 impl Visit for EventFieldVisitor {
7960 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
7961 self.fields
7962 .insert(field.name().to_string(), format!("{value:?}"));
7963 }
7964 }
7965
7966 fn health_response(corr: u64, status: HealthStatus) -> Frame {
7967 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
7968 status,
7969 detail: Some("warming".to_string()),
7970 metrics: Some(json!({"queue_depth": 3})),
7971 })
7972 .unwrap();
7973 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7974 }
7975
7976 fn route_bind_ack(corr: u64) -> Frame {
7977 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
7978 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7979 }
7980
7981 fn unique_project_root(label: &str) -> ScratchDir {
7982 ScratchDir::new(label)
7983 }
7984
7985 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
7986 match parse_route_poll(frame) {
7987 ClientControlResponse::RoutePoll {
7988 status: None,
7989 live: Some(live),
7990 ..
7991 } => assert_eq!(live, expected_live),
7992 other => panic!("unexpected route.poll response: {other:?}"),
7993 }
7994 }
7995
7996 fn bind_liveness_route(
7997 registry: &Registry,
7998 forwarding: &ForwardingTable,
7999 module_id: &str,
8000 ) -> (RouteCtx, u16, u32) {
8001 let module_connection = ConnectionId::new(101);
8002 let client_connection = ConnectionId::new(202);
8003 let registration = registry
8004 .register_with_control_ops(
8005 manifest(module_id, PROTOCOL_VERSION),
8006 PROTOCOL_VERSION,
8007 module_connection,
8008 module_baseline_control_ops(),
8009 )
8010 .unwrap();
8011 let (module_tx, _module_rx) = mpsc::channel(8);
8012 let endpoint = forwarding
8013 .register_module_connection(
8014 module_connection,
8015 module_id.to_string(),
8016 PROTOCOL_VERSION,
8017 manifest_concurrency(®istration.manifest),
8018 FrameSink::new(module_tx),
8019 )
8020 .unwrap();
8021 let (client_ctx, _client_rx) = route_ctx(client_connection);
8022 let pending = forwarding
8023 .begin_route_bind_relay_for_test(
8024 client_connection,
8025 client_ctx.egress.clone(),
8026 1,
8027 module_id,
8028 )
8029 .unwrap();
8030 assert_eq!(pending.endpoint, endpoint);
8031 let route_channel = pending.client_channel;
8032 let route_epoch = pending.client_epoch;
8033 forwarding
8034 .complete_pending_relay(
8035 module_connection,
8036 pending.corr,
8037 RouteBindRelayOutcome::Accepted,
8038 )
8039 .unwrap();
8040 (client_ctx, route_channel, route_epoch)
8041 }
8042
8043 struct FakeProcessLiveness {
8044 live: Option<bool>,
8045 }
8046
8047 impl ModuleProcessLiveness for FakeProcessLiveness {
8048 fn process_live(&self, _module_id: &str) -> Option<bool> {
8049 self.live
8050 }
8051 }
8052
8053 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8054 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8055 {
8056 let registry = Arc::new(Registry::default());
8057 let supervisor_handle = SupervisorHandle::new();
8058 let supervisor = Supervisor::new_for_test(
8059 Arc::clone(®istry),
8060 RestartPolicy::new(1, Duration::from_millis(10)),
8061 )
8062 .with_handle(supervisor_handle.clone());
8063 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8064 let module = supervisor
8065 .spawn(ModuleSpec {
8066 module_id: "stderr-tail-wire".to_string(),
8067 program: fake_aft_stub_path(),
8068 args: Vec::new(),
8069 env: vec![
8070 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8071 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8072 ],
8073 reserved: false,
8074 reserved_prefixes: Vec::new(),
8075 protocol: ModuleProtocol::Subc,
8076 overlap: Default::default(),
8077 })
8078 .unwrap();
8079
8080 let deadline = Instant::now() + Duration::from_secs(5);
8081 loop {
8082 let tail = module.stderr_tail(None, None);
8083 if tail
8084 .entries
8085 .iter()
8086 .any(|entry| matches!(entry, TailEntry::ProcessStart))
8087 && tail.entries.iter().any(|entry| {
8088 matches!(
8089 entry,
8090 TailEntry::Line {
8091 truncated: true,
8092 ..
8093 }
8094 )
8095 })
8096 {
8097 break;
8098 }
8099 assert!(
8100 Instant::now() < deadline,
8101 "module did not produce a truncated line and restart boundary: {tail:?}"
8102 );
8103 sleep(Duration::from_millis(10)).await;
8104 }
8105
8106 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8107 let request = ClientControlRequest::SupervisorStderrTail {
8108 module_id: "stderr-tail-wire".to_string(),
8109 max_lines: None,
8110 max_bytes: None,
8111 };
8112 let frame = Frame::build(
8113 FrameType::Request,
8114 control_flags(),
8115 0,
8116 0,
8117 1,
8118 serde_json::to_vec(&request).unwrap(),
8119 )
8120 .unwrap();
8121 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8122 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8123 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8124 serde_json::from_slice(&responses[0].body).unwrap()
8125 else {
8126 panic!("expected supervisor.stderr_tail response");
8127 };
8128
8129 assert!(
8130 tail.entries
8131 .iter()
8132 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8133 "the control response lost the restart boundary"
8134 );
8135 let Some(StderrTailEntry::Line {
8136 text,
8137 truncated,
8138 at_ms,
8139 }) = tail.entries.iter().find(|entry| {
8140 matches!(
8141 entry,
8142 StderrTailEntry::Line {
8143 truncated: true,
8144 ..
8145 }
8146 )
8147 })
8148 else {
8149 panic!("the control response lost the truncated line");
8150 };
8151 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8152 assert!(*truncated);
8153 assert!(
8154 at_ms.is_some(),
8155 "the control response lost the line's capture time"
8156 );
8157 }
8158
8159 #[tokio::test(flavor = "current_thread")]
8164 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8165 let dir = ScratchDir::new("terminals-off-worker");
8166 let journal_path = dir.join("terminals.jsonl");
8167 let registry = Arc::new(Registry::default());
8168 let supervisor_handle = SupervisorHandle::new();
8169 let supervisor =
8170 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8171 .with_handle(supervisor_handle.clone())
8172 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8173 let module = supervisor
8174 .spawn(ModuleSpec {
8175 module_id: "terminal-off-worker".to_string(),
8176 program: fake_aft_stub_path(),
8177 args: Vec::new(),
8178 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8179 reserved: false,
8180 reserved_prefixes: Vec::new(),
8181 protocol: ModuleProtocol::Subc,
8182 overlap: Default::default(),
8183 })
8184 .unwrap();
8185 let deadline = Instant::now() + Duration::from_secs(5);
8186 while module.terminal_history().entries.len() != 2 {
8187 assert!(Instant::now() < deadline, "module did not record two exits");
8188 sleep(Duration::from_millis(10)).await;
8189 }
8190
8191 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8192 let handler =
8193 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
8194 let frame = Frame::build(
8195 FrameType::Request,
8196 control_flags(),
8197 0,
8198 0,
8199 1,
8200 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8201 module_id: "terminal-off-worker".to_string(),
8202 })
8203 .unwrap(),
8204 )
8205 .unwrap();
8206 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8207 let spawned_at = std::time::Instant::now();
8208 let read = tokio::spawn({
8209 let handler = Arc::clone(&handler);
8210 async move { handler.handle_control_frame(&ctx, frame).await }
8211 });
8212 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8215 .await
8216 .unwrap()
8217 .expect("the history read reached its pause");
8218 let elapsed = spawned_at.elapsed();
8219 assert!(
8220 elapsed < Duration::from_secs(2) && !read.is_finished(),
8221 "this task could not run while the history read was paused \
8222 (resumed after {elapsed:?}, read finished: {})",
8223 read.is_finished()
8224 );
8225
8226 drop(release);
8227 let responses = read.await.unwrap().unwrap();
8228 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8229 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8230 panic!("expected supervisor.terminals response");
8231 };
8232 assert_eq!(terminals.entries.len(), 2);
8233 assert_eq!(terminals.journal_skipped_lines, 0);
8234 assert_eq!(terminals.journal_read_errors, 0);
8235 }
8236
8237 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8238 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8239 let registry = Arc::new(Registry::default());
8240 let supervisor_handle = SupervisorHandle::new();
8241 let supervisor =
8242 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8243 .with_handle(supervisor_handle.clone());
8244 let module = supervisor
8245 .spawn(ModuleSpec {
8246 module_id: "terminal-golden".to_string(),
8247 program: fake_aft_stub_path(),
8248 args: Vec::new(),
8249 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8250 reserved: false,
8251 reserved_prefixes: Vec::new(),
8252 protocol: ModuleProtocol::Subc,
8253 overlap: Default::default(),
8254 })
8255 .unwrap();
8256
8257 let deadline = Instant::now() + Duration::from_secs(5);
8258 while module.terminal_history().entries.len() != 2 {
8259 assert!(
8260 Instant::now() < deadline,
8261 "module did not retain two terminal exits: {:?}",
8262 module.terminal_history()
8263 );
8264 sleep(Duration::from_millis(10)).await;
8265 }
8266
8267 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8268 let request = ClientControlRequest::SupervisorTerminals {
8269 module_id: "terminal-golden".to_string(),
8270 };
8271 let frame = Frame::build(
8272 FrameType::Request,
8273 control_flags(),
8274 0,
8275 0,
8276 1,
8277 serde_json::to_vec(&request).unwrap(),
8278 )
8279 .unwrap();
8280 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8281 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8282 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8283 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8284 panic!("expected supervisor.terminals response");
8285 };
8286 assert_eq!(terminals.entries.len(), 2);
8287 assert_eq!(terminals.dropped, 0);
8288
8289 let mut rendered = serde_json::to_value(response).unwrap();
8290 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
8293 for (index, entry) in rendered["entries"]
8294 .as_array_mut()
8295 .expect("terminal response entries array")
8296 .iter_mut()
8297 .enumerate()
8298 {
8299 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
8300 }
8301
8302 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8303 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
8304 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
8305 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8306 std::fs::write(&golden_path, &serialized).unwrap();
8307 }
8308 let expected: Value =
8309 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
8310 assert_eq!(rendered, expected);
8311 }
8312
8313 #[test]
8314 fn hello_registers_manifest_and_returns_ack() {
8315 let registry = Arc::new(Registry::default());
8316 let handler = ControlHandler::new(Arc::clone(®istry));
8317 let conn = ConnectionId::new(1);
8318
8319 let responses = handler
8320 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
8321 .unwrap();
8322
8323 assert_eq!(responses.len(), 1);
8324 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8325 assert_eq!(responses[0].header.channel, 0);
8326 assert_eq!(responses[0].header.corr, 7);
8327 let ack = parse_ack(&responses[0]);
8328 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8329 assert!(ack
8330 .subc_capabilities
8331 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
8332 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
8333 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
8334 assert!(ack
8335 .subc_ops
8336 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
8337 assert!(ack
8338 .subc_ops
8339 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
8340
8341 let registration = registry.get_module("aft").unwrap().unwrap();
8342 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
8343 assert_eq!(registration.state, ChannelState::Active);
8344 assert_eq!(registration.connection_id, conn);
8345 assert_eq!(registration.control_ops, module_baseline_control_ops());
8346 }
8347
8348 #[test]
8349 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
8350 let invalid_identifiers = [
8351 ("case_change", "credentials-Provider/v1"),
8352 ("leading_zero", "credentials-provider/v01"),
8353 ("trailing_hyphen", "credentials-provider-/v1"),
8354 ("consecutive_hyphens", "credentials--provider/v1"),
8355 ("uppercase", "Credentials-provider/v1"),
8356 ("missing_v", "credentials-provider/1"),
8357 ("whitespace", "credentials provider/v1"),
8358 ("zero_version", "credentials-provider/v0"),
8359 ("out_of_range_version", "credentials-provider/v4294967296"),
8360 (
8361 "overlength_name",
8362 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
8363 ),
8364 ];
8365 let mut cases = invalid_identifiers
8366 .into_iter()
8367 .map(|(name, identifier)| {
8368 (
8369 format!("identifier_{name}"),
8370 "capabilities.provides[0]".to_string(),
8371 identifier.to_string(),
8372 json!({ "provides": [identifier] }),
8373 None,
8374 )
8375 })
8376 .collect::<Vec<_>>();
8377 cases.extend([
8378 (
8379 "unknown_need".to_string(),
8380 "capabilities.requires[0].need".to_string(),
8381 "deferred".to_string(),
8382 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
8383 None,
8384 ),
8385 (
8386 "duplicate_provides".to_string(),
8387 "capabilities.provides[1]".to_string(),
8388 "credentials-provider/v1".to_string(),
8389 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
8390 None,
8391 ),
8392 (
8393 "duplicate_must_never_reach".to_string(),
8394 "capabilities.must_never_reach[1]".to_string(),
8395 "credentials-provider/v1".to_string(),
8396 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
8397 None,
8398 ),
8399 (
8400 "duplicate_requires_same_need".to_string(),
8401 "capabilities.requires[1]".to_string(),
8402 "credentials-provider/v1".to_string(),
8403 json!({ "requires": [
8404 { "capability": "credentials-provider/v1", "need": "required" },
8405 { "capability": "credentials-provider/v1", "need": "required" }
8406 ] }),
8407 None,
8408 ),
8409 (
8410 "duplicate_requires_conflicting_need".to_string(),
8411 "capabilities.requires[1]".to_string(),
8412 "credentials-provider/v1".to_string(),
8413 json!({ "requires": [
8414 { "capability": "credentials-provider/v1", "need": "required" },
8415 { "capability": "credentials-provider/v1", "need": "optional" }
8416 ] }),
8417 None,
8418 ),
8419 (
8420 "capabilities_root_pointer".to_string(),
8421 "runtime_computed[0]".to_string(),
8422 "/capabilities".to_string(),
8423 json!({}),
8424 Some(json!(["/capabilities"])),
8425 ),
8426 (
8427 "capabilities_descendant_pointer".to_string(),
8428 "runtime_computed[0]".to_string(),
8429 "/capabilities/provides".to_string(),
8430 json!({}),
8431 Some(json!(["/capabilities/provides"])),
8432 ),
8433 (
8434 "malformed_pointer_without_leading_slash".to_string(),
8435 "runtime_computed[0]".to_string(),
8436 "capabilities".to_string(),
8437 json!({}),
8438 Some(json!(["capabilities"])),
8439 ),
8440 (
8441 "malformed_pointer_escape".to_string(),
8442 "runtime_computed[0]".to_string(),
8443 "/roles/~2/tools".to_string(),
8444 json!({}),
8445 Some(json!(["/roles/~2/tools"])),
8446 ),
8447 (
8448 "unknown_capabilities_field".to_string(),
8449 "capabilities.future".to_string(),
8450 "<array>".to_string(),
8451 json!({ "future": [] }),
8452 None,
8453 ),
8454 ]);
8455
8456 for (index, (name, field, value, capabilities, runtime_computed)) in
8457 cases.into_iter().enumerate()
8458 {
8459 let registry = Arc::new(Registry::default());
8460 let handler = ControlHandler::new(Arc::clone(®istry));
8461 let response = handler
8462 .handle_control(
8463 ConnectionId::new((index + 1) as u64),
8464 capability_grammar_hello_frame(
8465 capabilities,
8466 runtime_computed,
8467 index as u64 + 1,
8468 ),
8469 )
8470 .expect("invalid HELLO returns a refusal");
8471
8472 assert_eq!(response.len(), 1, "{name} must emit one refusal");
8473 let error = parse_error(&response[0]);
8474 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
8475 let message = error["message"]
8476 .as_str()
8477 .expect("error message is a string");
8478 assert!(
8479 message.contains(&field),
8480 "{name}: field missing from {message}"
8481 );
8482 assert!(
8483 message.contains(&value),
8484 "{name}: value missing from {message}"
8485 );
8486 assert_eq!(
8487 registry
8488 .active_registration_count()
8489 .expect("registry reads"),
8490 0,
8491 "{name}: refused HELLO must not create a catalog entry"
8492 );
8493 }
8494 }
8495
8496 #[test]
8497 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
8498 let registry = Arc::new(Registry::default());
8499 let handler = ControlHandler::new(Arc::clone(®istry));
8500 let capabilities = json!({
8501 "provides": ["credentials-provider/v1"],
8502 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
8503 "must_never_reach": ["federation-transport/v1"]
8504 });
8505 let response = handler
8506 .handle_control(
8507 ConnectionId::new(99),
8508 capability_grammar_hello_frame(
8509 capabilities.clone(),
8510 Some(json!(["/roles/0/tools"])),
8511 99,
8512 ),
8513 )
8514 .expect("valid HELLO registers");
8515 assert_eq!(response[0].header.ty, FrameType::HelloAck);
8516
8517 let request = Frame::build(
8518 FrameType::Request,
8519 control_flags(),
8520 0,
8521 0,
8522 100,
8523 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8524 .expect("catalog request serializes"),
8525 )
8526 .expect("catalog request frame builds");
8527 let response = handler
8528 .handle_catalog_list(request, None)
8529 .expect("catalog list succeeds");
8530 let ClientControlResponse::CatalogList { modules, .. } =
8531 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
8532 else {
8533 panic!("catalog request must return catalog.list");
8534 };
8535 assert_eq!(modules.len(), 1);
8536 assert_eq!(
8537 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
8538 capabilities
8539 );
8540 }
8541
8542 #[test]
8543 fn catalog_list_mirrors_management_operation_description() {
8544 let registry = Arc::new(Registry::default());
8545 let handler = ControlHandler::new(Arc::clone(®istry));
8546 let description = "List managed records and return their identifiers and metadata.";
8547 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
8548 manifest.provides = vec![ProviderRole::ManagementSurface {
8549 operations: vec![ManagementOperation {
8550 name: "records.list".to_string(),
8551 kind: ManagementOperationKind::Query,
8552 description: Some(description.to_string()),
8553 }],
8554 config_schema: json!({"type": "object"}),
8555 observability: vec![ObservabilitySurface {
8556 name: "records.stats".to_string(),
8557 kind: ObservabilityKind::Snapshot,
8558 }],
8559 identity_scope: vec![IdentityScope::Project],
8560 concurrency: Concurrency::ModuleManaged,
8561 }];
8562 registry
8563 .register_with_control_ops(
8564 manifest,
8565 PROTOCOL_VERSION,
8566 ConnectionId::new(99),
8567 Vec::new(),
8568 )
8569 .expect("described management manifest registers");
8570
8571 let request = Frame::build(
8572 FrameType::Request,
8573 control_flags(),
8574 0,
8575 0,
8576 100,
8577 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8578 .expect("catalog request serializes"),
8579 )
8580 .expect("catalog request frame builds");
8581 let response = handler
8582 .handle_catalog_list(request, None)
8583 .expect("catalog list succeeds");
8584 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8585 assert_eq!(
8586 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
8587 "catalog.list must preserve the declared operation description verbatim"
8588 );
8589 }
8590
8591 #[test]
8592 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
8593 let registry = Arc::new(Registry::default());
8594 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8595 [("vault".to_string(), true), ("squatter".to_string(), true)],
8596 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8597 );
8598 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
8599 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8600 provides: vec!["credentials-provider/v1".to_string()],
8601 requires: Vec::new(),
8602 must_never_reach: Vec::new(),
8603 });
8604 let frame = Frame::build(
8605 FrameType::Hello,
8606 control_flags(),
8607 0,
8608 0,
8609 77,
8610 serde_json::to_vec(&ModuleHelloBody {
8611 manifest: squatter,
8612 protocol_ver: PROTOCOL_VERSION,
8613 control_ops: None,
8614 launch_nonce: None,
8615 })
8616 .expect("HELLO serializes"),
8617 )
8618 .expect("HELLO frame builds");
8619 let response = handler
8620 .handle_control(ConnectionId::new(77), frame)
8621 .expect("reserved claim receives a typed refusal");
8622 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8623 assert_eq!(
8624 registry
8625 .active_registration_count()
8626 .expect("registry reads"),
8627 0,
8628 "a reserved capability refusal must not leave a catalog entry"
8629 );
8630 }
8631
8632 #[test]
8633 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
8634 for settlement in ["timeout", "inconclusive", "drop"] {
8635 let breakers = RouteBindBreakers::default();
8636 let RouteBindAdmission::Admitted {
8637 guard: mut old,
8638 probe: false,
8639 } = breakers.admit("prov")
8640 else {
8641 panic!("ordinary relay admitted")
8642 };
8643 let RouteBindAdmission::Admitted {
8644 guard: mut opener, ..
8645 } = breakers.admit("prov")
8646 else {
8647 panic!("second relay admitted")
8648 };
8649 assert!(
8650 !opener
8651 .record_timeout(1, Duration::ZERO)
8652 .unwrap()
8653 .reopened_after_probe
8654 );
8655 let RouteBindAdmission::Admitted {
8656 guard: mut probe,
8657 probe: true,
8658 } = breakers.admit("prov")
8659 else {
8660 panic!("one cooldown probe admitted")
8661 };
8662 match settlement {
8663 "timeout" => assert!(
8664 !old.record_timeout(1, Duration::ZERO)
8665 .unwrap()
8666 .reopened_after_probe
8667 ),
8668 "inconclusive" => old.record_inconclusive(),
8669 "drop" => drop(old),
8670 _ => unreachable!(),
8671 }
8672 assert!(
8673 matches!(
8674 breakers.admit("prov"),
8675 RouteBindAdmission::Refused {
8676 probe_in_flight: true,
8677 ..
8678 }
8679 ),
8680 "{settlement} of a pre-open relay cannot release the real probe"
8681 );
8682 assert!(
8683 probe
8684 .record_timeout(1, Duration::ZERO)
8685 .unwrap()
8686 .reopened_after_probe
8687 );
8688 assert!(matches!(
8689 breakers.admit("prov"),
8690 RouteBindAdmission::Admitted { probe: true, .. }
8691 ));
8692 }
8693 let breakers = RouteBindBreakers::default();
8694 let admit = || match breakers.admit("prov") {
8695 RouteBindAdmission::Admitted { guard, .. } => guard,
8696 _ => panic!("relay admitted"),
8697 };
8698 admit().record_timeout(1, Duration::ZERO);
8699 let mut old_probe = admit();
8700 breakers.reset_for_new_module_connection("prov");
8701 admit().record_timeout(1, Duration::ZERO);
8702 let _new_probe = admit();
8703 old_probe.record_inconclusive();
8704 assert!(matches!(
8705 breakers.admit("prov"),
8706 RouteBindAdmission::Refused {
8707 probe_in_flight: true,
8708 ..
8709 }
8710 ));
8711 }
8712
8713 #[tokio::test]
8714 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
8715 for candidate in [false, true] {
8716 let registry = Arc::new(Registry::default());
8717 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8718 [("vault".to_string(), true), ("squatter".to_string(), true)],
8719 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8720 );
8721 let conn = ConnectionId::new(77);
8722 let (ctx, mut rx) = route_ctx(conn);
8723 let initial = capability_manifest("squatter", &[], &[]);
8724 if candidate {
8725 registry
8726 .register_candidate_with_control_ops(
8727 initial.clone(),
8728 PROTOCOL_VERSION,
8729 conn,
8730 module_baseline_control_ops(),
8731 )
8732 .unwrap();
8733 handler
8734 .forwarding
8735 .register_candidate_module_connection(
8736 conn,
8737 "squatter".to_string(),
8738 PROTOCOL_VERSION,
8739 manifest_concurrency(&initial),
8740 ctx.egress.clone(),
8741 )
8742 .unwrap();
8743 } else {
8744 hello_via_sink(
8745 &handler,
8746 &ctx,
8747 &mut rx,
8748 hello_frame_with_manifest(initial.clone(), 1),
8749 )
8750 .await;
8751 }
8752 let response = handler
8753 .handle_control_frame(
8754 &ctx,
8755 catalog_update_with_capabilities_frame(
8756 2,
8757 capability_manifest("squatter", &["credentials-provider/v1"], &[])
8758 .capabilities
8759 .unwrap(),
8760 ),
8761 )
8762 .await
8763 .unwrap();
8764 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8765 assert_eq!(
8766 registry
8767 .get_module_by_connection(conn)
8768 .unwrap()
8769 .unwrap()
8770 .manifest,
8771 initial
8772 );
8773 }
8774 }
8775
8776 #[test]
8777 fn server_describe_surfaces_required_capability_verdict_fields() {
8778 let registry = Arc::new(Registry::default());
8779 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8780 [
8781 ("consumer".to_string(), true),
8782 ("provider".to_string(), false),
8783 ],
8784 BTreeMap::new(),
8785 );
8786 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
8787 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8788 provides: Vec::new(),
8789 requires: vec![subc_protocol::manifest::CapabilityRequirement {
8790 capability: "credentials-provider/v1".to_string(),
8791 need: subc_protocol::manifest::CapabilityNeed::Required,
8792 }],
8793 must_never_reach: Vec::new(),
8794 });
8795 let hello = Frame::build(
8796 FrameType::Hello,
8797 control_flags(),
8798 0,
8799 0,
8800 78,
8801 serde_json::to_vec(&ModuleHelloBody {
8802 manifest: consumer,
8803 protocol_ver: PROTOCOL_VERSION,
8804 control_ops: None,
8805 launch_nonce: None,
8806 })
8807 .expect("HELLO serializes"),
8808 )
8809 .expect("HELLO frame builds");
8810 handler
8811 .handle_control(ConnectionId::new(78), hello)
8812 .expect("consumer registers");
8813 let describe = Frame::build(
8814 FrameType::Request,
8815 control_flags(),
8816 0,
8817 0,
8818 79,
8819 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
8820 .expect("request serializes"),
8821 )
8822 .expect("describe frame builds");
8823 let response = handler
8824 .handle_server_describe(describe)
8825 .expect("server.describe succeeds");
8826 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
8827 let requirement = &rendered["capability_requirements"][0];
8828 assert_eq!(requirement["consumer"], "consumer");
8829 assert_eq!(requirement["verdict"], "never_provided");
8830 assert_eq!(requirement["episode_seq"], 1);
8831 assert_eq!(requirement["config_satisfiable"], false);
8832 assert_eq!(requirement["runtime_available"], false);
8833 assert!(requirement["detail"]
8834 .as_str()
8835 .expect("detail string")
8836 .contains("credentials-provider/v1"));
8837 }
8838
8839 #[test]
8840 fn catalog_list_omits_capabilities_for_legacy_manifest() {
8841 let registry = Arc::new(Registry::default());
8842 let handler = ControlHandler::new(Arc::clone(®istry));
8843 let hello = handler
8844 .handle_control(
8845 ConnectionId::new(101),
8846 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
8847 )
8848 .expect("legacy HELLO registers");
8849 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
8850
8851 let request = Frame::build(
8852 FrameType::Request,
8853 control_flags(),
8854 0,
8855 0,
8856 102,
8857 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8858 .expect("catalog request serializes"),
8859 )
8860 .expect("catalog request frame builds");
8861 let response = handler
8862 .handle_catalog_list(request, None)
8863 .expect("catalog list succeeds");
8864 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8865 assert!(
8866 body["modules"][0].get("capabilities").is_none(),
8867 "legacy manifest must retain an absent capabilities field on catalog.list"
8868 );
8869 }
8870
8871 #[test]
8872 fn hello_ack_omits_storage_when_no_storage_config() {
8873 let registry = Arc::new(Registry::default());
8874 let handler = ControlHandler::new(Arc::clone(®istry));
8875 let responses = handler
8876 .handle_control(
8877 ConnectionId::new(1),
8878 hello_frame("aft", PROTOCOL_VERSION, 7),
8879 )
8880 .unwrap();
8881 let ack = parse_ack(&responses[0]);
8882 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
8883 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
8884 }
8885
8886 #[tokio::test]
8887 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
8888 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
8889 let registry = Arc::new(Registry::default());
8890 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
8891 let responses = handler
8892 .handle_control(
8893 ConnectionId::new(1),
8894 hello_frame("aft", PROTOCOL_VERSION, 7),
8895 )
8896 .unwrap();
8897 let ack = parse_ack(&responses[0]);
8898 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
8899
8900 let described = handler
8901 .handle_control_frame(
8902 &route_ctx(ConnectionId::new(2)).0,
8903 Frame::build(
8904 FrameType::Request,
8905 control_flags(),
8906 0,
8907 0,
8908 9,
8909 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8910 )
8911 .unwrap(),
8912 )
8913 .await
8914 .unwrap();
8915 let ClientControlResponse::ServerDescribe { machine_id, .. } =
8916 serde_json::from_slice(&described[0].body).unwrap()
8917 else {
8918 panic!("server.describe answered with another shape");
8919 };
8920 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
8921 }
8922
8923 #[test]
8924 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
8925 let registry = Arc::new(Registry::default());
8928 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
8929 crate::daemon_config::StorageConfig::Sqlite {
8930 data_home: std::path::PathBuf::from("/data"),
8931 },
8932 ));
8933
8934 let responses = handler
8935 .handle_control(
8936 ConnectionId::new(1),
8937 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
8938 )
8939 .unwrap();
8940 let ack = parse_ack(&responses[0]);
8941 assert_eq!(
8942 ack.storage,
8943 Some(serde_json::json!({
8944 "module_id": "alfonso-routing",
8945 "storage_namespace": "default",
8946 "isolation": { "kind": "module" },
8947 "backend": {
8948 "backend": "sqlite",
8949 "path": "/data/cortexkit/alfonso-routing/store.db"
8950 }
8951 })),
8952 "the delivered descriptor is the module's own sqlite store path"
8953 );
8954 }
8955
8956 #[test]
8957 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
8958 let registry = Arc::new(Registry::default());
8959 let handler = ControlHandler::new(Arc::clone(®istry));
8960 let conn = ConnectionId::new(1);
8961 let responses = handler
8962 .handle_control(
8963 conn,
8964 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8965 )
8966 .unwrap();
8967 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8968 let registration = registry.get_module("aft").unwrap().unwrap();
8969 assert_eq!(registration.control_ops, module_baseline_control_ops());
8970
8971 let frame =
8972 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
8973 assert!(handler
8974 .guard_module_control_op(&frame, "aft", "route.bind")
8975 .unwrap()
8976 .is_none());
8977 let error = handler
8978 .guard_module_control_op(&frame, "aft", "test.synthetic")
8979 .unwrap()
8980 .expect("synthetic ungranted op should be rejected");
8981 assert_eq!(error.header.ty, FrameType::Error);
8982 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
8983 }
8984
8985 #[test]
8986 fn hello_control_ops_some_adds_optional_grants() {
8987 let registry = Arc::new(Registry::default());
8988 let handler = ControlHandler::new(Arc::clone(®istry));
8989 handler
8990 .handle_control(
8991 ConnectionId::new(1),
8992 hello_frame_with_control_ops(
8993 "aft",
8994 PROTOCOL_VERSION,
8995 7,
8996 Some(vec![
8997 "future.synthetic".to_string(),
8998 "route.bind".to_string(),
8999 ]),
9000 ),
9001 )
9002 .unwrap();
9003 let registration = registry.get_module("aft").unwrap().unwrap();
9004 assert_eq!(
9005 registration.control_ops,
9006 vec![
9007 "route.bind".to_string(),
9008 "route.status".to_string(),
9009 "future.synthetic".to_string(),
9010 ]
9011 );
9012 let frame =
9013 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
9014 assert!(handler
9015 .guard_module_control_op(&frame, "aft", "future.synthetic")
9016 .unwrap()
9017 .is_none());
9018 }
9019
9020 #[tokio::test]
9021 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
9022 let registry = Arc::new(Registry::default());
9023 let forwarding = Arc::new(ForwardingTable::default());
9024 let handler =
9025 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9026 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
9027 hello_via_sink(
9028 &handler,
9029 &module_ctx,
9030 &mut module_rx,
9031 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9032 )
9033 .await;
9034
9035 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
9036 let responses = handler
9037 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
9038 .await
9039 .unwrap();
9040 assert_eq!(responses.len(), 1);
9041 assert_eq!(responses[0].header.ty, FrameType::Error);
9042 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
9043 assert!(module_rx.try_recv().is_err());
9044 }
9045
9046 #[tokio::test]
9047 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
9048 let registry = Arc::new(Registry::default());
9049 let forwarding = Arc::new(ForwardingTable::default());
9050 let handler =
9051 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9052 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
9053 hello_via_sink(
9054 &handler,
9055 &module_ctx,
9056 &mut module_rx,
9057 hello_frame_with_control_ops(
9058 "aft",
9059 PROTOCOL_VERSION,
9060 7,
9061 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9062 ),
9063 )
9064 .await;
9065
9066 let project_root = unique_project_root("demux");
9067 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
9068 let route_handler = handler.clone();
9069 let route_task = tokio::spawn(async move {
9070 route_handler
9071 .handle_control_frame(
9072 &route_client_ctx,
9073 route_open_frame(100, "aft", project_root),
9074 )
9075 .await
9076 .unwrap()
9077 });
9078 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9079 .await
9080 .unwrap()
9081 .unwrap();
9082 assert!(matches!(
9083 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
9084 ModuleControlRequest::RouteBind { .. }
9085 ));
9086
9087 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
9088 let health_handler = handler.clone();
9089 let health_task = tokio::spawn(async move {
9090 health_handler
9091 .handle_control_frame(
9092 &health_client_ctx,
9093 supervisor_health_probe_frame(101, "aft"),
9094 )
9095 .await
9096 .unwrap()
9097 });
9098 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9099 .await
9100 .unwrap()
9101 .unwrap();
9102 assert_eq!(
9103 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9104 ModuleControlRequest::HealthCheck {}
9105 );
9106
9107 handler
9108 .handle_control_frame(
9109 &module_ctx,
9110 health_response(health_frame.header.corr, HealthStatus::Degraded),
9111 )
9112 .await
9113 .unwrap();
9114 let health_response = health_task.await.unwrap();
9115 assert_eq!(health_response.len(), 1);
9116 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9117 ClientControlResponse::SupervisorHealthProbe {
9118 module_id,
9119 status,
9120 detail,
9121 metrics,
9122 } => {
9123 assert_eq!(module_id, "aft");
9124 assert_eq!(status, HealthStatus::Degraded);
9125 assert_eq!(detail.as_deref(), Some("warming"));
9126 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
9127 }
9128 other => panic!("unexpected health response: {other:?}"),
9129 }
9130
9131 handler
9132 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9133 .await
9134 .unwrap();
9135 let route_response = route_task.await.unwrap();
9136 assert!(route_response.is_empty());
9137 let published = route_client_rx.recv().await.unwrap();
9138 assert!(matches!(
9139 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9140 ClientControlResponse::RouteOpen { .. }
9141 ));
9142 }
9143
9144 async fn relay_route_open(
9149 handler: &ControlHandler,
9150 client_connection: ConnectionId,
9151 client_egress: &FrameSink,
9152 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9153 corr: u64,
9154 module_id: &str,
9155 project_root_label: &str,
9156 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
9157 let ctx = RouteCtx {
9158 connection_id: client_connection,
9159 egress: client_egress.clone(),
9160 };
9161 let handler = handler.clone();
9162 let project_root = unique_project_root(project_root_label);
9163 let module_id = module_id.to_string();
9164 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
9165 let task = tokio::spawn(async move {
9166 let _guard = tracing::dispatcher::set_default(&dispatch);
9167 handler
9168 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
9169 .await
9170 .unwrap()
9171 });
9172 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
9173 .await
9174 .expect("module receives the relayed route.bind")
9175 .expect("module egress is open");
9176 (task, bind.frame)
9177 }
9178
9179 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
9180 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
9181 ModuleControlRequest::RouteBind {
9182 route_channel,
9183 epoch,
9184 ..
9185 } => (route_channel, epoch),
9186 other => panic!("expected a route.bind request, got {other:?}"),
9187 }
9188 }
9189
9190 fn published_route(frame: &Frame) -> (u16, u32) {
9191 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
9192 ClientControlResponse::RouteOpen {
9193 route_channel,
9194 route_epoch,
9195 } => (route_channel, route_epoch),
9196 other => panic!("expected a route.open response, got {other:?}"),
9197 }
9198 }
9199
9200 #[tokio::test]
9220 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
9221 let registry = Arc::new(Registry::default());
9222 let forwarding = Arc::new(ForwardingTable::default());
9223 let handler =
9224 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9225
9226 let module_connection = ConnectionId::new(30);
9227 let (module_ctx, mut module_rx) = route_ctx(module_connection);
9228 hello_via_sink(
9229 &handler,
9230 &module_ctx,
9231 &mut module_rx,
9232 hello_frame("aft", PROTOCOL_VERSION, 7),
9233 )
9234 .await;
9235
9236 let dying_client = ConnectionId::new(31);
9237 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
9238
9239 let (first_task, first_bind) = relay_route_open(
9242 &handler,
9243 dying_client,
9244 &dying_ctx.egress,
9245 &mut module_rx,
9246 100,
9247 "aft",
9248 "closing-first",
9249 )
9250 .await;
9251 handler
9252 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
9253 .await
9254 .unwrap();
9255 assert!(first_task.await.unwrap().is_empty());
9256 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
9257
9258 let (second_task, second_bind) = relay_route_open(
9260 &handler,
9261 dying_client,
9262 &dying_ctx.egress,
9263 &mut module_rx,
9264 101,
9265 "aft",
9266 "closing-second",
9267 )
9268 .await;
9269 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
9270
9271 assert!(forwarding
9274 .escalate_client_delivery_failure(
9275 dying_client,
9276 first_channel,
9277 first_epoch,
9278 CloseReason::new(
9279 "module_to_client_delivery_failed",
9280 "client egress refused a module frame",
9281 ),
9282 crate::forwarding::UndeliveredFrame {
9283 module_id: None,
9284 sink: &dying_ctx.egress,
9285 },
9286 )
9287 .unwrap());
9288 assert!(!dying_ctx.egress.is_closed());
9289
9290 let ack = handler
9292 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
9293 .await;
9294 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
9295 if module_loop_error.is_some() {
9296 handler.cleanup_connection(module_connection).unwrap();
9300 }
9301 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9305 .await
9306 .ok()
9307 .flatten();
9308
9309 assert!(
9311 registry
9312 .get_module_by_connection(module_connection)
9313 .unwrap()
9314 .is_some(),
9315 "one client's closing connection ended the shared module connection: \
9316 {module_loop_error:?}"
9317 );
9318 let cotenant = ConnectionId::new(32);
9320 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
9321 let (cotenant_task, cotenant_bind) = relay_route_open(
9322 &handler,
9323 cotenant,
9324 &cotenant_ctx.egress,
9325 &mut module_rx,
9326 102,
9327 "aft",
9328 "closing-cotenant",
9329 )
9330 .await;
9331 handler
9332 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
9333 .await
9334 .unwrap();
9335 assert!(cotenant_task.await.unwrap().is_empty());
9336 let (cotenant_channel, cotenant_epoch) =
9337 published_route(&cotenant_rx.recv().await.unwrap());
9338 assert!(matches!(
9339 forwarding
9340 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
9341 .unwrap(),
9342 DataRoute::Client(DataRouteState::Bound(_))
9343 ));
9344
9345 let goodbye = post_ack_module_frame
9348 .expect("module receives a GOODBYE for the abandoned route channel");
9349 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9350 assert_eq!(goodbye.header.channel, abandoned_channel);
9351 assert_eq!(goodbye.header.epoch, abandoned_epoch);
9352
9353 assert!(dying_rx.try_recv().is_err());
9357 let second_response = second_task.await.unwrap();
9358 assert_eq!(second_response.len(), 1);
9359 assert_eq!(
9360 parse_error(&second_response[0])["code"],
9361 "target_unavailable"
9362 );
9363 }
9364
9365 async fn catalog_snapshot(
9368 handler: &ControlHandler,
9369 client_ctx: &RouteCtx,
9370 corr: u64,
9371 ) -> (u64, Vec<(String, bool)>) {
9372 let body =
9373 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
9374 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
9375 let reply = handler
9376 .handle_control_frame(client_ctx, request)
9377 .await
9378 .unwrap()
9379 .pop()
9380 .unwrap();
9381 let ClientControlResponse::CatalogList {
9382 generation,
9383 modules,
9384 ..
9385 } = serde_json::from_slice(&reply.body).unwrap()
9386 else {
9387 panic!("not a catalog.list reply");
9388 };
9389 (
9390 generation,
9391 modules
9392 .into_iter()
9393 .map(|entry| (entry.module_id, entry.ready))
9394 .collect(),
9395 )
9396 }
9397
9398 fn hello_held_before_forwarding() -> (
9404 ControlHandler,
9405 Arc<std::sync::Barrier>,
9406 std::thread::JoinHandle<Vec<Frame>>,
9407 RouteCtx,
9408 mpsc::Receiver<crate::router::OutboundFrame>,
9409 ) {
9410 let hold = Arc::new(std::sync::Barrier::new(2));
9411 let handler = ControlHandler::with_forwarding(
9412 Arc::new(Registry::default()),
9413 Arc::new(ForwardingTable::default()),
9414 )
9415 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
9416 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9417 let hello = {
9418 let handler = handler.clone();
9419 let module_ctx = module_ctx.clone();
9420 std::thread::spawn(move || {
9421 handler
9422 .handle_hello(
9423 module_ctx.connection_id,
9424 Some(module_ctx.egress.clone()),
9425 hello_frame("aft", PROTOCOL_VERSION, 7),
9426 )
9427 .unwrap()
9428 })
9429 };
9430 (handler, hold, hello, module_ctx, module_rx)
9431 }
9432
9433 #[tokio::test]
9442 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
9443 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
9444 let client = ConnectionId::new(41);
9445 let (client_ctx, mut client_rx) = route_ctx(client);
9446
9447 hold.wait();
9448 assert!(handler.registry.get_module("aft").unwrap().is_some());
9450 assert!(!handler
9451 .forwarding
9452 .has_live_module_connection("aft")
9453 .unwrap());
9454 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
9455 assert!(
9456 listed.is_empty(),
9457 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
9458 );
9459 let refused = handler
9460 .handle_control_frame(
9461 &client_ctx,
9462 route_open_frame(301, "aft", unique_project_root("listed-window")),
9463 )
9464 .await
9465 .unwrap();
9466 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
9467 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
9468 let duplicate = handler
9469 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
9470 .await
9471 .unwrap();
9472 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9473
9474 hold.wait();
9475 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9476 let ack = module_rx.try_recv().unwrap().frame;
9477 assert_eq!(ack.header.ty, FrameType::HelloAck);
9478 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9479 assert_eq!(listed, vec![("aft".to_string(), true)]);
9480
9481 let (open_task, bind) = relay_route_open(
9482 &handler,
9483 client,
9484 &client_ctx.egress,
9485 &mut module_rx,
9486 303,
9487 "aft",
9488 "listed-after",
9489 )
9490 .await;
9491 assert!(handler
9492 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9493 .await
9494 .unwrap()
9495 .is_empty());
9496 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
9497 published_route(&client_rx.try_recv().unwrap().frame);
9498 }
9499
9500 #[tokio::test]
9507 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
9508 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
9509 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9510 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9511
9512 hold.wait();
9513 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
9514 assert!(listed.is_empty());
9515 assert_eq!(
9516 during, before,
9517 "the generation moved while the module was not listed"
9518 );
9519
9520 hold.wait();
9521 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9522 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9523 assert_eq!(listed, vec![("aft".to_string(), true)]);
9524 assert_eq!(
9525 after,
9526 before + 1,
9527 "the module appeared without exactly one generation bump"
9528 );
9529 }
9530
9531 #[tokio::test]
9536 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
9537 let registry = Arc::new(Registry::default());
9538 let handler = ControlHandler::with_forwarding(
9539 Arc::clone(®istry),
9540 Arc::new(ForwardingTable::default()),
9541 );
9542 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9543 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9544 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9545 drop(module_rx);
9546
9547 let replies = handler
9548 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
9549 .await
9550 .unwrap();
9551 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
9552 assert!(registry.get_module("aft").unwrap().is_none());
9553 assert!(registry
9554 .get_module_by_connection(module_ctx.connection_id)
9555 .unwrap()
9556 .is_none());
9557 assert_eq!(
9558 catalog_snapshot(&handler, &client_ctx, 301).await,
9559 (before, Vec::new())
9560 );
9561 }
9562
9563 async fn route_open_held_after_bind_send() -> (
9567 ControlHandler,
9568 Arc<tokio::sync::Barrier>,
9569 RouteCtx,
9570 mpsc::Receiver<crate::router::OutboundFrame>,
9571 ) {
9572 let hold = Arc::new(tokio::sync::Barrier::new(2));
9573 let handler = ControlHandler::with_forwarding(
9574 Arc::new(Registry::default()),
9575 Arc::new(ForwardingTable::default()),
9576 )
9577 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
9578 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
9579 hello_via_sink(
9580 &handler,
9581 &module_ctx,
9582 &mut module_rx,
9583 hello_frame("aft", PROTOCOL_VERSION, 7),
9584 )
9585 .await;
9586 (handler, hold, module_ctx, module_rx)
9587 }
9588
9589 #[tokio::test]
9596 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
9597 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9598 let client = ConnectionId::new(41);
9599 let (client_ctx, mut client_rx) = route_ctx(client);
9600
9601 let (open_task, bind) = relay_route_open(
9602 &handler,
9603 client,
9604 &client_ctx.egress,
9605 &mut module_rx,
9606 200,
9607 "aft",
9608 "acked-before-mark",
9609 )
9610 .await;
9611 let (module_channel, module_epoch) = route_bind_channel(&bind);
9612 hold.wait().await;
9614
9615 assert!(handler
9617 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9618 .await
9619 .unwrap()
9620 .is_empty());
9621 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
9622
9623 hold.wait().await;
9624 assert!(
9625 open_task.await.unwrap().is_empty(),
9626 "route.open answers accepted"
9627 );
9628
9629 match module_rx.try_recv() {
9632 Err(mpsc::error::TryRecvError::Empty) => {}
9633 Ok(outbound) => panic!(
9634 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
9635 outbound.frame.header.ty,
9636 outbound.frame.header.channel,
9637 outbound.frame.header.epoch
9638 ),
9639 Err(err) => panic!("module egress closed: {err}"),
9640 }
9641
9642 let router = Router::with_control_handler(Arc::new(handler.clone()));
9645 let request = Frame::build(
9646 FrameType::Request,
9647 Flags::new(true, Priority::Interactive, false),
9648 client_channel,
9649 client_epoch,
9650 201,
9651 b"first request".to_vec(),
9652 )
9653 .unwrap();
9654 router
9655 .route_for_connection(&client_ctx, request)
9656 .await
9657 .unwrap();
9658 let delivered = module_rx.try_recv().unwrap().frame;
9659 assert_eq!(delivered.header.ty, FrameType::Request);
9660 assert_eq!(
9661 (delivered.header.channel, delivered.header.epoch),
9662 (module_channel, module_epoch)
9663 );
9664 assert_eq!(delivered.body, b"first request");
9665 }
9666
9667 #[tokio::test]
9673 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
9674 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9675 let client = ConnectionId::new(42);
9676 let (client_ctx, _client_rx) = route_ctx(client);
9677
9678 let (open_task, bind) = relay_route_open(
9679 &handler,
9680 client,
9681 &client_ctx.egress,
9682 &mut module_rx,
9683 300,
9684 "aft",
9685 "released-before-mark",
9686 )
9687 .await;
9688 let (module_channel, module_epoch) = route_bind_channel(&bind);
9689 hold.wait().await;
9690
9691 handler.cleanup_connection(client).unwrap();
9692 assert!(
9693 module_rx.try_recv().is_err(),
9694 "client teardown must not GOODBYE a bind route.open is still queueing"
9695 );
9696
9697 hold.wait().await;
9698 let response = open_task.await.unwrap();
9699 assert_eq!(response.len(), 1);
9700 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9701
9702 let goodbye = module_rx
9703 .try_recv()
9704 .expect("module is told to drop the binding the daemon abandoned")
9705 .frame;
9706 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9707 assert_eq!(
9708 (goodbye.header.channel, goodbye.header.epoch),
9709 (module_channel, module_epoch)
9710 );
9711 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9712 }
9713
9714 #[tokio::test]
9719 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
9720 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9721 let client = ConnectionId::new(43);
9722 let (client_ctx, client_rx) = route_ctx(client);
9723
9724 let (open_task, bind) = relay_route_open(
9725 &handler,
9726 client,
9727 &client_ctx.egress,
9728 &mut module_rx,
9729 400,
9730 "aft",
9731 "gone-client-before-mark",
9732 )
9733 .await;
9734 let (module_channel, module_epoch) = route_bind_channel(&bind);
9735 hold.wait().await;
9736
9737 drop(client_rx);
9738 assert!(client_ctx.egress.is_closed());
9739 handler
9740 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9741 .await
9742 .unwrap();
9743
9744 hold.wait().await;
9745 let response = open_task.await.unwrap();
9746 assert_eq!(response.len(), 1);
9747 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9748
9749 let goodbye = module_rx
9750 .try_recv()
9751 .expect("module is told to drop the binding it acked")
9752 .frame;
9753 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9754 assert_eq!(
9755 (goodbye.header.channel, goodbye.header.epoch),
9756 (module_channel, module_epoch)
9757 );
9758 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9759 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
9760 }
9761
9762 #[test]
9769 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
9770 let handler = ControlHandler::default();
9771 let module_connection = ConnectionId::new(30);
9772 let client_connection = ConnectionId::new(31);
9773
9774 handler
9775 .refuse_to_end_module_connection_for_a_client(
9776 module_connection,
9777 77,
9778 ForwardingError::ConnectionClosing {
9779 connection_id: client_connection,
9780 },
9781 )
9782 .expect("a closing client must never end the module connection");
9783
9784 assert!(matches!(
9785 handler.refuse_to_end_module_connection_for_a_client(
9786 module_connection,
9787 78,
9788 ForwardingError::ConnectionClosing {
9789 connection_id: module_connection,
9790 },
9791 ),
9792 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
9793 connection_id
9794 })) if connection_id == module_connection
9795 ));
9796 assert!(matches!(
9797 handler.refuse_to_end_module_connection_for_a_client(
9798 module_connection,
9799 79,
9800 ForwardingError::Poisoned,
9801 ),
9802 Err(RouterError::Forwarding(ForwardingError::Poisoned))
9803 ));
9804 assert!(matches!(
9805 handler.refuse_to_end_module_connection_for_a_client(
9806 module_connection,
9807 80,
9808 ForwardingError::StaleModuleEndpoint,
9809 ),
9810 Err(RouterError::Forwarding(
9811 ForwardingError::StaleModuleEndpoint
9812 ))
9813 ));
9814 }
9815
9816 #[tokio::test]
9829 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
9830 let registry = Arc::new(Registry::default());
9831 let forwarding = Arc::new(ForwardingTable::default());
9832 let supervisor = SupervisorHandle::new();
9833 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9834 let handler =
9835 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9836 .with_supervisor(supervisor);
9837
9838 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9839 hello_via_sink(
9840 &handler,
9841 &target_ctx,
9842 &mut target_rx,
9843 hello_frame("target", PROTOCOL_VERSION, 1),
9844 )
9845 .await;
9846
9847 let wrong_nonce = handler
9851 .handle_control_frame(
9852 &route_ctx(ConnectionId::new(91)).0,
9853 route_open_frame_with_admission_facts(
9854 20,
9855 "target",
9856 unique_project_root("admission-facts"),
9857 Some(subc_control::ConsumerIdentity {
9858 module_id: "fed".to_string(),
9859 launch_nonce: "not-the-real-nonce".to_string(),
9860 }),
9861 None,
9862 ),
9863 )
9864 .await
9865 .unwrap();
9866 assert_eq!(
9867 parse_error(&wrong_nonce[0])["code"],
9868 "bad_consumer_identity",
9869 "a mismatched launch nonce must be refused, not stamped Reserved"
9870 );
9871
9872 let never_spawned = handler
9876 .handle_control_frame(
9877 &route_ctx(ConnectionId::new(92)).0,
9878 route_open_frame_with_admission_facts(
9879 21,
9880 "target",
9881 unique_project_root("admission-facts"),
9882 Some(subc_control::ConsumerIdentity {
9883 module_id: "never-spawned".to_string(),
9884 launch_nonce: "any-nonce".to_string(),
9885 }),
9886 None,
9887 ),
9888 )
9889 .await
9890 .unwrap();
9891 assert_eq!(
9892 parse_error(&never_spawned[0])["code"],
9893 "bad_consumer_identity",
9894 "an unspawned module_id must be refused rather than accepted for lack of a record"
9895 );
9896 }
9897
9898 #[tokio::test]
9921 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
9922 let registry = Arc::new(Registry::default());
9923 let forwarding = Arc::new(ForwardingTable::default());
9924 let supervisor = SupervisorHandle::new();
9925 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9926 let handler =
9927 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9928 .with_supervisor(supervisor);
9929
9930 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
9931 hello_via_sink(
9932 &handler,
9933 &target_ctx,
9934 &mut target_rx,
9935 hello_frame("target", PROTOCOL_VERSION, 1),
9936 )
9937 .await;
9938
9939 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
9940 let route_handler = handler.clone();
9941 let route_task = tokio::spawn(async move {
9942 route_handler
9943 .handle_control_frame(
9944 &client_ctx,
9945 route_open_frame_with_admission_facts(
9946 30,
9947 "target",
9948 unique_project_root("admission-facts"),
9949 Some(subc_control::ConsumerIdentity {
9950 module_id: "fed".to_string(),
9951 launch_nonce: "fed-nonce".to_string(),
9952 }),
9953 None,
9954 ),
9955 )
9956 .await
9957 .unwrap()
9958 });
9959
9960 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9967 .await
9968 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
9969 .expect("module control channel closed before route.bind");
9970 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9971 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
9972 panic!("expected route.bind")
9973 };
9974 assert_eq!(
9975 principal,
9976 Some(Principal::Reserved {
9977 module_id: "fed".to_string()
9978 }),
9979 "a correctly attested consumer must be stamped Reserved for its own id"
9980 );
9981
9982 handler
9983 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9984 .await
9985 .unwrap();
9986 assert!(route_task.await.unwrap().is_empty());
9987 assert!(
9988 matches!(
9989 serde_json::from_slice::<ClientControlResponse>(
9990 &client_rx.recv().await.unwrap().body
9991 )
9992 .unwrap(),
9993 ClientControlResponse::RouteOpen { .. }
9994 ),
9995 "the route must actually open, not merely avoid an error"
9996 );
9997 }
9998
9999 #[tokio::test(start_paused = true)]
10000 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
10001 let registry = Arc::new(Registry::default());
10002 let forwarding = Arc::new(ForwardingTable::default());
10003 let supervisor = SupervisorHandle::new();
10004 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10005 let handler =
10006 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10007 .with_supervisor(supervisor);
10008
10009 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
10010 hello_via_sink(
10011 &handler,
10012 &target_ctx,
10013 &mut target_rx,
10014 hello_frame("target", PROTOCOL_VERSION, 1),
10015 )
10016 .await;
10017
10018 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
10019 let direct_handler = handler.clone();
10020 let direct_open = tokio::spawn(async move {
10021 direct_handler
10022 .handle_control_frame(
10023 &direct_ctx,
10024 route_open_frame(2, "target", unique_project_root("route-census-direct")),
10025 )
10026 .await
10027 .unwrap()
10028 });
10029 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10030 .await
10031 .expect("no direct route.bind within 5s")
10032 .expect("target control channel closed before direct route.bind");
10033 handler
10034 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
10035 .await
10036 .unwrap();
10037 assert!(direct_open.await.unwrap().is_empty());
10038 let _ = direct_rx.recv().await.unwrap();
10039
10040 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
10041 let reserved_handler = handler.clone();
10042 let reserved_open = tokio::spawn(async move {
10043 reserved_handler
10044 .handle_control_frame(
10045 &reserved_ctx,
10046 route_open_frame_with_admission_facts(
10047 3,
10048 "target",
10049 unique_project_root("admission-facts"),
10050 Some(ConsumerIdentity {
10051 module_id: "fed".to_string(),
10052 launch_nonce: "fed-nonce".to_string(),
10053 }),
10054 None,
10055 ),
10056 )
10057 .await
10058 .unwrap()
10059 });
10060 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10061 .await
10062 .expect("no reserved route.bind within 5s")
10063 .expect("target control channel closed before reserved route.bind");
10064 handler
10065 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
10066 .await
10067 .unwrap();
10068 assert!(reserved_open.await.unwrap().is_empty());
10069 let _ = reserved_rx.recv().await.unwrap();
10070
10071 forwarding
10072 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
10073 .unwrap();
10074 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
10075 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
10076 module_id: Some("target".to_string()),
10077 })
10078 .unwrap();
10079 let census_frame =
10080 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
10081 let response = handler
10082 .handle_control_frame(&census_ctx, census_frame)
10083 .await
10084 .unwrap()
10085 .pop()
10086 .unwrap();
10087 let actual: Value = serde_json::from_slice(&response.body).unwrap();
10088 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
10089 assert!(matches!(
10090 decoded,
10091 ClientControlResponse::SupervisorRoutes { .. }
10092 ));
10093 let routes = actual["modules"][0]["routes"].as_array().unwrap();
10094 assert_eq!(routes.len(), 2);
10095 assert!(routes.iter().all(|route| route["draining"] == true));
10096 assert!(
10099 routes.iter().all(|route| route["drain_reason"] == "reload"),
10100 "draining routes must name the drain's reason: {routes:?}"
10101 );
10102 assert!(routes.iter().any(|route| {
10103 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
10104 }));
10105 assert!(routes.iter().any(|route| {
10106 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
10107 }));
10108
10109 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10110 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
10111 if std::env::var_os("UPDATE_GOLDEN").is_some() {
10112 std::fs::write(
10113 &golden_path,
10114 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
10115 )
10116 .unwrap();
10117 }
10118 let expected: Value =
10119 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
10120 assert_eq!(actual, expected);
10121 }
10122
10123 async fn query_live_roots(
10124 handler: &ControlHandler,
10125 module_ctx: &RouteCtx,
10126 ) -> ModuleControlResponseToModule {
10127 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
10128 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
10129 let response = handler
10130 .handle_control_frame(module_ctx, frame)
10131 .await
10132 .unwrap()
10133 .pop()
10134 .unwrap();
10135 serde_json::from_slice(&response.body).unwrap()
10136 }
10137
10138 #[tokio::test(start_paused = true)]
10139 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
10140 let registry = Arc::new(Registry::default());
10141 let forwarding = Arc::new(ForwardingTable::default());
10142 let handler = ControlHandler::with_forwarding(registry, forwarding);
10143 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
10144 hello_via_sink(
10145 &handler,
10146 &target_ctx,
10147 &mut target_rx,
10148 hello_frame("target", PROTOCOL_VERSION, 1),
10149 )
10150 .await;
10151 let root = unique_project_root("live-roots-known");
10152 let path = ProjectRootId::from_path_allowing_missing(root.path())
10153 .unwrap()
10154 .as_path()
10155 .to_path_buf();
10156 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
10157 let open_handler = handler.clone();
10158 let opened = tokio::spawn(async move {
10159 open_handler
10160 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
10161 .await
10162 .unwrap()
10163 });
10164 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10165 .await
10166 .unwrap()
10167 .unwrap();
10168 handler
10169 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
10170 .await
10171 .unwrap();
10172 assert!(opened.await.unwrap().is_empty());
10173 let _ = client_rx.recv().await.unwrap();
10174
10175 let root = unique_project_root("live-roots-pending");
10176 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
10177 .unwrap()
10178 .as_path()
10179 .to_path_buf();
10180 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
10181 let open_handler = handler.clone();
10182 let pending = tokio::spawn(async move {
10183 open_handler
10184 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
10185 .await
10186 .unwrap()
10187 });
10188 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10189 .await
10190 .unwrap()
10191 .unwrap();
10192 let actual = query_live_roots(&handler, &target_ctx).await;
10193 let ModuleControlResponseToModule::LiveRoots {
10194 roots,
10195 unknown_root_bindings,
10196 total_bindings,
10197 } = actual
10198 else {
10199 panic!("expected live roots")
10200 };
10201 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
10202 assert_eq!(unknown_root_bindings, 0);
10203 assert_eq!(
10204 roots.len(),
10205 2,
10206 "root-known arm must retain each canonical root"
10207 );
10208 assert_eq!(
10209 total_bindings,
10210 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10211 );
10212 let counts = roots
10213 .iter()
10214 .map(|root| (root.project_root.clone(), root.bound, root.pending))
10215 .collect::<Vec<_>>();
10216 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
10217 expected.sort_by(|a, b| a.0.cmp(&b.0));
10218 assert_eq!(
10219 counts, expected,
10220 "roots must sort by path and count pending separately"
10221 );
10222 handler
10223 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
10224 .await
10225 .unwrap();
10226 assert!(pending.await.unwrap().is_empty());
10227 }
10228
10229 #[tokio::test(start_paused = true)]
10230 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
10231 let forwarding = Arc::new(ForwardingTable::default());
10232 let handler =
10233 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10234 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
10235 hello_via_sink(
10236 &handler,
10237 &target_ctx,
10238 &mut target_rx,
10239 hello_frame("target", PROTOCOL_VERSION, 1),
10240 )
10241 .await;
10242 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
10243 let pending = forwarding
10244 .begin_route_bind_relay_for_test(
10245 client_ctx.connection_id,
10246 client_ctx.egress.clone(),
10247 2,
10248 "target",
10249 )
10250 .unwrap();
10251 forwarding
10252 .complete_pending_relay(
10253 target_ctx.connection_id,
10254 pending.corr,
10255 RouteBindRelayOutcome::Accepted,
10256 )
10257 .unwrap();
10258 let actual = query_live_roots(&handler, &target_ctx).await;
10259 let ModuleControlResponseToModule::LiveRoots {
10260 roots,
10261 unknown_root_bindings,
10262 total_bindings,
10263 } = actual
10264 else {
10265 panic!("expected live roots")
10266 };
10267 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
10268 assert_eq!(
10269 unknown_root_bindings, 1,
10270 "unknown-root arm must not read as no bindings"
10271 );
10272 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
10273 assert_eq!(
10274 total_bindings,
10275 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10276 );
10277 }
10278
10279 #[tokio::test(start_paused = true)]
10287 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
10288 let forwarding = Arc::new(ForwardingTable::default());
10289 let handler =
10290 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10291 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
10292 let replies = handler
10293 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
10294 .await
10295 .unwrap();
10296 let queued_by_hello = module_rx.len();
10297
10298 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
10299 let open_handler = handler.clone();
10300 let open = tokio::spawn(async move {
10301 open_handler
10302 .handle_control_frame(
10303 &client_ctx,
10304 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
10305 )
10306 .await
10307 .unwrap()
10308 });
10309 let mut spins = 0;
10311 while module_rx.len() == queued_by_hello {
10312 spins += 1;
10313 assert!(spins < 10_000, "route.open never queued a route.bind");
10314 tokio::task::yield_now().await;
10315 }
10316
10317 for reply in replies {
10319 module_ctx.egress.send(reply).await.unwrap();
10320 }
10321
10322 let first = module_rx.recv().await.unwrap().frame;
10323 assert_eq!(
10324 first.header.ty,
10325 FrameType::HelloAck,
10326 "the first frame a registering module reads must be its HELLO_ACK"
10327 );
10328 assert_eq!(first.header.corr, 7);
10329 let second = module_rx.recv().await.unwrap().frame;
10330 assert_eq!(second.header.ty, FrameType::Request);
10331 assert!(
10332 matches!(
10333 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
10334 ModuleControlRequest::RouteBind { .. }
10335 ),
10336 "the route.bind follows the ack"
10337 );
10338 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
10339
10340 handler
10341 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
10342 .await
10343 .unwrap();
10344 assert!(open.await.unwrap().is_empty());
10345 let _ = client_rx.recv().await.unwrap();
10346 }
10347
10348 #[tokio::test(start_paused = true)]
10349 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
10350 let handler = ControlHandler::with_forwarding(
10351 Arc::new(Registry::default()),
10352 Arc::new(ForwardingTable::default()),
10353 );
10354 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
10355 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
10356 hello_via_sink(
10357 &handler,
10358 &first_ctx,
10359 &mut first_rx,
10360 hello_frame("first", PROTOCOL_VERSION, 1),
10361 )
10362 .await;
10363 hello_via_sink(
10364 &handler,
10365 &second_ctx,
10366 &mut second_rx,
10367 hello_frame("second", PROTOCOL_VERSION, 2),
10368 )
10369 .await;
10370 let root = unique_project_root("second-only");
10371 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
10372 let cloned = handler.clone();
10373 let open = tokio::spawn(async move {
10374 cloned
10375 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
10376 .await
10377 .unwrap()
10378 });
10379 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
10380 .await
10381 .unwrap()
10382 .unwrap();
10383 let first = query_live_roots(&handler, &first_ctx).await;
10384 let second = query_live_roots(&handler, &second_ctx).await;
10385 assert!(
10386 matches!(
10387 first,
10388 ModuleControlResponseToModule::LiveRoots {
10389 total_bindings: 0,
10390 ..
10391 }
10392 ),
10393 "cross-module scope must not expose another module's roots"
10394 );
10395 assert!(
10396 matches!(
10397 second,
10398 ModuleControlResponseToModule::LiveRoots {
10399 total_bindings: 1,
10400 ..
10401 }
10402 ),
10403 "second module must see its pending route"
10404 );
10405 handler
10406 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
10407 .await
10408 .unwrap();
10409 assert!(open.await.unwrap().is_empty());
10410 }
10411
10412 #[tokio::test(start_paused = true)]
10413 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
10414 let handler = ControlHandler::with_forwarding(
10415 Arc::new(Registry::default()),
10416 Arc::new(ForwardingTable::default()),
10417 );
10418 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
10419 hello_via_sink(
10420 &handler,
10421 &target_ctx,
10422 &mut target_rx,
10423 hello_frame("target", PROTOCOL_VERSION, 1),
10424 )
10425 .await;
10426 let actual = query_live_roots(&handler, &target_ctx).await;
10427 let ModuleControlResponseToModule::LiveRoots {
10428 roots,
10429 unknown_root_bindings,
10430 total_bindings,
10431 } = actual
10432 else {
10433 panic!("expected live roots")
10434 };
10435 assert!(roots.is_empty());
10436 assert_eq!(unknown_root_bindings, 0);
10437 assert_eq!(total_bindings, 0);
10438 assert_eq!(
10439 total_bindings,
10440 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10441 );
10442 }
10443
10444 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
10451 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10452 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
10453 let text = std::fs::read_to_string(&path)
10454 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
10455 let vectors: Vec<(String, Value)> = text
10456 .lines()
10457 .filter(|line| !line.trim().is_empty())
10458 .map(|line| {
10459 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
10460 let id = entry["corpus_id"]
10461 .as_str()
10462 .expect("every vector carries a corpus_id")
10463 .to_string();
10464 (id, entry["package"].clone())
10465 })
10466 .collect();
10467 assert_eq!(
10471 vectors.len(),
10472 3,
10473 "vendored fed corpus changed size; re-sync from subc-federation"
10474 );
10475
10476 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
10491 let richest = vectors
10492 .iter()
10493 .filter_map(|(_, package)| package.as_object())
10494 .map(|object| {
10495 object
10496 .keys()
10497 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
10498 .count()
10499 })
10500 .max()
10501 .unwrap_or(0);
10502 assert!(
10503 richest >= 2,
10504 "vendored corpus no longer carries a package with unmodelled fields, \
10505 so the relay test can no longer distinguish a verbatim relay from a lossy one"
10506 );
10507
10508 vectors
10509 }
10510
10511 #[tokio::test]
10519 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
10520 for (corpus_id, package) in fed_admission_facts_vectors() {
10521 let registry = Arc::new(Registry::default());
10522 let forwarding = Arc::new(ForwardingTable::default());
10523 let supervisor = SupervisorHandle::new();
10524 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10525 let handler =
10526 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10527 .with_supervisor(supervisor)
10528 .with_admission_facts_config(
10529 Some("fed".to_string()),
10530 Some(vec!["target".to_string()]),
10531 );
10532
10533 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10534 hello_via_sink(
10535 &handler,
10536 &target_ctx,
10537 &mut target_rx,
10538 hello_frame("target", PROTOCOL_VERSION, 1),
10539 )
10540 .await;
10541
10542 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
10543 let route_handler = handler.clone();
10544 let expected = package.clone();
10545 let route_task = tokio::spawn(async move {
10546 route_handler
10547 .handle_control_frame(
10548 &client_ctx,
10549 route_open_frame_with_admission_facts(
10550 20,
10551 "target",
10552 unique_project_root("admission-facts"),
10553 Some(subc_control::ConsumerIdentity {
10554 module_id: "fed".to_string(),
10555 launch_nonce: "fed-nonce".to_string(),
10556 }),
10557 Some(package),
10558 ),
10559 )
10560 .await
10561 .unwrap()
10562 });
10563
10564 let bind_frame = target_rx.recv().await.unwrap();
10565 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10566 let ModuleControlRequest::RouteBind {
10567 admission_facts, ..
10568 } = bind
10569 else {
10570 panic!("{corpus_id}: expected route.bind")
10571 };
10572 assert_eq!(
10573 admission_facts,
10574 Some(expected),
10575 "{corpus_id}: relay must not add, drop or reshape any field"
10576 );
10577
10578 handler
10579 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10580 .await
10581 .unwrap();
10582 route_task.await.unwrap();
10583 }
10584 }
10585
10586 #[tokio::test]
10587 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
10588 let registry = Arc::new(Registry::default());
10589 let forwarding = Arc::new(ForwardingTable::default());
10590 let supervisor = SupervisorHandle::new();
10591 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10592 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
10593 let handler =
10594 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10595 .with_supervisor(supervisor)
10596 .with_admission_facts_config(
10597 Some("fed".to_string()),
10598 Some(vec!["target".to_string()]),
10599 );
10600
10601 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
10602 hello_via_sink(
10603 &handler,
10604 &target_ctx,
10605 &mut target_rx,
10606 hello_frame("target", PROTOCOL_VERSION, 1),
10607 )
10608 .await;
10609 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
10610 hello_via_sink(
10611 &handler,
10612 &other_ctx,
10613 &mut other_rx,
10614 hello_frame("other", PROTOCOL_VERSION, 2),
10615 )
10616 .await;
10617
10618 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
10619 let expected_facts = facts.clone();
10620 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
10621 let route_handler = handler.clone();
10622 let route_task = tokio::spawn(async move {
10623 route_handler
10624 .handle_control_frame(
10625 &client_ctx,
10626 route_open_frame_with_admission_facts(
10627 10,
10628 "target",
10629 unique_project_root("admission-facts"),
10630 Some(subc_control::ConsumerIdentity {
10631 module_id: "fed".to_string(),
10632 launch_nonce: "fed-nonce".to_string(),
10633 }),
10634 Some(facts.clone()),
10635 ),
10636 )
10637 .await
10638 .unwrap()
10639 });
10640 let bind_frame = target_rx.recv().await.unwrap();
10641 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10642 let ModuleControlRequest::RouteBind {
10643 admission_facts, ..
10644 } = bind
10645 else {
10646 panic!("expected route.bind")
10647 };
10648 assert_eq!(admission_facts, Some(expected_facts));
10649 handler
10650 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10651 .await
10652 .unwrap();
10653 assert!(route_task.await.unwrap().is_empty());
10654 assert!(matches!(
10655 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
10656 .unwrap(),
10657 ClientControlResponse::RouteOpen { .. }
10658 ));
10659
10660 let direct = handler
10661 .handle_control_frame(
10662 &route_ctx(ConnectionId::new(73)).0,
10663 route_open_frame_with_admission_facts(
10664 11,
10665 "target",
10666 unique_project_root("admission-facts"),
10667 None,
10668 Some(json!({"x": 1})),
10669 ),
10670 )
10671 .await
10672 .unwrap();
10673 assert_eq!(
10674 parse_error(&direct[0])["code"],
10675 "admission_facts_not_permitted"
10676 );
10677
10678 let different_reserved = handler
10679 .handle_control_frame(
10680 &route_ctx(ConnectionId::new(77)).0,
10681 route_open_frame_with_admission_facts(
10682 15,
10683 "target",
10684 unique_project_root("admission-facts"),
10685 Some(subc_control::ConsumerIdentity {
10686 module_id: "other".to_string(),
10687 launch_nonce: "other-nonce".to_string(),
10688 }),
10689 Some(json!({"x": 1})),
10690 ),
10691 )
10692 .await
10693 .unwrap();
10694 assert_eq!(
10695 parse_error(&different_reserved[0])["code"],
10696 "admission_facts_not_permitted"
10697 );
10698
10699 let other_target = handler
10700 .handle_control_frame(
10701 &route_ctx(ConnectionId::new(74)).0,
10702 route_open_frame_with_admission_facts(
10703 12,
10704 "other",
10705 unique_project_root("admission-facts"),
10706 Some(subc_control::ConsumerIdentity {
10707 module_id: "fed".to_string(),
10708 launch_nonce: "fed-nonce".to_string(),
10709 }),
10710 Some(json!({"x": 1})),
10711 ),
10712 )
10713 .await
10714 .unwrap();
10715 assert_eq!(
10716 parse_error(&other_target[0])["code"],
10717 "admission_facts_target_not_allowed"
10718 );
10719
10720 let nonexistent = handler
10721 .handle_control_frame(
10722 &route_ctx(ConnectionId::new(75)).0,
10723 route_open_frame_with_admission_facts(
10724 13,
10725 "missing",
10726 unique_project_root("admission-facts"),
10727 None,
10728 Some(json!({"x": 1})),
10729 ),
10730 )
10731 .await
10732 .unwrap();
10733 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
10734
10735 let described = handler
10736 .handle_control_frame(
10737 &route_ctx(ConnectionId::new(76)).0,
10738 Frame::build(
10739 FrameType::Request,
10740 control_flags(),
10741 0,
10742 0,
10743 14,
10744 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10745 )
10746 .unwrap(),
10747 )
10748 .await
10749 .unwrap();
10750 let ClientControlResponse::ServerDescribe { capabilities, .. } =
10751 serde_json::from_slice(&described[0].body).unwrap()
10752 else {
10753 panic!("expected server.describe response")
10754 };
10755 assert!(capabilities
10756 .iter()
10757 .any(|cap| cap == "admission_facts_relay_v1"));
10758 }
10759
10760 #[tokio::test]
10761 async fn admission_facts_without_configured_carrier_are_rejected() {
10762 let registry = Arc::new(Registry::default());
10763 let forwarding = Arc::new(ForwardingTable::default());
10764 let handler = ControlHandler::with_forwarding(registry, forwarding);
10765 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
10766 hello_via_sink(
10767 &handler,
10768 &target_ctx,
10769 &mut target_rx,
10770 hello_frame("target", PROTOCOL_VERSION, 1),
10771 )
10772 .await;
10773
10774 let responses = handler
10775 .handle_control_frame(
10776 &route_ctx(ConnectionId::new(79)).0,
10777 route_open_frame_with_admission_facts(
10778 16,
10779 "target",
10780 unique_project_root("admission-facts"),
10781 None,
10782 Some(json!({"x": 1})),
10783 ),
10784 )
10785 .await
10786 .unwrap();
10787 assert_eq!(
10788 parse_error(&responses[0])["code"],
10789 "admission_facts_not_permitted"
10790 );
10791 }
10792
10793 #[tokio::test]
10794 async fn route_open_relays_consumer_capabilities_verbatim() {
10795 let registry = Arc::new(Registry::default());
10796 let forwarding = Arc::new(ForwardingTable::default());
10797 let handler =
10798 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10799 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
10800 hello_via_sink(
10801 &handler,
10802 &module_ctx,
10803 &mut module_rx,
10804 hello_frame("aft", PROTOCOL_VERSION, 7),
10805 )
10806 .await;
10807
10808 let expected = vec!["elicitation".to_string(), "roots".to_string()];
10809 let expected_for_request = expected.clone();
10810 let project_root = unique_project_root("consumer-capabilities-present");
10811 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
10812 let route_handler = handler.clone();
10813 let route_task = tokio::spawn(async move {
10814 route_handler
10815 .handle_control_frame(
10816 &client_ctx,
10817 route_open_frame_with_consumer_capabilities(
10818 401,
10819 "aft",
10820 project_root,
10821 Some(expected_for_request),
10822 ),
10823 )
10824 .await
10825 .unwrap()
10826 });
10827 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10828 .await
10829 .unwrap()
10830 .unwrap();
10831 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10832 let ModuleControlRequest::RouteBind {
10833 consumer_capabilities,
10834 ..
10835 } = bind
10836 else {
10837 panic!("expected route.bind request, got {bind:?}");
10838 };
10839 assert_eq!(consumer_capabilities, Some(expected.clone()));
10840
10841 handler
10842 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10843 .await
10844 .unwrap();
10845 let route_response = route_task.await.unwrap();
10846 assert!(route_response.is_empty());
10847 let published = client_rx.recv().await.unwrap();
10848 assert!(matches!(
10849 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10850 ClientControlResponse::RouteOpen { .. }
10851 ));
10852 }
10853
10854 #[tokio::test]
10855 async fn route_open_without_consumer_capabilities_relays_none() {
10856 let registry = Arc::new(Registry::default());
10857 let forwarding = Arc::new(ForwardingTable::default());
10858 let handler =
10859 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10860 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
10861 hello_via_sink(
10862 &handler,
10863 &module_ctx,
10864 &mut module_rx,
10865 hello_frame("aft", PROTOCOL_VERSION, 7),
10866 )
10867 .await;
10868
10869 let project_root = unique_project_root("consumer-capabilities-absent");
10870 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
10871 let route_handler = handler.clone();
10872 let route_task = tokio::spawn(async move {
10873 route_handler
10874 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
10875 .await
10876 .unwrap()
10877 });
10878 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10879 .await
10880 .unwrap()
10881 .unwrap();
10882 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10883 let ModuleControlRequest::RouteBind {
10884 consumer_capabilities,
10885 ..
10886 } = bind
10887 else {
10888 panic!("expected route.bind request, got {bind:?}");
10889 };
10890 assert_eq!(consumer_capabilities, None);
10891
10892 handler
10893 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10894 .await
10895 .unwrap();
10896 let route_response = route_task.await.unwrap();
10897 assert!(route_response.is_empty());
10898 let published = client_rx.recv().await.unwrap();
10899 assert!(matches!(
10900 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10901 ClientControlResponse::RouteOpen { .. }
10902 ));
10903 }
10904
10905 async fn bind_role_versions_for(
10909 sent: Option<BTreeMap<String, String>>,
10910 connection: u64,
10911 ) -> Option<BTreeMap<String, String>> {
10912 let registry = Arc::new(Registry::default());
10913 let forwarding = Arc::new(ForwardingTable::default());
10914 let handler =
10915 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10916 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
10917 hello_via_sink(
10918 &handler,
10919 &module_ctx,
10920 &mut module_rx,
10921 hello_frame("aft", PROTOCOL_VERSION, 7),
10922 )
10923 .await;
10924 let project_root = unique_project_root("role-versions");
10925 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
10926 let route_handler = handler.clone();
10927 let route_task = tokio::spawn(async move {
10928 route_handler
10929 .handle_control_frame(
10930 &client_ctx,
10931 route_open_frame_with_role_versions(403, "aft", project_root, sent),
10932 )
10933 .await
10934 .unwrap()
10935 });
10936 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10937 .await
10938 .expect("a well-formed route.open reaches the module as a bind")
10939 .unwrap();
10940 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10941 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
10942 panic!("expected route.bind request, got {bind:?}");
10943 };
10944 handler
10945 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10946 .await
10947 .unwrap();
10948 assert!(route_task.await.unwrap().is_empty());
10949 let published = client_rx.recv().await.unwrap();
10950 assert!(matches!(
10951 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10952 ClientControlResponse::RouteOpen { .. }
10953 ));
10954 role_versions
10955 }
10956
10957 #[tokio::test]
10958 async fn route_open_relays_role_versions_verbatim() {
10959 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
10960 assert_eq!(
10961 bind_role_versions_for(Some(sent.clone()), 141).await,
10962 Some(sent)
10963 );
10964 }
10965
10966 #[tokio::test]
10969 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
10970 assert_eq!(bind_role_versions_for(None, 143).await, None);
10971 assert_eq!(
10972 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
10973 None
10974 );
10975 }
10976
10977 #[tokio::test]
10978 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
10979 let registry = Arc::new(Registry::default());
10980 let forwarding = Arc::new(ForwardingTable::default());
10981 let handler =
10982 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10983 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
10984 hello_via_sink(
10985 &handler,
10986 &module_ctx,
10987 &mut module_rx,
10988 hello_frame("aft", PROTOCOL_VERSION, 7),
10989 )
10990 .await;
10991
10992 let nine: BTreeMap<String, String> = (0..9)
10993 .map(|index| (format!("role-{index}"), "v1".to_string()))
10994 .collect();
10995 for (label, malformed) in [
10996 (
10997 "invalid role name",
10998 role_versions(&[("Tool_Provider", "v1")]),
10999 ),
11000 ("invalid version", role_versions(&[("tool-provider", "v0")])),
11001 ("nine entries", nine),
11002 ] {
11003 let responses = tokio::time::timeout(
11006 Duration::from_secs(1),
11007 handler.handle_control_frame(
11008 &route_ctx(ConnectionId::new(148)).0,
11009 route_open_frame_with_role_versions(
11010 404,
11011 "aft",
11012 unique_project_root("role-versions-malformed"),
11013 Some(malformed),
11014 ),
11015 ),
11016 )
11017 .await
11018 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
11019 .unwrap();
11020 assert_eq!(responses.len(), 1, "{label}");
11021 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
11022 let error = parse_error(&responses[0]);
11023 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
11024 assert_eq!(
11025 error["detail"]["field"], "role_versions",
11026 "{label}: {error}"
11027 );
11028 assert!(
11029 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
11030 "{label}: a malformed declaration is terminal"
11031 );
11032 assert!(
11033 module_rx.try_recv().is_err(),
11034 "{label}: the module must never see a bind"
11035 );
11036 }
11037 }
11038
11039 #[tokio::test]
11043 async fn route_role_versions_capability_is_advertised() {
11044 let handler = ControlHandler::new(Arc::new(Registry::default()));
11045 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
11046 let ack = hello_via_sink(
11047 &handler,
11048 &ctx,
11049 &mut rx,
11050 hello_frame("m", PROTOCOL_VERSION, 1),
11051 )
11052 .await;
11053 let ack = parse_ack(&ack);
11054 assert!(
11055 ack.subc_capabilities
11056 .iter()
11057 .any(|c| c == "route-role-versions/v1"),
11058 "{:?}",
11059 ack.subc_capabilities
11060 );
11061
11062 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11063 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11064 let reply = handler
11065 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
11066 .await
11067 .unwrap()
11068 .pop()
11069 .unwrap();
11070 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11071 serde_json::from_slice(&reply.body).unwrap()
11072 else {
11073 panic!("not a server.describe reply");
11074 };
11075 assert!(
11076 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
11077 "{capabilities:?}"
11078 );
11079 }
11080
11081 #[tokio::test]
11085 async fn catalog_events_capability_is_advertised() {
11086 let handler = ControlHandler::new(Arc::new(Registry::default()));
11087 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11088 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11089 let reply = handler
11090 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
11091 .await
11092 .unwrap()
11093 .pop()
11094 .unwrap();
11095 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11096 serde_json::from_slice(&reply.body).unwrap()
11097 else {
11098 panic!("not a server.describe reply");
11099 };
11100 assert!(
11101 capabilities
11102 .iter()
11103 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
11104 "{capabilities:?}"
11105 );
11106 }
11107
11108 #[tokio::test]
11109 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
11110 let handler = ControlHandler::new(Arc::new(Registry::default()));
11111 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
11112 let ack = hello_via_sink(
11113 &handler,
11114 &module,
11115 &mut module_rx,
11116 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
11117 )
11118 .await;
11119 let ack = parse_ack(&ack);
11120 assert!(
11121 ack.subc_capabilities
11122 .iter()
11123 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11124 "{:?}",
11125 ack.subc_capabilities
11126 );
11127
11128 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
11129 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11130 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11131 let reply = handler
11132 .handle_control_frame(&client, frame)
11133 .await
11134 .unwrap()
11135 .pop()
11136 .unwrap();
11137 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11138 serde_json::from_slice(&reply.body).unwrap()
11139 else {
11140 panic!("not a server.describe reply");
11141 };
11142 assert!(
11143 capabilities
11144 .iter()
11145 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11146 "{capabilities:?}"
11147 );
11148 }
11149
11150 #[tokio::test]
11151 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
11152 let handler = ControlHandler::new(Arc::new(Registry::default()));
11153 let (module, mut rx) = route_ctx(ConnectionId::new(154));
11154 let ack = hello_via_sink(
11155 &handler,
11156 &module,
11157 &mut rx,
11158 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
11159 )
11160 .await;
11161 assert!(parse_ack(&ack)
11162 .subc_capabilities
11163 .iter()
11164 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11165 let (client, _rx) = route_ctx(ConnectionId::new(155));
11166 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11167 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11168 let reply = handler
11169 .handle_control_frame(&client, frame)
11170 .await
11171 .unwrap()
11172 .pop()
11173 .unwrap();
11174 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11175 serde_json::from_slice(&reply.body).unwrap()
11176 else {
11177 panic!("not a server.describe reply")
11178 };
11179 assert!(capabilities
11180 .iter()
11181 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11182 }
11183
11184 #[tokio::test]
11185 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
11186 let registry = Arc::new(Registry::default());
11187 let forwarding = Arc::new(ForwardingTable::default());
11188 let handler =
11189 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11190 .with_health_probe_timeout(Duration::from_secs(5));
11191 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
11192 hello_via_sink(
11193 &handler,
11194 &module_ctx,
11195 &mut module_rx,
11196 non_routable_hello_frame_with_control_ops(
11197 "mcp",
11198 300,
11199 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11200 ),
11201 )
11202 .await;
11203 assert!(registry
11204 .get_module("mcp")
11205 .unwrap()
11206 .unwrap()
11207 .manifest
11208 .provides
11209 .is_empty());
11210
11211 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
11212 let route_response = handler
11213 .handle_control_frame(
11214 &route_client_ctx,
11215 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
11216 )
11217 .await
11218 .unwrap();
11219 assert_eq!(route_response[0].header.ty, FrameType::Error);
11220 assert_eq!(
11221 parse_error(&route_response[0])["code"],
11222 "target_unavailable"
11223 );
11224 assert!(parse_error(&route_response[0])["message"]
11225 .as_str()
11226 .unwrap()
11227 .contains("does not provide the requested target"));
11228 assert!(module_rx.try_recv().is_err());
11229
11230 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
11231 let health_handler = handler.clone();
11232 let health_task = tokio::spawn(async move {
11233 health_handler
11234 .handle_control_frame(
11235 &health_client_ctx,
11236 supervisor_health_probe_frame(302, "mcp"),
11237 )
11238 .await
11239 .unwrap()
11240 });
11241 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11242 .await
11243 .unwrap()
11244 .unwrap();
11245 assert_eq!(
11246 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11247 ModuleControlRequest::HealthCheck {}
11248 );
11249 handler
11250 .handle_control_frame(
11251 &module_ctx,
11252 health_response(health_frame.header.corr, HealthStatus::Ok),
11253 )
11254 .await
11255 .unwrap();
11256 let health_response = health_task.await.unwrap();
11257 assert_eq!(health_response[0].header.ty, FrameType::Response);
11258 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11259 ClientControlResponse::SupervisorHealthProbe {
11260 module_id, status, ..
11261 } => {
11262 assert_eq!(module_id, "mcp");
11263 assert_eq!(status, HealthStatus::Ok);
11264 }
11265 other => panic!("unexpected health response: {other:?}"),
11266 }
11267
11268 forwarding
11273 .cleanup_connection(module_ctx.connection_id)
11274 .unwrap();
11275 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
11276 let cleanup_response = tokio::time::timeout(
11277 Duration::from_millis(200),
11278 handler.handle_control_frame(
11279 &cleanup_probe_ctx,
11280 supervisor_health_probe_frame(303, "mcp"),
11281 ),
11282 )
11283 .await
11284 .expect("probe should fail immediately when the control lane is gone")
11285 .unwrap();
11286 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
11287 assert_eq!(
11288 parse_error(&cleanup_response[0])["code"],
11289 "target_unavailable"
11290 );
11291 assert!(parse_error(&cleanup_response[0])["message"]
11292 .as_str()
11293 .unwrap()
11294 .contains("no module connection"));
11295
11296 handler
11297 .cleanup_connection(module_ctx.connection_id)
11298 .unwrap();
11299 }
11300
11301 #[tokio::test]
11302 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
11303 let registry = Arc::new(Registry::default());
11304 let supervisor_handle = SupervisorHandle::new();
11305 let supervisor =
11306 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11307 .with_handle(supervisor_handle.clone())
11308 .with_connection_file_path(
11309 std::env::temp_dir()
11310 .join(format!("subc-route-open-warming-{}", std::process::id())),
11311 );
11312 let module = supervisor
11313 .supervise_configured(
11314 ModuleSpec {
11315 module_id: "warming".to_string(),
11316 program: fake_aft_stub_path(),
11317 args: Vec::new(),
11318 env: Vec::new(),
11319 reserved: false,
11320 reserved_prefixes: Vec::new(),
11321 protocol: ModuleProtocol::Subc,
11322 overlap: Default::default(),
11323 },
11324 true,
11325 )
11326 .unwrap();
11327 assert_eq!(module.state().unwrap(), ModuleState::Running);
11328
11329 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11330 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
11331 let response = handler
11332 .handle_control_frame(
11333 &ctx,
11334 route_open_frame(304, "warming", unique_project_root("warming")),
11335 )
11336 .await
11337 .unwrap();
11338 module.stop().await.unwrap();
11339
11340 assert_eq!(response[0].header.ty, FrameType::Error);
11341 let error = parse_error(&response[0]);
11342 assert_eq!(error["code"], "module_warming");
11343 assert!(error["message"]
11344 .as_str()
11345 .unwrap()
11346 .contains("state=running, enabled=true, live=false"));
11347 }
11348
11349 #[test]
11350 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
11351 let handler = ControlHandler::new(Arc::new(Registry::default()));
11352 let capture = EventCapture::default();
11353 let _subscriber =
11354 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11355 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
11356 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
11357 let pending = (0..limit).collect::<Vec<_>>();
11358 let response = handler
11359 .route_open_capacity_refusal(
11360 &ctx,
11361 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
11362 "busy",
11363 pending.len(),
11364 limit,
11365 )
11366 .unwrap();
11367 assert_eq!(parse_error(&response)["code"], "target_unavailable");
11368 let event = capture
11369 .events()
11370 .into_iter()
11371 .find(|event| {
11372 event.target == "control"
11373 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
11374 })
11375 .expect("connection admission refusal event");
11376 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11377 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11378 }
11379
11380 #[test]
11381 fn route_open_target_cap_logs_admission_reason_and_capacity() {
11382 let handler = ControlHandler::new(Arc::new(Registry::default()));
11383 let capture = EventCapture::default();
11384 let _subscriber =
11385 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11386 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
11387 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
11388 let guards = (0..limit)
11389 .map(|_| {
11390 handler
11391 .route_bind_concurrency
11392 .try_admit("busy", limit)
11393 .unwrap()
11394 })
11395 .collect::<Vec<_>>();
11396 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
11397 Err(in_flight) => in_flight,
11398 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
11399 };
11400 let response = handler
11401 .route_open_target_capacity_refusal(
11402 &ctx,
11403 &route_open_frame(397, "busy", unique_project_root("target-cap")),
11404 "busy",
11405 in_flight,
11406 )
11407 .unwrap();
11408 assert_eq!(parse_error(&response)["code"], "target_unavailable");
11409 let event = capture
11410 .events()
11411 .into_iter()
11412 .find(|event| {
11413 event.target == "control"
11414 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
11415 })
11416 .expect("target admission refusal event");
11417 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11418 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11419 drop(guards);
11420 }
11421
11422 #[tokio::test(flavor = "current_thread")]
11423 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
11424 let handler = ControlHandler::new(Arc::new(Registry::default()));
11425 let capture = EventCapture::default();
11426 let _subscriber =
11427 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11428 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
11429 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11430
11431 let _ = handler
11432 .handle_client_control_request(
11433 &ctx,
11434 frame,
11435 ClientControlRequest::SupervisorSetEnabled {
11436 module_id: "broca".to_string(),
11437 enabled: false,
11438 },
11439 )
11440 .await
11441 .unwrap();
11442
11443 let events = capture
11444 .events()
11445 .into_iter()
11446 .filter(|event| {
11447 event.target == "control"
11448 && event.fields.get("message").map(String::as_str)
11449 == Some("supervisor request received")
11450 })
11451 .collect::<Vec<_>>();
11452 assert_eq!(events.len(), 1, "one supervisor request log line");
11453 let event = &events[0];
11454 assert_eq!(event.level, tracing::Level::INFO);
11455 assert_eq!(
11456 event.fields.get("op"),
11457 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
11458 );
11459 assert_eq!(
11460 event.fields.get("module_id"),
11461 Some(&"\"broca\"".to_string())
11462 );
11463 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
11464 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
11465 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
11466 }
11467
11468 #[tokio::test(flavor = "current_thread")]
11469 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
11470 let registry = Arc::new(Registry::default());
11471 let module_connection = ConnectionId::new(303);
11472 registry
11473 .register_with_control_ops(
11474 manifest("aft", PROTOCOL_VERSION),
11475 PROTOCOL_VERSION,
11476 module_connection,
11477 module_baseline_control_ops(),
11478 )
11479 .unwrap();
11480 let handler = ControlHandler::new(registry);
11481 let capture = EventCapture::default();
11482 let _subscriber =
11483 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11484 let (ctx, _rx) = route_ctx(module_connection);
11485 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11486
11487 let _ = handler
11488 .handle_client_control_request(
11489 &ctx,
11490 frame,
11491 ClientControlRequest::SupervisorSetEnabled {
11492 module_id: "broca".to_string(),
11493 enabled: false,
11494 },
11495 )
11496 .await
11497 .unwrap();
11498
11499 let callers = capture
11500 .events()
11501 .into_iter()
11502 .filter(|event| {
11503 event.fields.get("message").map(String::as_str)
11504 == Some("supervisor request received")
11505 })
11506 .map(|event| event.fields.get("caller").cloned())
11507 .collect::<Vec<_>>();
11508 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
11509 }
11510
11511 #[tokio::test(flavor = "current_thread")]
11512 async fn supervisor_list_does_not_log_request_received() {
11513 let handler = ControlHandler::new(Arc::new(Registry::default()));
11514 let capture = EventCapture::default();
11515 let _subscriber =
11516 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11517 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
11518 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
11519
11520 handler
11521 .handle_client_control_request(&ctx, frame, ClientControlRequest::SupervisorList {})
11522 .await
11523 .unwrap();
11524
11525 assert!(capture.events().into_iter().all(|event| {
11526 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11527 }));
11528 }
11529
11530 #[tokio::test(flavor = "current_thread")]
11531 async fn supervisor_rescan_preview_does_not_log_request_received() {
11532 let handler = ControlHandler::new(Arc::new(Registry::default()));
11533 let capture = EventCapture::default();
11534 let _subscriber =
11535 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11536 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
11537 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
11538
11539 handler
11540 .handle_client_control_request(
11541 &ctx,
11542 frame,
11543 ClientControlRequest::SupervisorRescan { preview: true },
11544 )
11545 .await
11546 .unwrap();
11547
11548 assert!(capture.events().into_iter().all(|event| {
11549 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11550 }));
11551 }
11552
11553 struct ApproveOperator;
11554 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
11555 fn prompt(
11556 &self,
11557 _: &str,
11558 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
11559 ) -> crate::operator_confirm::ProviderResult {
11560 crate::operator_confirm::ProviderResult::Approved
11561 }
11562 }
11563
11564 #[tokio::test]
11565 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
11566 assert_eq!(
11567 ControlHandler::default().privileged_operator_prompt,
11568 PrivilegedOperatorPrompt::Log
11569 );
11570 for (initial, changed, running) in [
11571 ("log", "enforce", PrivilegedOperatorPrompt::Log),
11572 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
11573 ] {
11574 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
11575 let path = dir.join("subc.jsonc");
11576 std::fs::write(
11577 &path,
11578 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
11579 )
11580 .unwrap();
11581 let config = crate::daemon_config::load(&path).unwrap().unwrap();
11582 let registry = Arc::new(Registry::default());
11583 let handle = SupervisorHandle::new();
11584 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
11585 .with_handle(handle.clone());
11586 let handler = ControlHandler::new(registry)
11587 .with_supervisor(handle)
11588 .with_privileged_operator_prompt(config.privileged_operator_prompt)
11589 .with_supervisor_rescan(supervisor, &path, config.port);
11590 handler.forwarding.operator_confirms().configure(
11591 Arc::new(ApproveOperator),
11592 crate::operator_confirm::OperatorLimits::default(),
11593 );
11594 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
11595
11596 for (disk_mode, needs_restart) in [
11597 (Some(initial), false),
11598 (Some(changed), true),
11599 (Some(changed), true),
11600 (None, running == PrivilegedOperatorPrompt::Enforce),
11601 (Some(initial), false),
11602 ] {
11603 let mut doc = json!({"version":1});
11604 if let Some(mode) = disk_mode {
11605 doc["privileged_operator_prompt"] = json!(mode);
11606 }
11607 std::fs::write(&path, doc.to_string()).unwrap();
11608 for preview in [true, false] {
11609 let request = ClientControlRequest::SupervisorRescan { preview };
11610 let frame = Frame::build(
11611 FrameType::Request,
11612 control_flags(),
11613 0,
11614 0,
11615 1,
11616 serde_json::to_vec(&request).unwrap(),
11617 )
11618 .unwrap();
11619 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
11620 assert_eq!(frames.len(), 1);
11621 let ClientControlResponse::SupervisorRescan { result } =
11622 serde_json::from_slice(&frames[0].body).unwrap()
11623 else {
11624 panic!("expected a rescan response");
11625 };
11626 let expected = if needs_restart {
11627 vec!["privileged_operator_prompt"]
11628 } else {
11629 Vec::new()
11630 };
11631 assert_eq!(result.restart_required, expected);
11632 assert_eq!(handler.privileged_operator_prompt, running);
11633 assert_eq!(
11634 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
11635 running
11636 );
11637 }
11638 }
11639 }
11640 }
11641
11642 #[tokio::test]
11643 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
11644 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
11645 let path = dir.join("subc.jsonc");
11646 let registry = Arc::new(Registry::default());
11647 let handle = SupervisorHandle::new();
11648 let supervisor =
11649 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
11650 let handler = ControlHandler::new(registry)
11651 .with_supervisor(handle)
11652 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
11653 .with_supervisor_rescan(supervisor, &path, None);
11654 handler.forwarding.operator_confirms().configure(
11655 Arc::new(ApproveOperator),
11656 crate::operator_confirm::OperatorLimits::default(),
11657 );
11658 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
11659 for preview in [true, false] {
11660 std::fs::write(
11661 &path,
11662 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
11663 )
11664 .unwrap();
11665 let request = ClientControlRequest::SupervisorRescan { preview };
11666 let frame = Frame::build(
11667 FrameType::Request,
11668 control_flags(),
11669 0,
11670 0,
11671 1,
11672 serde_json::to_vec(&request).unwrap(),
11673 )
11674 .unwrap();
11675 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
11676 assert_eq!(frames.len(), 1);
11677 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
11678 assert_eq!(
11679 handler.privileged_operator_prompt,
11680 PrivilegedOperatorPrompt::Enforce
11681 );
11682 }
11683 }
11684
11685 #[tokio::test]
11690 async fn route_open_refusal_names_the_check_that_refused() {
11691 let handler = ControlHandler::new(Arc::new(Registry::default()));
11692 let capture = EventCapture::default();
11693 let _subscriber =
11694 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11695 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
11696 let response = handler
11697 .handle_control_frame(
11698 &ctx,
11699 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
11700 )
11701 .await
11702 .unwrap();
11703
11704 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11705 let event = capture
11706 .events()
11707 .into_iter()
11708 .find(|event| {
11709 event.target == "control"
11710 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
11711 })
11712 .expect("route.open refusal event");
11713 assert_eq!(
11714 event.fields.get("reason"),
11715 Some(&"\"not_registered\"".to_string())
11716 );
11717 }
11718
11719 #[tokio::test]
11720 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
11721 let registry = Arc::new(Registry::default());
11722 let supervisor_handle = SupervisorHandle::new();
11723 let supervisor =
11724 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11725 .with_handle(supervisor_handle.clone())
11726 .with_connection_file_path(std::env::temp_dir().join(format!(
11727 "subc-route-open-refusal-info-{}",
11728 std::process::id()
11729 )));
11730 let module = supervisor
11731 .supervise_configured(
11732 ModuleSpec {
11733 module_id: "warming".to_string(),
11734 program: fake_aft_stub_path(),
11735 args: Vec::new(),
11736 env: Vec::new(),
11737 reserved: false,
11738 reserved_prefixes: Vec::new(),
11739 protocol: ModuleProtocol::Subc,
11740 overlap: Default::default(),
11741 },
11742 true,
11743 )
11744 .unwrap();
11745 assert_eq!(module.state().unwrap(), ModuleState::Running);
11746
11747 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11748 assert!(handler
11749 .counters()
11750 .snapshot()
11751 .get("route_open_refused_by_code")
11752 .is_none());
11753 let capture = EventCapture::default();
11754 let _subscriber =
11755 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11756 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
11757 let response = handler
11758 .handle_control_frame(
11759 &ctx,
11760 route_open_frame(394, "warming", unique_project_root("refusal-info")),
11761 )
11762 .await
11763 .unwrap();
11764 module.stop().await.unwrap();
11765
11766 assert_eq!(parse_error(&response[0])["code"], "module_warming");
11767 let event = capture
11768 .events()
11769 .into_iter()
11770 .find(|event| {
11771 event.target == "control"
11772 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
11773 })
11774 .expect("route.open refusal event");
11775 assert_eq!(
11776 event.fields.get("module_id"),
11777 Some(&"\"warming\"".to_string())
11778 );
11779 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
11780 assert_eq!(
11781 event.fields.get("reason"),
11782 Some(&"\"supervised_not_registered\"".to_string())
11783 );
11784 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
11785 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
11786 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
11787 assert_eq!(
11788 handler.counters().snapshot()["route_open_refused_by_code"],
11789 json!({ "module_warming": 1 })
11790 );
11791 }
11792
11793 const OUTAGE_START: &str = "route.open refusing module: not serving";
11794 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
11795
11796 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
11797 capture
11798 .events()
11799 .into_iter()
11800 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
11801 .collect()
11802 }
11803
11804 fn supervise_stub(
11805 registry: &Arc<Registry>,
11806 module_id: &str,
11807 enabled: bool,
11808 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
11809 let supervisor_handle = SupervisorHandle::new();
11810 let supervisor =
11811 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
11812 .with_handle(supervisor_handle.clone())
11813 .with_connection_file_path(std::env::temp_dir().join(format!(
11814 "subc-route-outage-{module_id}-{}",
11815 std::process::id()
11816 )));
11817 let module = supervisor
11818 .supervise_configured(
11819 ModuleSpec {
11820 module_id: module_id.to_string(),
11821 program: fake_aft_stub_path(),
11822 args: Vec::new(),
11823 env: Vec::new(),
11824 reserved: false,
11825 reserved_prefixes: Vec::new(),
11826 protocol: ModuleProtocol::Subc,
11827 overlap: Default::default(),
11828 },
11829 enabled,
11830 )
11831 .unwrap();
11832 (supervisor_handle, module)
11833 }
11834
11835 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
11836 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
11837 module_id: module_id.to_string(),
11838 drain_timeout_ms: Some(50),
11839 })
11840 .unwrap();
11841 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11842 }
11843
11844 #[test]
11848 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
11849 let registry = Arc::new(Registry::default());
11850 let forwarding = Arc::new(ForwardingTable::default());
11851 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11852 let second = ControlHandler::with_forwarding(registry, forwarding);
11853 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
11854 }
11855
11856 #[tokio::test(flavor = "current_thread")]
11860 async fn route_open_unknown_module_refusals_add_no_outage_state() {
11861 let handler = ControlHandler::new(Arc::new(Registry::default()));
11862 let capture = EventCapture::default();
11863 let _subscriber =
11864 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11865 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
11866 for corr in 0..8 {
11867 let response = handler
11868 .handle_control_frame(
11869 &ctx,
11870 route_open_frame(
11871 380 + corr,
11872 &format!("nobody-{corr}"),
11873 unique_project_root("outage-unknown"),
11874 ),
11875 )
11876 .await
11877 .unwrap();
11878 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11879 }
11880
11881 assert_eq!(handler.route_outages.tracked_module_count(), 0);
11882 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
11883 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
11884 }
11885
11886 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11891 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
11892 let registry = Arc::new(Registry::default());
11893 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
11894 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11895 let capture = EventCapture::default();
11896 let _subscriber =
11897 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11898 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
11899 handler.route_outages.record_accepted("outage-restart");
11902
11903 let response = handler
11904 .handle_control_frame(
11905 &ctx,
11906 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
11907 )
11908 .await
11909 .unwrap();
11910 assert_eq!(response[0].header.ty, FrameType::Error);
11911 let starts = outage_lines(&capture, OUTAGE_START);
11912 assert_eq!(starts.len(), 1, "{starts:?}");
11913 assert_eq!(starts[0].level, tracing::Level::WARN);
11914 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
11915 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
11916 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
11917 handler.route_outages.record_accepted("outage-restart");
11918 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
11919
11920 let restart = handler
11921 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
11922 .await
11923 .unwrap();
11924 assert_eq!(
11925 restart[0].header.ty,
11926 FrameType::Response,
11927 "{:?}",
11928 parse_error(&restart[0])
11929 );
11930 handler
11931 .handle_control_frame(
11932 &ctx,
11933 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
11934 )
11935 .await
11936 .unwrap();
11937 module.stop().await.unwrap();
11938
11939 let starts = outage_lines(&capture, OUTAGE_START);
11940 assert_eq!(starts.len(), 2, "{starts:?}");
11941 assert_eq!(starts[1].level, tracing::Level::INFO);
11942 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
11943 }
11944
11945 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11949 async fn failed_operator_restart_leaves_no_operator_mark() {
11950 let registry = Arc::new(Registry::default());
11951 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
11952 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11953 let capture = EventCapture::default();
11954 let _subscriber =
11955 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11956 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
11957 handler.route_outages.record_accepted("outage-disabled");
11958
11959 let restart = handler
11960 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
11961 .await
11962 .unwrap();
11963 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
11964 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
11965
11966 handler
11967 .handle_control_frame(
11968 &ctx,
11969 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
11970 )
11971 .await
11972 .unwrap();
11973 let starts = outage_lines(&capture, OUTAGE_START);
11974 assert_eq!(starts.len(), 1, "{starts:?}");
11975 assert_eq!(starts[0].level, tracing::Level::WARN);
11976 }
11977
11978 #[tokio::test(flavor = "current_thread")]
11979 async fn route_open_unknown_module_escapes_target_module_id() {
11980 let handler = ControlHandler::new(Arc::new(Registry::default()));
11981 let capture = EventCapture::default();
11982 let _subscriber =
11983 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11984 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
11985 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
11986 let response = handler
11987 .handle_control_frame(
11988 &ctx,
11989 route_open_frame(
11990 395,
11991 hostile_module_id,
11992 unique_project_root("hostile-target-module-id"),
11993 ),
11994 )
11995 .await
11996 .unwrap();
11997
11998 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11999 let event = capture
12000 .events()
12001 .into_iter()
12002 .find(|event| {
12003 event.target == "control"
12004 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12005 })
12006 .expect("route.open unknown-module refusal event");
12007 let logged = event.fields.get("module_id").expect("module_id field");
12008 assert!(!logged.bytes().any(|byte| byte < 0x20));
12009 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12010 }
12011
12012 #[tokio::test(flavor = "current_thread")]
12013 async fn route_open_module_rejection_uses_daemon_counter_key() {
12014 let registry = Arc::new(Registry::default());
12015 let forwarding = Arc::new(ForwardingTable::default());
12016 let handler =
12017 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12018 let module_connection = ConnectionId::new(95);
12019 let (module_ctx, mut module_rx) = route_ctx(module_connection);
12020 hello_via_sink(
12021 &handler,
12022 &module_ctx,
12023 &mut module_rx,
12024 hello_frame("aft", PROTOCOL_VERSION, 395),
12025 )
12026 .await;
12027
12028 let client_connection = ConnectionId::new(96);
12029 let (client_ctx, _client_rx) = route_ctx(client_connection);
12030 let capture = EventCapture::default();
12031 let _subscriber =
12032 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12033 let (route_task, bind) = relay_route_open(
12034 &handler,
12035 client_connection,
12036 &client_ctx.egress,
12037 &mut module_rx,
12038 396,
12039 "aft",
12040 "hostile-module-code",
12041 )
12042 .await;
12043 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
12044 let rejection = Frame::build(
12045 FrameType::Error,
12046 control_flags(),
12047 0,
12048 0,
12049 bind.header.corr,
12050 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
12051 )
12052 .unwrap();
12053 handler
12054 .handle_control_frame(&module_ctx, rejection)
12055 .await
12056 .unwrap();
12057
12058 let response = route_task.await.unwrap();
12059 assert_eq!(parse_error(&response[0])["code"], hostile_code);
12060 let counters = handler.counters().snapshot();
12061 assert_eq!(
12062 counters["route_open_refused_by_code"],
12063 json!({ "module_rejected": 1 })
12064 );
12065 assert!(counters["route_open_refused_by_code"]
12066 .get(hostile_code)
12067 .is_none());
12068
12069 let event = capture
12070 .events()
12071 .into_iter()
12072 .find(|event| {
12073 event.target == "control"
12074 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
12075 })
12076 .expect("route.open module-rejection refusal event");
12077 let logged = event.fields.get("module_code").expect("module_code field");
12078 assert!(!logged.bytes().any(|byte| byte < 0x20));
12079 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12080 }
12081
12082 #[tokio::test]
12083 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
12084 let registry = Arc::new(Registry::default());
12085 let supervisor_handle = SupervisorHandle::new();
12086 let missing_program = std::env::temp_dir().join(format!(
12087 "subc-route-open-missing-program-{}",
12088 std::process::id()
12089 ));
12090 let supervisor =
12091 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12092 .with_handle(supervisor_handle.clone());
12093 let module = supervisor
12094 .supervise_configured(
12095 ModuleSpec {
12096 module_id: "failed".to_string(),
12097 program: missing_program,
12098 args: Vec::new(),
12099 env: Vec::new(),
12100 reserved: false,
12101 reserved_prefixes: Vec::new(),
12102 protocol: ModuleProtocol::Subc,
12103 overlap: Default::default(),
12104 },
12105 true,
12106 )
12107 .unwrap();
12108 assert_eq!(module.state().unwrap(), ModuleState::Failed);
12109
12110 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12111 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
12112 let response = handler
12113 .handle_control_frame(
12114 &ctx,
12115 route_open_frame(305, "failed", unique_project_root("failed")),
12116 )
12117 .await
12118 .unwrap();
12119
12120 assert_eq!(response[0].header.ty, FrameType::Error);
12121 let error = parse_error(&response[0]);
12122 assert_eq!(error["code"], "target_unavailable");
12123 assert!(error["message"]
12124 .as_str()
12125 .unwrap()
12126 .contains("state=failed, enabled=true, live=false"));
12127 }
12128
12129 #[tokio::test]
12130 async fn route_open_role_mismatch_remains_target_unavailable() {
12131 let registry = Arc::new(Registry::default());
12132 let handler = ControlHandler::new(Arc::clone(®istry));
12133 handler
12134 .handle_control(
12135 ConnectionId::new(41),
12136 non_routable_hello_frame_with_control_ops("health-only", 306, None),
12137 )
12138 .unwrap();
12139
12140 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
12141 let response = handler
12142 .handle_control_frame(
12143 &ctx,
12144 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
12145 )
12146 .await
12147 .unwrap();
12148
12149 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12150 assert!(parse_error(&response[0])["message"]
12151 .as_str()
12152 .unwrap()
12153 .contains("does not provide the requested target"));
12154 }
12155
12156 #[tokio::test]
12157 async fn route_open_inactive_registration_remains_target_unavailable() {
12158 let registry = Arc::new(Registry::default());
12159 let handler = ControlHandler::new(Arc::clone(®istry));
12160 handler
12161 .handle_control(
12162 ConnectionId::new(43),
12163 hello_frame("inactive", PROTOCOL_VERSION, 308),
12164 )
12165 .unwrap();
12166 assert!(registry
12167 .set_module_state_for_test("inactive", ChannelState::Closed)
12168 .unwrap());
12169
12170 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
12171 let response = handler
12172 .handle_control_frame(
12173 &ctx,
12174 route_open_frame(309, "inactive", unique_project_root("inactive")),
12175 )
12176 .await
12177 .unwrap();
12178
12179 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12180 assert!(parse_error(&response[0])["message"]
12181 .as_str()
12182 .unwrap()
12183 .contains("is not active"));
12184 }
12185
12186 #[tokio::test]
12187 async fn late_health_reply_is_recorded_through_the_module_response_path() {
12188 let registry = Arc::new(Registry::default());
12189 let forwarding = Arc::new(ForwardingTable::default());
12190 let supervisor_handle = SupervisorHandle::new();
12191 let supervisor =
12192 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
12193 .with_forwarding(Arc::clone(&forwarding))
12194 .with_handle(supervisor_handle.clone());
12195 let module = supervisor
12196 .supervise_configured(
12197 crate::ModuleSpec {
12198 module_id: "late-health-response".to_string(),
12199 program: PathBuf::from("disabled-module"),
12200 args: Vec::new(),
12201 env: Vec::new(),
12202 reserved: false,
12203 reserved_prefixes: Vec::new(),
12204 protocol: ModuleProtocol::Subc,
12205 overlap: Default::default(),
12206 },
12207 false,
12208 )
12209 .unwrap();
12210 let handler =
12211 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12212 .with_supervisor(supervisor_handle);
12213 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
12214 handler
12215 .handle_control_frame(
12216 &module_ctx,
12217 hello_frame_with_control_ops(
12218 "late-health-response",
12219 PROTOCOL_VERSION,
12220 7,
12221 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12222 ),
12223 )
12224 .await
12225 .unwrap();
12226 let probe_started_at = Instant::now() - Duration::from_millis(80);
12227 let pending = forwarding
12228 .begin_health_probe_rpc_for(
12229 "late-health-response",
12230 MODULE_CONTROL_OP_HEALTH_CHECK,
12231 probe_started_at,
12232 Instant::now() - Duration::from_millis(1),
12233 )
12234 .unwrap();
12235 assert!(forwarding
12236 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
12237 .unwrap());
12238
12239 let responses = handler
12240 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
12241 .await
12242 .unwrap();
12243
12244 assert!(responses.is_empty());
12245 let health = module.status().unwrap().health;
12246 assert_eq!(health.late_answer_count, 1);
12247 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
12248 }
12249
12250 #[tokio::test]
12251 async fn health_probe_timeout_and_module_death_are_typed() {
12252 let registry = Arc::new(Registry::default());
12253 let forwarding = Arc::new(ForwardingTable::default());
12254 let handler =
12255 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12256 .with_health_probe_timeout(Duration::from_millis(50));
12257 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
12258 hello_via_sink(
12259 &handler,
12260 &module_ctx,
12261 &mut module_rx,
12262 hello_frame_with_control_ops(
12263 "aft",
12264 PROTOCOL_VERSION,
12265 7,
12266 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12267 ),
12268 )
12269 .await;
12270
12271 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12272 let responses = handler
12273 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
12274 .await
12275 .unwrap();
12276 assert_eq!(responses[0].header.ty, FrameType::Error);
12277 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
12278 let _ = module_rx.try_recv();
12279
12280 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
12281 let health_handler = handler.clone();
12282 let death_task = tokio::spawn(async move {
12283 health_handler
12284 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
12285 .await
12286 .unwrap()
12287 });
12288 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12289 .await
12290 .unwrap()
12291 .unwrap();
12292 handler
12293 .cleanup_connection(module_ctx.connection_id)
12294 .unwrap();
12295 let responses = death_task.await.unwrap();
12296 assert_eq!(responses[0].header.ty, FrameType::Error);
12297 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
12298 }
12299
12300 #[test]
12301 fn hello_requires_exact_protocol_version() {
12302 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
12303 let registry = Arc::new(Registry::default());
12304 let handler = ControlHandler::new(Arc::clone(®istry));
12305 let responses = handler
12306 .handle_control(
12307 ConnectionId::new(connection),
12308 hello_frame("aft", offered, 9),
12309 )
12310 .unwrap();
12311
12312 assert_eq!(responses.len(), 1);
12313 assert_eq!(responses[0].header.ty, FrameType::Error);
12314 let error = parse_error(&responses[0]);
12315 assert_eq!(error["code"], "version_unsupported");
12316 assert!(registry.get_module("aft").unwrap().is_none());
12317 assert_eq!(registry.active_registration_count().unwrap(), 0);
12318 }
12319 }
12320
12321 #[test]
12322 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
12323 let registry = Arc::new(Registry::default());
12324 let forwarding = Arc::new(ForwardingTable::default());
12325 let handler =
12326 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12327 let module_connection = ConnectionId::new(301);
12328 let registration = registry
12329 .register_with_control_ops(
12330 manifest("aft-push", PROTOCOL_VERSION),
12331 PROTOCOL_VERSION,
12332 module_connection,
12333 module_baseline_control_ops(),
12334 )
12335 .unwrap();
12336 let (module_tx, _module_rx) = mpsc::channel(8);
12337 let endpoint = forwarding
12338 .register_module_connection(
12339 module_connection,
12340 "aft-push".to_string(),
12341 PROTOCOL_VERSION,
12342 manifest_concurrency(®istration.manifest),
12343 FrameSink::new(module_tx),
12344 )
12345 .unwrap();
12346
12347 let unknown = Frame::build(
12349 FrameType::Push,
12350 control_flags(),
12351 0,
12352 0,
12353 5,
12354 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
12355 )
12356 .unwrap();
12357 let out = handler.handle_status_update(endpoint, unknown).unwrap();
12358 assert!(
12359 out.is_empty(),
12360 "unknown push op must be ignored, got {out:?}"
12361 );
12362
12363 let malformed = Frame::build(
12365 FrameType::Push,
12366 control_flags(),
12367 0,
12368 0,
12369 6,
12370 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
12371 )
12372 .unwrap();
12373 let out = handler.handle_status_update(endpoint, malformed).unwrap();
12374 assert_eq!(out.len(), 1);
12375 assert_eq!(out[0].header.ty, FrameType::Error);
12376 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
12377 }
12378
12379 #[test]
12380 fn hello_rejected_when_connection_already_owns_client_routes() {
12381 let registry = Arc::new(Registry::default());
12382 let forwarding = Arc::new(ForwardingTable::default());
12383 let handler =
12384 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12385 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
12387 let client_connection = ConnectionId::new(202);
12388
12389 let responses = handler
12392 .handle_control(
12393 client_connection,
12394 hello_frame("aft-second", PROTOCOL_VERSION, 9),
12395 )
12396 .unwrap();
12397 assert_eq!(responses[0].header.ty, FrameType::Error);
12398 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
12399 assert!(registry.get_module("aft-second").unwrap().is_none());
12400 }
12401
12402 #[tokio::test]
12403 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
12404 let registry = Arc::new(Registry::default());
12405 let forwarding = Arc::new(ForwardingTable::default());
12406 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
12407 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
12408 hello_via_sink(
12409 &handler,
12410 &module_ctx,
12411 &mut module_rx,
12412 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
12413 )
12414 .await;
12415 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
12416 let pending = forwarding
12417 .begin_route_bind_relay_for_test(
12418 client_ctx.connection_id,
12419 client_ctx.egress.clone(),
12420 2,
12421 "alpha",
12422 )
12423 .unwrap();
12424 forwarding
12425 .complete_pending_relay(
12426 module_ctx.connection_id,
12427 pending.corr,
12428 RouteBindRelayOutcome::Accepted,
12429 )
12430 .unwrap();
12431 client_rx.try_recv().unwrap();
12432 for module_id in ["beta", "alpha"] {
12433 let replies = handler
12434 .handle_control_frame(
12435 &module_ctx,
12436 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
12437 )
12438 .await
12439 .unwrap();
12440 assert_eq!(replies.len(), 1, "second HELLO must be refused");
12441 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
12442 }
12443 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
12444 assert!(registry.get_module("beta").unwrap().is_none());
12445 assert!(matches!(
12446 forwarding
12447 .lookup_data_route(
12448 client_ctx.connection_id,
12449 pending.client_channel,
12450 pending.client_epoch,
12451 )
12452 .unwrap(),
12453 DataRoute::Client(DataRouteState::Bound(_))
12454 ));
12455 assert!(handler
12456 .hello_launch_nonces
12457 .lock()
12458 .unwrap()
12459 .presented(module_ctx.connection_id, Some("alpha-nonce")));
12460 assert!(module_rx.try_recv().is_err());
12461 }
12462
12463 #[test]
12464 fn reserved_module_hello_requires_matching_launch_nonce() {
12465 let registry = Arc::new(Registry::default());
12466 let supervisor = SupervisorHandle::new();
12467 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
12470 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12471
12472 let no_nonce = handler
12474 .handle_control(
12475 ConnectionId::new(1),
12476 hello_frame("vault", PROTOCOL_VERSION, 1),
12477 )
12478 .unwrap();
12479 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
12480 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
12481 assert!(registry.get_module("vault").unwrap().is_none());
12482
12483 let wrong = handler
12485 .handle_control(
12486 ConnectionId::new(2),
12487 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
12488 )
12489 .unwrap();
12490 assert_eq!(wrong[0].header.ty, FrameType::Error);
12491 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
12492 assert!(registry.get_module("vault").unwrap().is_none());
12493
12494 let ok = handler
12496 .handle_control(
12497 ConnectionId::new(3),
12498 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
12499 )
12500 .unwrap();
12501 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
12502 assert!(registry.get_module("vault").unwrap().is_some());
12503 }
12504
12505 #[test]
12506 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
12507 let registry = Arc::new(Registry::default());
12508 let supervisor = SupervisorHandle::new();
12509 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
12510 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
12511 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12512
12513 let squat = handler
12514 .handle_control(
12515 ConnectionId::new(1),
12516 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
12517 )
12518 .unwrap();
12519 assert_eq!(squat[0].header.ty, FrameType::Error);
12520 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
12521 assert!(parse_error(&squat[0])["message"]
12522 .as_str()
12523 .unwrap()
12524 .contains("fed:"));
12525
12526 let accepted_peer = handler
12527 .handle_control(
12528 ConnectionId::new(2),
12529 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
12530 )
12531 .unwrap();
12532 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
12533
12534 let accepted_short = handler
12535 .handle_control(
12536 ConnectionId::new(3),
12537 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
12538 )
12539 .unwrap();
12540 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
12541
12542 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
12543 let response = handler
12544 .handle_control(
12545 ConnectionId::new(conn),
12546 hello_frame(module_id, PROTOCOL_VERSION, conn),
12547 )
12548 .unwrap();
12549 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
12550 }
12551 }
12552
12553 #[test]
12554 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
12555 let registry = Arc::new(Registry::default());
12556 let supervisor = SupervisorHandle::new();
12557 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
12558 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
12559 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
12560 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12561
12562 let owner_nonce = handler
12563 .handle_control(
12564 ConnectionId::new(1),
12565 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
12566 )
12567 .unwrap();
12568 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
12569 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
12570 assert!(registry.get_module("fed:special").unwrap().is_none());
12571
12572 let exact_nonce = handler
12573 .handle_control(
12574 ConnectionId::new(2),
12575 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
12576 )
12577 .unwrap();
12578 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
12579 assert!(registry.get_module("fed:special").unwrap().is_some());
12580 }
12581
12582 #[test]
12583 fn non_reserved_module_ignores_launch_nonce() {
12584 let registry = Arc::new(Registry::default());
12585 let handler = ControlHandler::new(Arc::clone(®istry));
12588 let no_nonce = handler
12589 .handle_control(
12590 ConnectionId::new(1),
12591 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
12592 )
12593 .unwrap();
12594 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
12595 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
12596
12597 let echoed_nonce = handler
12598 .handle_control(
12599 ConnectionId::new(2),
12600 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
12601 )
12602 .unwrap();
12603 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
12604 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
12605 }
12606
12607 #[test]
12608 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
12609 let handler = ControlHandler::default();
12610 let conn = ConnectionId::new(1);
12611 let malformed = Frame::build(
12612 FrameType::Hello,
12613 control_flags(),
12614 0,
12615 0,
12616 3,
12617 b"{not json".to_vec(),
12618 )
12619 .unwrap();
12620
12621 let error = handler.handle_control(conn, malformed).unwrap();
12622 assert_eq!(error[0].header.ty, FrameType::Error);
12623 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
12624
12625 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
12626 let pong = handler.handle_control(conn, ping).unwrap();
12627 assert_eq!(pong[0].header.ty, FrameType::Pong);
12628 assert_eq!(pong[0].header.corr, 4);
12629 }
12630
12631 #[test]
12632 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
12633 let registry = Arc::new(Registry::default());
12634 let handler = ControlHandler::new(Arc::clone(®istry));
12635
12636 handler
12637 .handle_control(
12638 ConnectionId::new(1),
12639 hello_frame("aft", PROTOCOL_VERSION, 1),
12640 )
12641 .unwrap();
12642 let duplicate = handler
12643 .handle_control(
12644 ConnectionId::new(2),
12645 hello_frame("aft", PROTOCOL_VERSION, 2),
12646 )
12647 .unwrap();
12648
12649 assert_eq!(duplicate[0].header.ty, FrameType::Error);
12650 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
12651 let registration = registry.get_module("aft").unwrap().unwrap();
12652 assert_eq!(registration.connection_id, ConnectionId::new(1));
12653 }
12654
12655 #[test]
12656 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
12657 let registry = Arc::new(Registry::default());
12658 let forwarding = Arc::new(ForwardingTable::default());
12659 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
12660 let handler =
12661 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12662 .with_process_liveness(process_liveness);
12663 let (ctx, route_channel, route_epoch) =
12664 bind_liveness_route(®istry, &forwarding, "aft-dead");
12665 let responses = handler
12666 .handle_route_poll(
12667 &ctx,
12668 route_poll_frame(41, PollKind::Liveness, route_channel),
12669 route_channel,
12670 route_epoch,
12671 PollKind::Liveness,
12672 )
12673 .unwrap();
12674
12675 assert_eq!(responses.len(), 1);
12676 assert_eq!(responses[0].header.ty, FrameType::Response);
12677 assert_route_poll_liveness(&responses[0], false);
12678 }
12679
12680 #[test]
12681 fn liveness_poll_without_process_source_uses_bound_route() {
12682 let registry = Arc::new(Registry::default());
12683 let forwarding = Arc::new(ForwardingTable::default());
12684 let handler =
12685 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12686 let (ctx, route_channel, route_epoch) =
12687 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
12688 let responses = handler
12689 .handle_route_poll(
12690 &ctx,
12691 route_poll_frame(42, PollKind::Liveness, route_channel),
12692 route_channel,
12693 route_epoch,
12694 PollKind::Liveness,
12695 )
12696 .unwrap();
12697
12698 assert_route_poll_liveness(&responses[0], true);
12699 }
12700
12701 #[test]
12702 fn liveness_poll_untracked_process_source_uses_bound_route() {
12703 let registry = Arc::new(Registry::default());
12704 let forwarding = Arc::new(ForwardingTable::default());
12705 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
12706 let handler =
12707 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12708 .with_process_liveness(process_liveness);
12709 let (ctx, route_channel, route_epoch) =
12710 bind_liveness_route(®istry, &forwarding, "aft-untracked");
12711 let responses = handler
12712 .handle_route_poll(
12713 &ctx,
12714 route_poll_frame(43, PollKind::Liveness, route_channel),
12715 route_channel,
12716 route_epoch,
12717 PollKind::Liveness,
12718 )
12719 .unwrap();
12720
12721 assert_route_poll_liveness(&responses[0], true);
12722 }
12723
12724 #[tokio::test]
12725 async fn unknown_op_returns_unknown_control_op() {
12726 let handler = ControlHandler::default();
12727 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
12728 let request = Frame::build(
12729 FrameType::Request,
12730 control_flags(),
12731 0,
12732 0,
12733 55,
12734 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
12735 )
12736 .unwrap();
12737
12738 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12739
12740 assert_eq!(response.len(), 1);
12741 assert_eq!(response[0].header.ty, FrameType::Error);
12742 assert_eq!(response[0].header.corr, 55);
12743 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
12744 }
12745
12746 #[tokio::test]
12747 async fn supervisor_provenance_rejects_unknown_exact_module() {
12748 let handler = ControlHandler::default();
12749 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
12750 let request = Frame::build(
12751 FrameType::Request,
12752 control_flags(),
12753 0,
12754 0,
12755 57,
12756 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
12757 )
12758 .unwrap();
12759
12760 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12761
12762 assert_eq!(response.len(), 1);
12763 assert_eq!(response[0].header.ty, FrameType::Error);
12764 assert_eq!(response[0].header.corr, 57);
12765 let error = parse_error(&response[0]);
12766 assert_eq!(error["code"], "unknown_module");
12767 assert_eq!(error["message"], "module_id 'missing' is not supervised");
12768 }
12769
12770 #[test]
12771 fn provenance_probe_override_keeps_handler_tests_deterministic() {
12772 let expected = subc_control::RunningImageAgreement::Unavailable {
12773 reason: subc_control::RunningImageUnavailableReason::HashFailed,
12774 };
12775 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
12776 assert_eq!(handler.provenance_probe_override, Some(expected));
12777 }
12778
12779 #[test]
12780 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
12781 let verdict = reload_verdict(
12782 std::path::Path::new("/bin/new"),
12783 Some(std::path::Path::new("/bin/old")),
12784 subc_control::RunningImageAgreement::Unavailable {
12785 reason: subc_control::RunningImageUnavailableReason::HashFailed,
12786 },
12787 );
12788 assert!(matches!(
12789 verdict.path,
12790 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
12791 if configured == std::path::Path::new("/bin/new")
12792 && spawned_from == std::path::Path::new("/bin/old")
12793 ));
12794 }
12795
12796 #[cfg(windows)]
12797 #[test]
12798 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
12799 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
12800 let configured = dir.join("module.exe");
12801 std::fs::write(&configured, b"executable path fixture").unwrap();
12802 let spawned = std::fs::canonicalize(&configured).unwrap();
12803 assert_ne!(
12804 configured, spawned,
12805 "fixture must exercise different path spellings"
12806 );
12807 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
12808 let lower_case = configured.to_string_lossy().to_lowercase();
12809 for spelling in [
12810 &configured,
12811 Path::new(&forward_slashes),
12812 Path::new(&lower_case),
12813 ] {
12814 let verdict = reload_verdict(
12815 spelling,
12816 Some(&spawned),
12817 subc_control::RunningImageAgreement::Unavailable {
12818 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12819 },
12820 );
12821 assert_eq!(
12822 verdict.path,
12823 ReloadPathAgreement::Match,
12824 "configured {spelling:?}, spawned {spawned:?}"
12825 );
12826 }
12827 }
12828
12829 #[cfg(windows)]
12830 #[test]
12831 fn windows_reload_path_keeps_different_existing_program_pending() {
12832 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
12833 let old = dir.join("old.exe");
12834 let new = dir.join("new.exe");
12835 std::fs::write(&old, b"same executable content").unwrap();
12836 std::fs::copy(&old, &new).unwrap();
12837 let spawned = std::fs::canonicalize(&old).unwrap();
12838 let image = subc_control::RunningImageAgreement::Unavailable {
12839 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12840 };
12841 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
12842 assert!(
12843 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
12844 if configured == new && spawned_from == spawned)
12845 );
12846 let missing = dir.join("missing.exe");
12847 assert!(matches!(
12848 reload_verdict(&missing, Some(&spawned), image).path,
12849 ReloadPathAgreement::Mismatch { .. }
12850 ));
12851 }
12852
12853 #[test]
12854 fn reload_verdict_detects_replaced_image_at_same_path() {
12855 let image = subc_control::RunningImageAgreement::Mismatch {
12856 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
12857 digest: "old".into(),
12858 },
12859 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
12860 digest: "new".into(),
12861 },
12862 };
12863 let verdict = reload_verdict(
12864 std::path::Path::new("/bin/same"),
12865 Some(std::path::Path::new("/bin/same")),
12866 image.clone(),
12867 );
12868 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
12869 assert_eq!(verdict.image, image);
12870 }
12871
12872 #[test]
12873 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
12874 let image = subc_control::RunningImageAgreement::Unavailable {
12875 reason: subc_control::RunningImageUnavailableReason::NotRunning,
12876 };
12877 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
12878 assert_eq!(
12879 verdict.path,
12880 subc_control::ReloadPathAgreement::Unavailable {
12881 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
12882 }
12883 );
12884 assert_eq!(verdict.image, image);
12885
12886 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
12887 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12888 };
12889 let verdict = reload_verdict(
12890 std::path::Path::new("/bin/same"),
12891 Some(std::path::Path::new("/bin/same")),
12892 unconfirmed.clone(),
12893 );
12894 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
12895 assert_eq!(verdict.image, unconfirmed);
12896 }
12897
12898 #[test]
12899 fn reload_verdict_preserves_each_image_unavailability_reason() {
12900 use subc_control::RunningImageUnavailableReason as Reason;
12901
12902 for reason in [
12903 Reason::NotRunning,
12904 Reason::UnsupportedPlatform,
12905 Reason::RunningExecutableUnreadable,
12906 Reason::SpawnedPathUnreadable,
12907 Reason::HashFailed,
12908 Reason::ProcessIdentityUnconfirmed,
12909 Reason::Unknown("future_probe_reason".to_string()),
12910 ] {
12911 let image = subc_control::RunningImageAgreement::Unavailable {
12912 reason: reason.clone(),
12913 };
12914 let verdict = reload_verdict(
12915 std::path::Path::new("/bin/same"),
12916 Some(std::path::Path::new("/bin/same")),
12917 image.clone(),
12918 );
12919 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
12920 assert_eq!(verdict.image, image, "{reason:?}");
12921 }
12922 }
12923
12924 #[tokio::test]
12925 async fn malformed_control_bodies_return_invalid_control_body() {
12926 let handler = ControlHandler::default();
12927 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
12928
12929 for (corr, body) in [
12930 (56, br#"{"route_channel":1}"#.as_slice()),
12931 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
12932 (
12933 58,
12934 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
12935 ),
12936 ] {
12937 let request = Frame::build(
12938 FrameType::Request,
12939 control_flags(),
12940 0,
12941 0,
12942 corr,
12943 body.to_vec(),
12944 )
12945 .unwrap();
12946 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12947
12948 assert_eq!(response.len(), 1);
12949 assert_eq!(response[0].header.ty, FrameType::Error);
12950 assert_eq!(response[0].header.corr, corr);
12951 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
12952 }
12953 }
12954
12955 #[tokio::test]
12956 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
12957 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12958 let registry = Arc::new(Registry::default());
12959 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
12960 let router = Router::with_control_handler(Arc::clone(&control));
12961 let connection = router.begin_connection();
12962 let (ctx, mut rx) = route_ctx(connection.id());
12963
12964 router
12965 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
12966 .await
12967 .unwrap();
12968 let response = rx.recv().await.unwrap();
12969 let ack = parse_ack(&response);
12970 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
12971 let channel = 1;
12972
12973 let goodbye =
12974 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
12975 router.route_for_connection(&ctx, goodbye).await.unwrap();
12976 assert!(rx.try_recv().is_err());
12977 assert!(registry.get_module("aft").unwrap().is_none());
12978
12979 router
12980 .route_for_connection(&ctx, channel_request(channel, 13))
12981 .await
12982 .unwrap();
12983 let error_frame = rx.recv().await.unwrap();
12984 assert_eq!(error_frame.header.ty, FrameType::Error);
12985 assert_eq!(error_frame.header.channel, channel);
12986 let captured = crate::router::test_log::captured_logs(&logs);
12987 assert_eq!(
12988 captured
12989 .lines()
12990 .filter(|line| {
12991 line.contains("module_id=aft")
12992 && line.contains("reason=explicit_goodbye")
12993 && line.contains("module registration ended")
12994 })
12995 .count(),
12996 1,
12997 "unexpected GOODBYE registry log: {captured}"
12998 );
12999 }
13000
13001 #[tokio::test]
13002 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
13003 let registry = Arc::new(Registry::default());
13004 let handler = ControlHandler::new(registry).with_capability_config(
13005 [("prov".to_string(), true), ("cons".to_string(), true)],
13006 BTreeMap::new(),
13007 );
13008 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
13009 register_capability_manifest(
13010 &handler,
13011 &provider_ctx,
13012 &mut provider_rx,
13013 capability_manifest("prov", &["thing/v1"], &[]),
13014 1,
13015 )
13016 .await;
13017 let mut consumer = capability_manifest("cons", &[], &[]);
13018 consumer.capabilities.as_mut().unwrap().requires.push(
13019 subc_protocol::manifest::CapabilityRequirement {
13020 capability: "thing/v1".to_string(),
13021 need: subc_protocol::manifest::CapabilityNeed::Required,
13022 },
13023 );
13024 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
13025 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
13026 assert_eq!(
13027 handler.capability_evaluator.verdict("cons", "thing/v1"),
13028 Some(CapabilityVerdict::Provided)
13029 );
13030 let (mut client_rx, _) = open_route_for_capability_test(
13031 &handler,
13032 &provider_ctx,
13033 &mut provider_rx,
13034 703,
13035 3,
13036 "prov",
13037 None,
13038 )
13039 .await;
13040 let goodbye =
13041 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13042 handler
13043 .handle_control_frame(&provider_ctx, goodbye)
13044 .await
13045 .unwrap();
13046 assert_eq!(
13047 handler.capability_evaluator.verdict("cons", "thing/v1"),
13048 Some(CapabilityVerdict::NeverProvided)
13049 );
13050 let closed = client_rx
13051 .try_recv()
13052 .expect("GOODBYE pushes route.closed before route GOODBYE");
13053 assert!(
13054 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
13055 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
13056 );
13057 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
13058 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
13059 }
13060
13061 struct RouteClosedFixture {
13062 handler: ControlHandler,
13063 #[cfg(unix)]
13064 supervisor: Supervisor,
13065 module_ctx: RouteCtx,
13066 #[cfg(unix)]
13067 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13068 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13069 route: Arc<crate::forwarding::RouteBinding>,
13070 }
13071
13072 async fn route_closed_fixture() -> RouteClosedFixture {
13073 let registry = Arc::new(Registry::default());
13074 let handler = ControlHandler::new(Arc::clone(®istry));
13075 #[cfg(unix)]
13076 let supervisor = Supervisor::new(registry, RestartPolicy::default())
13077 .with_forwarding(Arc::clone(&handler.forwarding));
13078 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
13079 register_capability_manifest(
13080 &handler,
13081 &module_ctx,
13082 &mut module_rx,
13083 capability_manifest("prov", &[], &[]),
13084 1,
13085 )
13086 .await;
13087 let (client_rx, bind) = open_route_for_capability_test(
13088 &handler,
13089 &module_ctx,
13090 &mut module_rx,
13091 702,
13092 2,
13093 "prov",
13094 None,
13095 )
13096 .await;
13097 let ModuleControlRequest::RouteBind {
13098 route_channel: channel,
13099 epoch: route_epoch,
13100 ..
13101 } = bind
13102 else {
13103 panic!("expected route.bind");
13104 };
13105 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
13106 handler
13107 .forwarding
13108 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
13109 .unwrap()
13110 else {
13111 panic!("the acknowledged route must be bound");
13112 };
13113 RouteClosedFixture {
13114 handler,
13115 #[cfg(unix)]
13116 supervisor,
13117 module_ctx,
13118 #[cfg(unix)]
13119 module_rx,
13120 client_rx,
13121 route,
13122 }
13123 }
13124
13125 impl RouteClosedFixture {
13126 #[cfg(unix)]
13127 async fn cut(&mut self) {
13128 let notices = async {
13129 let mut draining = self.module_rx.recv().await.unwrap();
13130 assert!(matches!(
13131 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
13132 &draining.body
13133 )
13134 .unwrap(),
13135 subc_protocol::session::ModuleControlCommand::Draining {
13136 reason: RouteCloseReason::Restart,
13137 ..
13138 }
13139 ));
13140 draining.flushed.take().unwrap().send(()).unwrap();
13141 let mut closing = self.client_rx.recv().await.unwrap();
13142 assert!(matches!(
13143 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
13144 ClientControlPush::RouteClosing {
13145 reason: RouteCloseReason::Restart,
13146 ..
13147 }
13148 ));
13149 closing.flushed.take().unwrap().send(()).unwrap();
13150 };
13151 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
13152 result.unwrap();
13153 }
13154
13155 fn disconnect(&mut self) -> ClientControlPush {
13156 self.handler
13157 .cleanup_connection(self.module_ctx.connection_id)
13158 .unwrap();
13159 let closed = self.client_rx.try_recv().unwrap();
13160 assert_eq!(closed.header.ty, FrameType::Push);
13161 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
13162 assert!(matches!(
13163 &push,
13164 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
13165 if channels == &[self.route.client_channel]
13166 ));
13167 assert_eq!(
13168 self.client_rx.try_recv().unwrap().header.ty,
13169 FrameType::Goodbye
13170 );
13171 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
13172 push
13173 }
13174 }
13175
13176 #[cfg(unix)]
13177 #[tokio::test(start_paused = true)]
13178 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
13179 let mut fixture = route_closed_fixture().await;
13180 fixture.cut().await;
13181 assert!(matches!(
13182 fixture.disconnect(),
13183 ClientControlPush::RouteClosed {
13184 reason: RouteCloseReason::Restart,
13185 drained: true,
13186 ..
13187 }
13188 ));
13189 }
13190
13191 #[cfg(unix)]
13192 #[tokio::test(start_paused = true)]
13193 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
13194 let mut fixture = route_closed_fixture().await;
13195 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
13196 let started = tokio::time::Instant::now();
13197 fixture.cut().await;
13198 assert!(started.elapsed() >= Duration::from_secs(2));
13199 assert_eq!(fixture.route.flow.in_flight(), 1);
13200 assert!(matches!(
13201 fixture.disconnect(),
13202 ClientControlPush::RouteClosed {
13203 reason: RouteCloseReason::Restart,
13204 drained: false,
13205 ..
13206 }
13207 ));
13208 }
13209
13210 #[tokio::test]
13211 async fn module_connection_crash_reports_crash_and_not_drained() {
13212 let mut fixture = route_closed_fixture().await;
13213 assert!(matches!(
13217 fixture.disconnect(),
13218 ClientControlPush::RouteClosed {
13219 reason: RouteCloseReason::Crash,
13220 drained: false,
13221 ..
13222 }
13223 ));
13224 }
13225
13226 #[tokio::test]
13227 async fn dropping_router_connection_releases_registration() {
13228 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13229 let registry = Arc::new(Registry::default());
13230 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
13231 let router = Router::with_control_handler(Arc::clone(&control));
13232 let connection = router.begin_connection();
13233 let connection_id = connection.id();
13234 let (ctx, mut rx) = route_ctx(connection_id);
13235
13236 router
13237 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
13238 .await
13239 .unwrap();
13240 let response = rx.recv().await.unwrap();
13241 let ack = parse_ack(&response);
13242 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13243 assert!(registry.get_module("aft").unwrap().is_some());
13244
13245 drop(connection);
13246
13247 assert!(registry.get_module("aft").unwrap().is_none());
13248 assert_eq!(registry.active_registration_count().unwrap(), 0);
13249
13250 control
13251 .cleanup_connection(ConnectionId::new(u64::MAX))
13252 .unwrap();
13253 let captured = crate::router::test_log::captured_logs(&logs);
13254 println!("captured registry lifecycle logs:\n{captured}");
13255 let events: Vec<_> = captured
13256 .lines()
13257 .filter(|line| {
13258 line.contains("module registered module_id=aft ")
13259 || line.contains("module registration ended")
13260 })
13261 .collect();
13262 assert_eq!(
13263 events.len(),
13264 2,
13265 "unexpected registry lifecycle logs: {captured}"
13266 );
13267 assert!(events[0].contains("module registered module_id=aft "));
13268 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
13269 assert!(events[1].contains(&format!(
13270 "module_id=aft connection_id={}",
13271 connection_id.get()
13272 )));
13273 assert!(events[1].contains("reason=connection_closed"));
13274 assert!(events[1].contains("module registration ended"));
13275 assert_eq!(
13276 captured
13277 .lines()
13278 .filter(|line| line.contains("module registered module_id=aft "))
13279 .count(),
13280 1,
13281 "legacy registration admission line must appear once: {captured}"
13282 );
13283 }
13284
13285 #[tokio::test]
13286 async fn hello_registration_keeps_legacy_module_registered_line_once() {
13287 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13288 let registry = Arc::new(Registry::default());
13289 let control = ControlHandler::new(Arc::clone(®istry));
13290 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
13291 hello_via_sink(
13292 &control,
13293 &ctx,
13294 &mut rx,
13295 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
13296 )
13297 .await;
13298
13299 let captured = crate::router::test_log::captured_logs(&logs);
13300 let admissions: Vec<_> = captured
13301 .lines()
13302 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
13303 .collect();
13304 assert_eq!(
13305 admissions.len(),
13306 1,
13307 "legacy admission line must remain exactly once: {captured}"
13308 );
13309 assert!(admissions[0].contains("routable_provider=true"));
13310 assert!(admissions[0].contains("connection_id=777"));
13311 }
13312
13313 fn capability_manifest(
13314 module_id: &str,
13315 provides: &[&str],
13316 must_never_reach: &[&str],
13317 ) -> ModuleManifest {
13318 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
13319 manifest.capabilities = Some(CapabilityDeclarations {
13320 provides: provides
13321 .iter()
13322 .map(|capability| (*capability).to_string())
13323 .collect(),
13324 requires: Vec::new(),
13325 must_never_reach: must_never_reach
13326 .iter()
13327 .map(|capability| (*capability).to_string())
13328 .collect(),
13329 });
13330 manifest
13331 }
13332
13333 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
13334 Frame::build(
13335 FrameType::Hello,
13336 control_flags(),
13337 0,
13338 0,
13339 corr,
13340 serde_json::to_vec(&ModuleHelloBody {
13341 protocol_ver: manifest.protocol_ver,
13342 manifest,
13343 control_ops: None,
13344 launch_nonce: None,
13345 })
13346 .expect("capability test HELLO serializes"),
13347 )
13348 .expect("capability test HELLO frame builds")
13349 }
13350
13351 fn catalog_update_with_capabilities_frame(
13352 corr: u64,
13353 capabilities: CapabilityDeclarations,
13354 ) -> Frame {
13355 Frame::build(
13356 FrameType::Request,
13357 control_flags(),
13358 0,
13359 0,
13360 corr,
13361 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
13362 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
13363 capabilities: Some(capabilities),
13364 ready: None,
13365 })
13366 .expect("capability catalog.update serializes"),
13367 )
13368 .expect("capability catalog.update frame builds")
13369 }
13370
13371 async fn register_capability_manifest(
13372 handler: &ControlHandler,
13373 ctx: &RouteCtx,
13374 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13375 manifest: ModuleManifest,
13376 corr: u64,
13377 ) {
13378 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
13379 }
13380
13381 async fn open_route_for_capability_test(
13382 handler: &ControlHandler,
13383 target_ctx: &RouteCtx,
13384 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13385 client_connection_id: u64,
13386 corr: u64,
13387 target_module_id: &str,
13388 consumer_identity: Option<ConsumerIdentity>,
13389 ) -> (
13390 mpsc::Receiver<crate::router::OutboundFrame>,
13391 ModuleControlRequest,
13392 ) {
13393 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
13394 let route_handler = handler.clone();
13395 let target_module_id = target_module_id.to_string();
13396 let route_task = tokio::spawn(async move {
13397 route_handler
13398 .handle_control_frame(
13399 &client_ctx,
13400 route_open_frame_with_admission_facts(
13401 corr,
13402 &target_module_id,
13403 unique_project_root("admission-facts"),
13404 consumer_identity,
13405 None,
13406 ),
13407 )
13408 .await
13409 .expect("capability test route.open succeeds")
13410 });
13411 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
13412 .await
13413 .expect("capability test route.open must reach route.bind")
13414 .expect("target control receiver stays open");
13415 let bind_request: ModuleControlRequest =
13416 serde_json::from_slice(&bind.body).expect("route.bind decodes");
13417 handler
13418 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
13419 .await
13420 .expect("capability test route.bind ACK succeeds");
13421 assert!(route_task.await.expect("route.open task joins").is_empty());
13422 let opened = client_rx
13423 .recv()
13424 .await
13425 .expect("successful route.open publishes a response");
13426 assert!(matches!(
13427 serde_json::from_slice::<ClientControlResponse>(&opened.body),
13428 Ok(ClientControlResponse::RouteOpen { .. })
13429 ));
13430 (client_rx, bind_request)
13431 }
13432
13433 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
13434 assert_eq!(frame.header.ty, FrameType::Push);
13435 assert_eq!(frame.header.channel, 0);
13436 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
13437 .expect("route.closed control push decodes");
13438 let ClientControlPush::RouteClosed { channels, .. } = &push else {
13439 panic!("expected route.closed");
13440 };
13441 assert_eq!(channels.len(), 1, "exactly one violating route closed");
13442 let channels = channels.clone();
13443 assert_eq!(
13444 push,
13445 ClientControlPush::RouteClosed {
13446 module_id: target_module_id.to_string(),
13447 channels,
13448 reason: RouteCloseReason::CapabilityDenied,
13449 drained: false,
13450 abandoned: 0,
13451 excluded_subscriptions: 0,
13452 terminal: Some(false),
13453 }
13454 );
13455 }
13456
13457 #[tokio::test]
13458 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
13459 let registry = Arc::new(Registry::default());
13460 let forwarding = Arc::new(ForwardingTable::default());
13461 let supervisor = SupervisorHandle::new();
13462 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13463 let handler =
13464 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13465 .with_supervisor(supervisor);
13466 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
13467 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
13468 register_capability_manifest(
13469 &handler,
13470 &target_ctx,
13471 &mut target_rx,
13472 capability_manifest("target", &["credentials-provider/v1"], &[]),
13473 1,
13474 )
13475 .await;
13476 register_capability_manifest(
13477 &handler,
13478 &opener_ctx,
13479 &mut opener_rx,
13480 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13481 2,
13482 )
13483 .await;
13484
13485 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
13486 let replies = handler
13487 .handle_control_frame(
13488 &client_ctx,
13489 route_open_frame_with_admission_facts(
13490 3,
13491 "target",
13492 unique_project_root("admission-facts"),
13493 Some(ConsumerIdentity {
13494 module_id: "opener".to_string(),
13495 launch_nonce: "opener-nonce".to_string(),
13496 }),
13497 None,
13498 ),
13499 )
13500 .await
13501 .expect("denied route.open returns a typed frame");
13502 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
13503 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13504 assert!(
13505 target_rx.try_recv().is_err(),
13506 "forbidden route.open must not relay route.bind"
13507 );
13508 }
13509
13510 #[tokio::test]
13511 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
13512 let registry = Arc::new(Registry::default());
13513 let forwarding = Arc::new(ForwardingTable::default());
13514 let supervisor = SupervisorHandle::new();
13515 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13516 let handler =
13517 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13518 .with_supervisor(supervisor);
13519 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
13520 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
13521 register_capability_manifest(
13522 &handler,
13523 &target_ctx,
13524 &mut target_rx,
13525 capability_manifest("target", &["credentials-provider/v1"], &[]),
13526 1,
13527 )
13528 .await;
13529 register_capability_manifest(
13530 &handler,
13531 &old_opener_ctx,
13532 &mut old_opener_rx,
13533 capability_manifest("opener", &[], &[]),
13534 2,
13535 )
13536 .await;
13537 let (mut client_rx, _) = open_route_for_capability_test(
13538 &handler,
13539 &target_ctx,
13540 &mut target_rx,
13541 712,
13542 3,
13543 "target",
13544 Some(ConsumerIdentity {
13545 module_id: "opener".to_string(),
13546 launch_nonce: "opener-nonce".to_string(),
13547 }),
13548 )
13549 .await;
13550 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13551
13552 handler
13553 .cleanup_connection(old_opener_ctx.connection_id)
13554 .expect("old opener registration cleans up");
13555 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
13556 register_capability_manifest(
13557 &handler,
13558 &new_opener_ctx,
13559 &mut new_opener_rx,
13560 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13561 4,
13562 )
13563 .await;
13564
13565 assert_capability_denied_push(
13566 client_rx
13567 .try_recv()
13568 .expect("HELLO deny addition must emit route.closed")
13569 .frame,
13570 "target",
13571 );
13572 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13573 assert!(matches!(
13574 target_rx.try_recv(),
13575 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
13576 ));
13577 }
13578
13579 #[tokio::test]
13580 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
13581 let registry = Arc::new(Registry::default());
13582 let forwarding = Arc::new(ForwardingTable::default());
13583 let supervisor = SupervisorHandle::new();
13584 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13585 let handler =
13586 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13587 .with_supervisor(supervisor);
13588 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
13589 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
13590 register_capability_manifest(
13591 &handler,
13592 &target_ctx,
13593 &mut target_rx,
13594 capability_manifest("target", &[], &[]),
13595 1,
13596 )
13597 .await;
13598 register_capability_manifest(
13599 &handler,
13600 &opener_ctx,
13601 &mut opener_rx,
13602 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13603 2,
13604 )
13605 .await;
13606 let (mut client_rx, _) = open_route_for_capability_test(
13607 &handler,
13608 &target_ctx,
13609 &mut target_rx,
13610 722,
13611 3,
13612 "target",
13613 Some(ConsumerIdentity {
13614 module_id: "opener".to_string(),
13615 launch_nonce: "opener-nonce".to_string(),
13616 }),
13617 )
13618 .await;
13619 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13620
13621 let replies = handler
13622 .handle_control_frame(
13623 &target_ctx,
13624 catalog_update_with_capabilities_frame(
13625 4,
13626 CapabilityDeclarations {
13627 provides: vec!["credentials-provider/v1".to_string()],
13628 requires: Vec::new(),
13629 must_never_reach: Vec::new(),
13630 },
13631 ),
13632 )
13633 .await
13634 .expect("claim catalog.update succeeds");
13635 assert!(matches!(
13636 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
13637 Ok(ModuleControlResponseToModule::CatalogUpdate {})
13638 ));
13639 assert_capability_denied_push(
13640 client_rx
13641 .try_recv()
13642 .expect("claim addition must emit route.closed")
13643 .frame,
13644 "target",
13645 );
13646 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13647 assert!(matches!(
13648 target_rx.try_recv(),
13649 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
13650 ));
13651 }
13652
13653 #[tokio::test]
13654 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
13655 let registry = Arc::new(Registry::default());
13656 let forwarding = Arc::new(ForwardingTable::default());
13657 let supervisor = SupervisorHandle::new();
13658 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13659 let handler =
13660 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13661 .with_supervisor(supervisor);
13662 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
13663 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
13664 register_capability_manifest(
13665 &handler,
13666 &target_ctx,
13667 &mut target_rx,
13668 capability_manifest("target", &["credentials-provider/v1"], &[]),
13669 1,
13670 )
13671 .await;
13672 register_capability_manifest(
13673 &handler,
13674 &opener_ctx,
13675 &mut opener_rx,
13676 capability_manifest("opener", &[], &[]),
13677 2,
13678 )
13679 .await;
13680 let (mut client_rx, _) = open_route_for_capability_test(
13681 &handler,
13682 &target_ctx,
13683 &mut target_rx,
13684 732,
13685 3,
13686 "target",
13687 Some(ConsumerIdentity {
13688 module_id: "opener".to_string(),
13689 launch_nonce: "opener-nonce".to_string(),
13690 }),
13691 )
13692 .await;
13693 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13694
13695 handler
13696 .handle_control_frame(
13697 &target_ctx,
13698 catalog_update_with_capabilities_frame(
13699 4,
13700 CapabilityDeclarations {
13701 provides: Vec::new(),
13702 requires: Vec::new(),
13703 must_never_reach: Vec::new(),
13704 },
13705 ),
13706 )
13707 .await
13708 .expect("claim removal catalog.update succeeds");
13709 assert_eq!(
13710 forwarding.active_binding_count().unwrap(),
13711 1,
13712 "removing an attested target claim must leave the route census unchanged"
13713 );
13714 assert!(
13715 client_rx.try_recv().is_err(),
13716 "claim removal must not emit route.closed capability_denied"
13717 );
13718 assert!(
13719 target_rx.try_recv().is_err(),
13720 "claim removal must not send the target a route GOODBYE"
13721 );
13722 }
13723
13724 #[tokio::test]
13727 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
13728 let registry = Arc::new(Registry::default());
13729 let forwarding = Arc::new(ForwardingTable::default());
13730 let supervisor = SupervisorHandle::new();
13731 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13732 let handler =
13733 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13734 .with_supervisor(supervisor);
13735 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
13736 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
13737 register_capability_manifest(
13738 &handler,
13739 &target_ctx,
13740 &mut target_rx,
13741 capability_manifest("target", &["credentials-provider/v1"], &[]),
13742 1,
13743 )
13744 .await;
13745 register_capability_manifest(
13746 &handler,
13747 &opener_ctx,
13748 &mut opener_rx,
13749 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13750 2,
13751 )
13752 .await;
13753
13754 let (_client_rx, bind) = open_route_for_capability_test(
13755 &handler,
13756 &target_ctx,
13757 &mut target_rx,
13758 742,
13759 3,
13760 "target",
13761 None,
13762 )
13763 .await;
13764 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
13765 panic!("direct scope-honesty route must bind");
13766 };
13767 assert_eq!(principal, Some(Principal::Direct));
13768 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13769 }
13770
13771 #[tokio::test]
13774 async fn must_never_reach_self_route_is_capability_forbidden() {
13775 let registry = Arc::new(Registry::default());
13776 let forwarding = Arc::new(ForwardingTable::default());
13777 let supervisor = SupervisorHandle::new();
13778 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
13779 let handler =
13780 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13781 .with_supervisor(supervisor);
13782 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
13783 register_capability_manifest(
13784 &handler,
13785 &self_ctx,
13786 &mut self_rx,
13787 capability_manifest(
13788 "self-provider",
13789 &["credentials-provider/v1"],
13790 &["credentials-provider/v1"],
13791 ),
13792 1,
13793 )
13794 .await;
13795
13796 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
13797 let replies = handler
13798 .handle_control_frame(
13799 &client_ctx,
13800 route_open_frame_with_admission_facts(
13801 2,
13802 "self-provider",
13803 unique_project_root("admission-facts"),
13804 Some(ConsumerIdentity {
13805 module_id: "self-provider".to_string(),
13806 launch_nonce: "self-nonce".to_string(),
13807 }),
13808 None,
13809 ),
13810 )
13811 .await
13812 .expect("self-route refusal returns a typed frame");
13813 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
13814 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13815 assert!(
13816 self_rx.try_recv().is_err(),
13817 "self denial must not relay route.bind"
13818 );
13819 }
13820
13821 #[test]
13822 fn unsupported_channel_zero_frame_returns_error() {
13823 let handler = ControlHandler::default();
13824 let request = Frame::build(
13825 FrameType::Request,
13826 control_flags(),
13827 0,
13828 0,
13829 21,
13830 b"opaque".to_vec(),
13831 )
13832 .unwrap();
13833
13834 let response = handler
13835 .handle_control(ConnectionId::new(1), request)
13836 .unwrap();
13837
13838 assert_eq!(response[0].header.ty, FrameType::Error);
13839 assert_eq!(
13840 parse_error(&response[0])["code"],
13841 "unsupported_control_frame"
13842 );
13843 }
13844
13845 mod swap {
13850 use super::*;
13851
13852 const INCUMBENT: ConnectionId = ConnectionId::new(30);
13853 const CANDIDATE: ConnectionId = ConnectionId::new(40);
13854
13855 struct Swap {
13856 registry: Arc<Registry>,
13857 forwarding: Arc<ForwardingTable>,
13858 handler: ControlHandler,
13859 incumbent_ctx: RouteCtx,
13860 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13861 candidate_ctx: RouteCtx,
13862 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13863 }
13864
13865 async fn swap_with_incumbent() -> Swap {
13866 let registry = Arc::new(Registry::default());
13867 let forwarding = Arc::new(ForwardingTable::default());
13868 let handler =
13869 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13870 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
13871 hello_via_sink(
13872 &handler,
13873 &incumbent_ctx,
13874 &mut incumbent_rx,
13875 hello_frame("aft", PROTOCOL_VERSION, 7),
13876 )
13877 .await;
13878 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
13879 Swap {
13880 registry,
13881 forwarding,
13882 handler,
13883 incumbent_ctx,
13884 incumbent_rx,
13885 candidate_ctx,
13886 candidate_rx,
13887 }
13888 }
13889
13890 fn register_candidate(swap: &Swap, ready: Option<bool>) {
13891 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
13892 candidate_manifest.ready = ready;
13893 let registration = swap
13894 .registry
13895 .register_candidate_with_control_ops(
13896 candidate_manifest,
13897 PROTOCOL_VERSION,
13898 CANDIDATE,
13899 module_baseline_control_ops(),
13900 )
13901 .unwrap();
13902 swap.forwarding
13903 .register_candidate_module_connection(
13904 CANDIDATE,
13905 "aft".to_string(),
13906 PROTOCOL_VERSION,
13907 manifest_concurrency(®istration.manifest),
13908 swap.candidate_ctx.egress.clone(),
13909 )
13910 .unwrap();
13911 }
13912
13913 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
13914 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
13915 swap.registry.promote_candidate("aft").unwrap().unwrap();
13916 cutover.incumbent.unwrap()
13917 }
13918
13919 #[tokio::test]
13928 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
13929 let mut swap = swap_with_incumbent().await;
13930 let hold = Arc::new(std::sync::Barrier::new(2));
13931 let handler = swap
13932 .handler
13933 .clone()
13934 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
13935 handler
13936 .supervisor
13937 .open_swap("aft", "candidate-nonce".to_string());
13938 let hello = {
13939 let handler = handler.clone();
13940 let candidate_ctx = swap.candidate_ctx.clone();
13941 std::thread::spawn(move || {
13942 handler
13943 .handle_hello(
13944 candidate_ctx.connection_id,
13945 Some(candidate_ctx.egress.clone()),
13946 hello_frame_with_nonce(
13947 "aft",
13948 PROTOCOL_VERSION,
13949 8,
13950 Some("candidate-nonce"),
13951 ),
13952 )
13953 .unwrap()
13954 })
13955 };
13956
13957 hold.wait();
13958 assert!(swap
13961 .registry
13962 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
13963 .unwrap()
13964 .is_some());
13965 assert!(swap
13966 .registry
13967 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
13968 .unwrap()
13969 .is_none());
13970 assert!(swap
13971 .forwarding
13972 .module_endpoint_for_connection(CANDIDATE)
13973 .unwrap()
13974 .is_none());
13975
13976 hold.wait();
13977 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
13978 assert_eq!(
13979 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
13980 FrameType::HelloAck
13981 );
13982 let published = swap
13983 .registry
13984 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
13985 .unwrap()
13986 .expect("candidate is published once its HELLO completes");
13987 assert_eq!(published.connection_id, CANDIDATE);
13988 assert!(swap
13989 .forwarding
13990 .module_endpoint_for_connection(CANDIDATE)
13991 .unwrap()
13992 .is_some());
13993 }
13994
13995 fn keyed_total(counters: &Value, key: &str) -> u64 {
13996 counters[key]
13997 .as_object()
13998 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
13999 .unwrap_or(0)
14000 }
14001
14002 #[tokio::test]
14003 async fn replacement_logs_old_end_and_new_admission_once() {
14004 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14005 let mut swap = swap_with_incumbent().await;
14006 swap.handler
14007 .supervisor
14008 .open_swap("aft", "candidate-nonce".to_string());
14009 hello_via_sink(
14010 &swap.handler,
14011 &swap.candidate_ctx,
14012 &mut swap.candidate_rx,
14013 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
14014 )
14015 .await;
14016 cutover(&swap);
14017 swap.handler.cleanup_connection(INCUMBENT).unwrap();
14018
14019 let captured = crate::router::test_log::captured_logs(&logs);
14020 println!("captured replacement registry logs:\n{captured}");
14021 let new_admissions: Vec<_> = captured
14022 .lines()
14023 .filter(|line| {
14024 line.contains("connection_id=40")
14025 && line.contains("swap candidate registered; not routable until cutover")
14026 })
14027 .collect();
14028 assert_eq!(
14029 new_admissions.len(),
14030 1,
14031 "unexpected admission logs: {captured}"
14032 );
14033 assert!(new_admissions[0].contains("module_id=aft"));
14034 assert!(new_admissions[0].contains("connection_id=40"));
14035 assert!(
14036 new_admissions[0].contains("swap candidate registered; not routable until cutover")
14037 );
14038 assert!(new_admissions[0].contains("ready=true"));
14039
14040 let old_admissions: Vec<_> = captured
14041 .lines()
14042 .filter(|line| {
14043 line.contains("module registered module_id=aft ")
14044 && line.contains("connection_id=30")
14045 })
14046 .collect();
14047 assert_eq!(
14048 old_admissions.len(),
14049 1,
14050 "unexpected incumbent admission: {captured}"
14051 );
14052
14053 let old_ends: Vec<_> = captured
14054 .lines()
14055 .filter(|line| {
14056 line.contains("connection_id=30") && line.contains("module registration ended")
14057 })
14058 .collect();
14059 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
14060 assert!(old_ends[0].contains("module_id=aft"));
14061 assert!(old_ends[0].contains("connection_id=30"));
14062 assert!(old_ends[0].contains("reason=replaced"));
14063 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
14064 assert_eq!(
14065 captured
14066 .lines()
14067 .filter(|line| line.contains("module registration promoted"))
14068 .count(),
14069 1,
14070 "unexpected promotion logs: {captured}"
14071 );
14072 }
14073
14074 #[tokio::test]
14080 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
14081 let mut swap = swap_with_incumbent().await;
14082 let handler = swap.handler.clone();
14083
14084 let cotenant = ConnectionId::new(31);
14086 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
14087 let (cotenant_task, cotenant_bind) = relay_route_open(
14088 &handler,
14089 cotenant,
14090 &cotenant_ctx.egress,
14091 &mut swap.incumbent_rx,
14092 100,
14093 "aft",
14094 "swap-cotenant",
14095 )
14096 .await;
14097 handler
14098 .handle_control_frame(
14099 &swap.incumbent_ctx,
14100 route_bind_ack(cotenant_bind.header.corr),
14101 )
14102 .await
14103 .unwrap();
14104 assert!(cotenant_task.await.unwrap().is_empty());
14105 let (cotenant_channel, cotenant_epoch) =
14106 published_route(&cotenant_rx.recv().await.unwrap());
14107
14108 let caller = ConnectionId::new(32);
14110 let (caller_ctx, mut caller_rx) = route_ctx(caller);
14111 let (caller_task, caller_bind) = relay_route_open(
14112 &handler,
14113 caller,
14114 &caller_ctx.egress,
14115 &mut swap.incumbent_rx,
14116 101,
14117 "aft",
14118 "swap-caller",
14119 )
14120 .await;
14121 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
14122
14123 register_candidate(&swap, None);
14124 cutover(&swap);
14125
14126 let ack = handler
14128 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
14129 .await;
14130 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
14131 if module_loop_error.is_some() {
14132 handler.cleanup_connection(INCUMBENT).unwrap();
14135 }
14136
14137 assert!(
14139 cotenant_rx.try_recv().is_err(),
14140 "the co-tenant route on the incumbent was torn down by one late ack: \
14141 {module_loop_error:?}"
14142 );
14143 assert!(matches!(
14144 swap.forwarding
14145 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
14146 .unwrap(),
14147 DataRoute::Client(DataRouteState::Bound(_))
14148 ));
14149 assert_eq!(module_loop_error, None);
14150 assert!(swap
14151 .registry
14152 .get_module_by_connection(INCUMBENT)
14153 .unwrap()
14154 .is_some());
14155
14156 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
14158 .await
14159 .expect("the incumbent is told to drop the abandoned binding")
14160 .unwrap()
14161 .frame;
14162 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14163 assert_eq!(goodbye.header.channel, abandoned_channel);
14164 assert_eq!(goodbye.header.epoch, abandoned_epoch);
14165 assert!(swap.incumbent_rx.try_recv().is_err());
14166
14167 let response = caller_task.await.unwrap();
14169 assert_eq!(response.len(), 1);
14170 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14171 assert!(caller_rx.try_recv().is_err());
14172
14173 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
14177 let counters = handler.counters().snapshot();
14178 assert_eq!(
14179 keyed_total(&counters, "route_open_accepted_by_principal"),
14180 1
14181 );
14182 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
14183 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
14184 }
14185
14186 #[tokio::test]
14190 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
14191 let mut swap = swap_with_incumbent().await;
14192 register_candidate(&swap, None);
14193 let incumbent = cutover(&swap);
14194 swap.forwarding
14195 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
14196 .unwrap()
14197 .expect("the incumbent is still registered");
14198
14199 let client = ConnectionId::new(33);
14200 let (client_ctx, mut client_rx) = route_ctx(client);
14201 let route_handler = swap.handler.clone();
14202 let open_ctx = RouteCtx {
14203 connection_id: client,
14204 egress: client_ctx.egress.clone(),
14205 };
14206 let mut route_task = tokio::spawn(async move {
14207 route_handler
14208 .handle_control_frame(
14209 &open_ctx,
14210 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
14211 )
14212 .await
14213 .unwrap()
14214 });
14215 let bind = tokio::select! {
14216 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
14217 response = &mut route_task => {
14218 let response = response.unwrap();
14219 panic!(
14220 "post-cutover route.open was refused instead of relayed to the candidate: {}",
14221 parse_error(&response[0])["code"]
14222 );
14223 }
14224 };
14225 swap.handler
14226 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
14227 .await
14228 .unwrap();
14229 assert!(route_task.await.unwrap().is_empty());
14230 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
14231 match swap
14232 .forwarding
14233 .lookup_data_route(client, channel, epoch)
14234 .unwrap()
14235 {
14236 DataRoute::Client(DataRouteState::Bound(route)) => {
14237 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
14238 }
14239 other => panic!("expected a bound route on the candidate, got {other:?}"),
14240 }
14241 assert!(swap.incumbent_rx.try_recv().is_err());
14242 }
14243
14244 #[tokio::test]
14249 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
14250 let swap = swap_with_incumbent().await;
14251 register_candidate(&swap, Some(false));
14252 let update = Frame::build(
14253 FrameType::Request,
14254 control_flags(),
14255 0,
14256 0,
14257 55,
14258 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14259 provides: manifest("aft", PROTOCOL_VERSION).provides,
14260 capabilities: None,
14261 ready: Some(true),
14262 })
14263 .unwrap(),
14264 )
14265 .unwrap();
14266
14267 let replies = swap
14268 .handler
14269 .handle_control_frame(&swap.candidate_ctx, update)
14270 .await
14271 .unwrap();
14272
14273 assert_eq!(replies.len(), 1);
14274 assert_eq!(
14275 replies[0].header.ty,
14276 FrameType::Response,
14277 "candidate catalog.update was refused: {:?}",
14278 serde_json::from_slice::<Value>(&replies[0].body).ok()
14279 );
14280 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
14281 assert_eq!(
14282 swap.registry
14283 .get_module("aft")
14284 .unwrap()
14285 .unwrap()
14286 .connection_id,
14287 INCUMBENT
14288 );
14289 }
14290 }
14291
14292 mod swap_admission {
14296 use super::*;
14297
14298 const INCUMBENT_NONCE: &str = "incumbent-nonce";
14299 const CANDIDATE_NONCE: &str = "candidate-nonce";
14300
14301 fn handler_with_incumbent(
14302 module_id: &str,
14303 reserved: bool,
14304 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
14305 let registry = Arc::new(Registry::default());
14306 let supervisor = SupervisorHandle::new();
14307 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
14308 if reserved {
14309 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
14310 }
14311 let handler =
14312 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
14313 let incumbent = handler
14314 .handle_control(
14315 ConnectionId::new(1),
14316 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
14317 )
14318 .unwrap();
14319 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
14320 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
14321 (registry, supervisor, handler)
14322 }
14323
14324 #[test]
14330 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
14331 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
14332
14333 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
14334 let replies = handler
14335 .handle_control(
14336 ConnectionId::new(connection),
14337 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
14338 )
14339 .unwrap();
14340 assert_eq!(replies[0].header.ty, FrameType::Error);
14341 assert_eq!(
14342 parse_error(&replies[0])["code"],
14343 "swap_token_invalid",
14344 "nonce {nonce:?}"
14345 );
14346 }
14347 assert!(registry.get_candidate("aft").unwrap().is_none());
14348 assert_eq!(
14349 registry.get_module("aft").unwrap().unwrap().connection_id,
14350 ConnectionId::new(1)
14351 );
14352
14353 let admitted = handler
14356 .handle_control(
14357 ConnectionId::new(4),
14358 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
14359 )
14360 .unwrap();
14361 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
14362 assert_eq!(
14363 registry
14364 .get_candidate("aft")
14365 .unwrap()
14366 .unwrap()
14367 .connection_id,
14368 ConnectionId::new(4)
14369 );
14370 assert_eq!(
14371 registry.get_module("aft").unwrap().unwrap().connection_id,
14372 ConnectionId::new(1),
14373 "the candidate must not take the active slot"
14374 );
14375 let replayed = handler
14376 .handle_control(
14377 ConnectionId::new(5),
14378 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
14379 )
14380 .unwrap();
14381 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
14382
14383 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
14387 let squatter = handler
14388 .handle_control(
14389 ConnectionId::new(6),
14390 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
14391 )
14392 .unwrap();
14393 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
14394 assert!(
14395 registry.get_module("aft").unwrap().is_none(),
14396 "a squatter took the active slot of an id being swapped"
14397 );
14398 }
14399
14400 #[test]
14405 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
14406 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
14407
14408 let replies = handler
14409 .handle_control(
14410 ConnectionId::new(2),
14411 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14412 )
14413 .unwrap();
14414
14415 assert_eq!(
14416 replies[0].header.ty,
14417 FrameType::HelloAck,
14418 "reserved candidate refused: {:?}",
14419 serde_json::from_slice::<Value>(&replies[0].body).ok()
14420 );
14421 assert_eq!(
14422 registry
14423 .get_candidate("vault")
14424 .unwrap()
14425 .unwrap()
14426 .connection_id,
14427 ConnectionId::new(2)
14428 );
14429 }
14430
14431 #[test]
14434 fn without_an_open_swap_the_ordinary_gates_decide() {
14435 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
14436 supervisor.close_swap("vault");
14437
14438 let candidate = handler
14439 .handle_control(
14440 ConnectionId::new(2),
14441 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14442 )
14443 .unwrap();
14444 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
14445 let duplicate = handler
14446 .handle_control(
14447 ConnectionId::new(3),
14448 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
14449 )
14450 .unwrap();
14451 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14452 assert!(registry.get_candidate("vault").unwrap().is_none());
14453 }
14454 }
14455
14456 mod scopes {
14460 mod idle_tests {
14461 use super::*;
14462 include!("scope_idle_tests.rs");
14463 }
14464 mod activity {
14465 include!("activity_scope_tests.rs");
14466 }
14467 mod delta {
14468 include!("scope_handler_delta_tests.rs");
14469 }
14470 mod call_modes {
14471 include!("scope_call_mode_tests.rs");
14472 }
14473
14474 use subc_protocol::scope::{
14475 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
14476 ScopeStatus,
14477 };
14478
14479 use super::*;
14480
14481 const OWNER: &str = "prefrontal-core";
14482
14483 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
14484 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
14485 }
14486
14487 async fn call(
14488 handler: &ControlHandler,
14489 ctx: &RouteCtx,
14490 request: &ModuleControlRequestFromModule,
14491 ) -> Frame {
14492 let body = serde_json::to_vec(request).unwrap();
14493 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
14494 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
14495 assert_eq!(replies.len(), 1, "{replies:?}");
14496 replies.pop().unwrap()
14497 }
14498
14499 async fn sync(
14500 handler: &ControlHandler,
14501 ctx: &RouteCtx,
14502 generation: u64,
14503 scopes: Vec<ScopeRecord>,
14504 ) -> Result<ModuleControlResponseToModule, String> {
14505 let reply = call(
14506 handler,
14507 ctx,
14508 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
14509 )
14510 .await;
14511 match reply.header.ty {
14512 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
14513 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
14514 }
14515 }
14516
14517 async fn describe(
14518 handler: &ControlHandler,
14519 ctx: &RouteCtx,
14520 owner: &str,
14521 scope_ref: &str,
14522 ) -> ModuleControlResponseToModule {
14523 let reply = call(
14524 handler,
14525 ctx,
14526 &ModuleControlRequestFromModule::ScopeDescribe {
14527 owner: Principal::Reserved {
14528 module_id: owner.to_string(),
14529 },
14530 scope_ref: scope_ref.to_string(),
14531 },
14532 )
14533 .await;
14534 assert_eq!(
14535 reply.header.ty,
14536 FrameType::Response,
14537 "{:?}",
14538 parse_error(&reply)
14539 );
14540 serde_json::from_slice(&reply.body).unwrap()
14541 }
14542
14543 #[tokio::test]
14544 async fn scope_describe_never_echoes_bind_proof_or_hash() {
14545 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
14546 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
14547 let mut rig = rig().await;
14548 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
14549 .await;
14550 let _bound = rig
14551 .bound(
14552 Some(AFT),
14553 PLEXUS,
14554 Some(
14555 rig.selector("s", Some(1))
14556 .with_bind_proof(Some(PROOF.into())),
14557 ),
14558 )
14559 .await;
14560 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
14561 let output = serde_json::to_string(&reply).unwrap();
14562 assert!(output.contains("\"status\":\"live\""));
14563 assert!(!output.contains(PROOF));
14564 assert!(!output.contains(HASH));
14565 }
14566
14567 async fn module(
14569 handler: &ControlHandler,
14570 connection: u64,
14571 module_id: &str,
14572 nonce: Option<&str>,
14573 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
14574 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
14575 hello_via_sink(
14576 handler,
14577 &ctx,
14578 &mut rx,
14579 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
14580 )
14581 .await;
14582 (ctx, rx)
14583 }
14584
14585 #[tokio::test]
14588 async fn a_client_connection_cannot_sync_or_describe() {
14589 let handler = ControlHandler::new(Arc::new(Registry::default()));
14590 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
14591 for request in [
14592 ModuleControlRequestFromModule::ScopeSync {
14593 generation: 1,
14594 scopes: vec![head("s", 1)],
14595 },
14596 ModuleControlRequestFromModule::ScopeDescribe {
14597 owner: Principal::Direct,
14598 scope_ref: "s".to_string(),
14599 },
14600 ] {
14601 let reply = call(&handler, &ctx, &request).await;
14602 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
14603 }
14604 assert!(
14605 !handler
14606 .scopes
14607 .read()
14608 .unwrap()
14609 .describe(
14610 &Principal::Reserved {
14611 module_id: OWNER.to_string()
14612 },
14613 "s"
14614 )
14615 .owner_synced
14616 );
14617 }
14618
14619 #[tokio::test]
14622 async fn a_module_without_a_supervised_launch_cannot_sync() {
14623 let handler = ControlHandler::new(Arc::new(Registry::default()));
14624 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
14625 assert_eq!(
14626 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
14627 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14628 );
14629 }
14630
14631 #[tokio::test]
14632 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
14633 let supervisor = SupervisorHandle::new();
14634 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14635 let handler = ControlHandler::new(Arc::new(Registry::default()))
14636 .with_supervisor(supervisor.clone());
14637 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
14638 sync(&handler, &incumbent, 1, vec![head("s", 1)])
14639 .await
14640 .expect("the current launch syncs");
14641
14642 supervisor.open_swap(OWNER, "n2".to_string());
14645 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
14646 assert_eq!(
14647 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
14648 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14649 );
14650 sync(&handler, &incumbent, 2, vec![head("s", 1)])
14651 .await
14652 .expect("the serving owner syncs during the swap");
14653
14654 supervisor.close_swap(OWNER);
14657 assert_eq!(
14658 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
14659 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14660 );
14661 sync(&handler, &incumbent, 3, vec![head("s", 1)])
14662 .await
14663 .expect("the serving owner syncs after the rollback");
14664 handler.cleanup_connection(candidate.connection_id).unwrap();
14665
14666 supervisor.open_swap(OWNER, "n3".to_string());
14671 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
14672 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
14673 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
14674 .await
14675 .expect("the promoted launch takes authority");
14676 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
14677 panic!("unexpected reply {reply:?}");
14678 };
14679 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
14680 assert_eq!(
14681 sync(&handler, &incumbent, 4, Vec::new()).await,
14682 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14683 );
14684 }
14685
14686 #[tokio::test]
14689 async fn closing_the_authority_connection_frees_sync_authority() {
14690 let supervisor = SupervisorHandle::new();
14691 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14692 let handler = ControlHandler::new(Arc::new(Registry::default()))
14693 .with_supervisor(supervisor.clone());
14694 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
14695 sync(&handler, &first, 10, vec![head("s", 1)])
14696 .await
14697 .unwrap();
14698 handler.cleanup_connection(first.connection_id).unwrap();
14699
14700 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
14701 sync(&handler, &second, 1, vec![head("s", 1)])
14702 .await
14703 .expect("the next connection takes the released authority");
14704 }
14705
14706 #[tokio::test]
14707 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
14708 let supervisor = SupervisorHandle::new();
14709 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14710 let handler =
14711 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
14712 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
14713 sync(&handler, &first, 10, vec![head("s", 1)])
14714 .await
14715 .unwrap();
14716 handler
14717 .handle_control_frame(
14718 &first,
14719 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
14720 )
14721 .await
14722 .unwrap();
14723 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
14724 sync(&handler, &second, 1, vec![head("s", 1)])
14725 .await
14726 .expect("GOODBYE releases authority even if the old socket remains open");
14727 }
14728
14729 #[tokio::test]
14730 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
14731 let registry = Arc::new(Registry::default());
14732 let supervisor_handle = SupervisorHandle::new();
14733 let supervisor =
14734 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
14735 .with_handle(supervisor_handle.clone())
14736 .with_daemon_incarnation("incarnation-7".to_string());
14737 supervisor
14740 .supervise_configured(
14741 ModuleSpec {
14742 module_id: OWNER.to_string(),
14743 program: PathBuf::from("/nonexistent/prefrontal-core"),
14744 args: Vec::new(),
14745 env: Vec::new(),
14746 reserved: false,
14747 reserved_prefixes: Vec::new(),
14748 protocol: ModuleProtocol::Subc,
14749 overlap: Default::default(),
14750 },
14751 false,
14752 )
14753 .unwrap();
14754 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
14755 let handler =
14756 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14757 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
14758
14759 let ModuleControlResponseToModule::ScopeDescribe {
14761 status,
14762 daemon_incarnation,
14763 owner_synced,
14764 owner_configured,
14765 scope,
14766 ..
14767 } = describe(&handler, &reader, OWNER, "s").await
14768 else {
14769 panic!("not a describe reply");
14770 };
14771 assert_eq!(status, ScopeStatus::NotLive);
14772 assert_eq!(daemon_incarnation, "incarnation-7");
14773 assert!(!owner_synced);
14774 assert!(owner_configured);
14775 assert!(scope.is_none());
14776
14777 let ModuleControlResponseToModule::ScopeDescribe {
14780 status,
14781 owner_configured,
14782 ..
14783 } = describe(&handler, &reader, "ghost", "s").await
14784 else {
14785 panic!("not a describe reply");
14786 };
14787 assert_eq!(status, ScopeStatus::NotLive);
14788 assert!(!owner_configured);
14789
14790 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
14792 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
14793 let ModuleControlResponseToModule::ScopeDescribe {
14794 status,
14795 scope_epoch,
14796 owner_synced,
14797 scope,
14798 ..
14799 } = describe(&handler, &reader, OWNER, "s").await
14800 else {
14801 panic!("not a describe reply");
14802 };
14803 assert_eq!(status, ScopeStatus::Live);
14804 assert_eq!(scope_epoch, Some(4));
14805 assert!(owner_synced);
14806 let stamp = scope.expect("a live scope carries its stamp");
14807 assert!(
14808 stamp.owner_authorized,
14809 "prefrontal-core is the default authority"
14810 );
14811 assert_eq!(stamp.kind, ScopeKind::Head);
14812 }
14813
14814 #[tokio::test]
14815 async fn scope_authority_owners_decides_owner_authorized() {
14816 let supervisor = SupervisorHandle::new();
14817 supervisor.set_spawn_nonce("broca", "b1".to_string());
14818 let handler = ControlHandler::new(Arc::new(Registry::default()))
14819 .with_supervisor(supervisor)
14820 .with_scope_authority_owners(vec!["broca".to_string()]);
14821 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
14822 let mut gated = head("s", 1);
14823 gated.attributes.agent_id = Some("agent".to_string());
14824 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
14825 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
14826 describe(&handler, &broca, "broca", "s").await
14827 else {
14828 panic!("not a describe reply");
14829 };
14830 assert!(scope.unwrap().owner_authorized);
14831 }
14832
14833 #[tokio::test]
14837 async fn scope_ops_and_the_scopes_capability_are_advertised() {
14838 let handler = ControlHandler::new(Arc::new(Registry::default()));
14839 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
14840 let ack = hello_via_sink(
14841 &handler,
14842 &ctx,
14843 &mut rx,
14844 hello_frame("m", PROTOCOL_VERSION, 1),
14845 )
14846 .await;
14847 let ack = parse_ack(&ack);
14848 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
14849 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
14850 }
14851 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
14852
14853 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
14854 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
14855 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
14856 let reply = handler
14857 .handle_control_frame(&client, frame)
14858 .await
14859 .unwrap()
14860 .pop()
14861 .unwrap();
14862 let ClientControlResponse::ServerDescribe { capabilities, .. } =
14863 serde_json::from_slice(&reply.body).unwrap()
14864 else {
14865 panic!("not a server.describe reply");
14866 };
14867 assert!(
14868 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
14869 "{capabilities:?}"
14870 );
14871 }
14872
14873 const PLEXUS: &str = "plexus";
14876 const OTHER: &str = "other";
14877 const AFT: &str = "aft";
14878 const BROCA: &str = "broca";
14879 const MAGIC: &str = "magic-context";
14880
14881 fn nonce(module_id: &str) -> String {
14882 format!("nonce-{module_id}")
14883 }
14884
14885 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
14886 let (tx, rx) = mpsc::channel(64);
14887 (
14888 RouteCtx {
14889 connection_id: ConnectionId::new(connection),
14890 egress: FrameSink::new(tx),
14891 },
14892 rx,
14893 )
14894 }
14895
14896 struct Rig {
14900 handler: ControlHandler,
14901 forwarding: Arc<ForwardingTable>,
14902 owner: RouteCtx,
14903 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14904 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
14905 generation: u64,
14906 next_connection: u64,
14907 _supervisor: Supervisor,
14908 }
14909
14910 async fn rig() -> Rig {
14911 rig_with_flow_support(true).await
14912 }
14913
14914 async fn rig_with_flow_support(flow_support: bool) -> Rig {
14915 rig_with_scope_capabilities(flow_support, false).await
14916 }
14917
14918 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
14919 let mut provides = Vec::new();
14920 if flow_support {
14921 provides.push("flow-scopes/v1");
14922 }
14923 if run_support {
14924 provides.push("agent-run-scopes/v1");
14925 }
14926 rig_with_target_capabilities(&provides).await
14927 }
14928
14929 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
14930 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
14931 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
14932 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
14935 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
14936 Frame::build(
14937 FrameType::Hello,
14938 control_flags(),
14939 0,
14940 0,
14941 connection,
14942 serde_json::to_vec(&body).unwrap(),
14943 )
14944 .unwrap()
14945 }
14946
14947 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
14948 let registry = Arc::new(Registry::default());
14949 let forwarding = Arc::new(ForwardingTable::default());
14950 let supervisor_handle = SupervisorHandle::new();
14951 let supervisor =
14952 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
14953 .with_handle(supervisor_handle.clone());
14954 supervisor
14955 .supervise_configured(
14956 ModuleSpec {
14957 module_id: OWNER.to_string(),
14958 program: PathBuf::from("/nonexistent/prefrontal-core"),
14959 args: Vec::new(),
14960 env: Vec::new(),
14961 reserved: false,
14962 reserved_prefixes: Vec::new(),
14963 protocol: ModuleProtocol::Subc,
14964 overlap: Default::default(),
14965 },
14966 false,
14967 )
14968 .unwrap();
14969 for module_id in [OWNER, AFT, BROCA, MAGIC] {
14970 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
14971 }
14972 let handler =
14973 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14974 .with_supervisor(supervisor_handle);
14975 let (owner, mut owner_rx) = wide_ctx(1);
14976 hello_via_sink(
14977 &handler,
14978 &owner,
14979 &mut owner_rx,
14980 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
14981 )
14982 .await;
14983 let mut modules = BTreeMap::new();
14984 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
14985 let (ctx, mut rx) = wide_ctx(connection);
14986 let hello = hello_with_capabilities(module_id, connection, provides);
14987 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
14988 modules.insert(module_id.to_string(), (ctx, rx));
14989 }
14990 Rig {
14991 handler,
14992 forwarding,
14993 owner,
14994 _owner_rx: owner_rx,
14995 modules,
14996 generation: 0,
14997 next_connection: 100,
14998 _supervisor: supervisor,
14999 }
15000 }
15001
15002 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
15003 ScopeCarrier::new(Principal::Reserved {
15004 module_id: module_id.to_string(),
15005 })
15006 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
15007 }
15008
15009 fn session(scope_epoch: u64) -> ScopeRecord {
15012 let mut record = head("s", scope_epoch);
15013 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
15014 record.attributes.agent_id = Some("agent-1".to_string());
15015 record.attributes.delegates = true;
15016 record
15017 }
15018
15019 impl Rig {
15020 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
15021 self.generation += 1;
15022 sync(&self.handler, &self.owner, self.generation, scopes)
15023 .await
15024 .expect("the owner's sync is accepted");
15025 }
15026
15027 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15028 ScopeSelector::new(
15029 Principal::Reserved {
15030 module_id: OWNER.to_string(),
15031 },
15032 scope_ref.to_string(),
15033 scope_epoch,
15034 )
15035 }
15036
15037 fn open_frame(
15038 &mut self,
15039 opener: Option<&str>,
15040 target: &str,
15041 scope: Option<ScopeSelector>,
15042 ) -> (
15043 RouteCtx,
15044 mpsc::Receiver<crate::router::OutboundFrame>,
15045 Frame,
15046 ) {
15047 self.next_connection += 1;
15048 let (ctx, rx) = wide_ctx(self.next_connection);
15049 let root = unique_project_root("scoped-open");
15050 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
15051 target: RouteTarget::ToolProvider {
15052 module_id: target.to_string(),
15053 },
15054 identity: BindIdentity::new(
15055 root.path().to_path_buf(),
15056 "unit".to_string(),
15057 "session".to_string(),
15058 ),
15059 consumer_identity: opener.map(|module_id| ConsumerIdentity {
15060 module_id: module_id.to_string(),
15061 launch_nonce: nonce(module_id),
15062 }),
15063 consumer_capabilities: None,
15064 role_versions: None,
15065 admission_facts: None,
15066 scope,
15067 })
15068 .unwrap();
15069 let frame = Frame::build(
15070 FrameType::Request,
15071 control_flags(),
15072 0,
15073 0,
15074 self.next_connection,
15075 body,
15076 )
15077 .unwrap();
15078 (ctx, rx, frame)
15079 }
15080
15081 async fn refused(
15083 &mut self,
15084 opener: Option<&str>,
15085 target: &str,
15086 scope: Option<ScopeSelector>,
15087 ) -> String {
15088 self.refusal_body(opener, target, scope).await["code"]
15089 .as_str()
15090 .unwrap()
15091 .to_string()
15092 }
15093
15094 async fn refusal_body(
15095 &mut self,
15096 opener: Option<&str>,
15097 target: &str,
15098 scope: Option<ScopeSelector>,
15099 ) -> Value {
15100 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
15101 let replies = tokio::time::timeout(
15102 Duration::from_secs(2),
15103 self.handler.handle_control_frame(&ctx, frame),
15104 )
15105 .await
15106 .expect("the open must be refused before waiting for a bind ack")
15107 .unwrap();
15108 assert_eq!(replies.len(), 1, "{replies:?}");
15109 assert_eq!(replies[0].header.ty, FrameType::Error);
15110 let (_, module_rx) = self.modules.get_mut(target).unwrap();
15111 assert!(
15112 module_rx.try_recv().is_err(),
15113 "a refused open relays nothing"
15114 );
15115 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
15116 parse_error(&replies[0])
15117 }
15118
15119 async fn relayed(
15121 &mut self,
15122 opener: Option<&str>,
15123 target: &str,
15124 scope: Option<ScopeSelector>,
15125 ) -> Relayed {
15126 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
15127 let handler = self.handler.clone();
15128 let task_ctx = ctx.clone();
15129 let task = tokio::spawn(async move {
15130 handler
15131 .handle_control_frame(&task_ctx, frame)
15132 .await
15133 .unwrap()
15134 });
15135 let (_, module_rx) = self.modules.get_mut(target).unwrap();
15136 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
15137 .await
15138 .expect("the target receives the relayed route.bind")
15139 .unwrap()
15140 .frame;
15141 Relayed {
15142 target: target.to_string(),
15143 client: ctx,
15144 client_rx: rx,
15145 task,
15146 bind,
15147 }
15148 }
15149
15150 async fn ack(&self, relayed: &Relayed) {
15151 let (module, _) = &self.modules[&relayed.target];
15152 self.handler
15153 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
15154 .await
15155 .unwrap();
15156 }
15157
15158 async fn bound(
15160 &mut self,
15161 opener: Option<&str>,
15162 target: &str,
15163 scope: Option<ScopeSelector>,
15164 ) -> Bound {
15165 let relayed = self.relayed(opener, target, scope).await;
15166 self.ack(&relayed).await;
15167 let Relayed {
15168 target,
15169 client,
15170 mut client_rx,
15171 task,
15172 bind,
15173 } = relayed;
15174 assert!(
15175 task.await.unwrap().is_empty(),
15176 "the open is answered by commit"
15177 );
15178 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
15179 Bound {
15180 target,
15181 client,
15182 client_rx,
15183 channel,
15184 epoch,
15185 bind,
15186 }
15187 }
15188
15189 fn live(&self, route: &Bound) -> bool {
15190 matches!(
15191 self.forwarding
15192 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
15193 .unwrap(),
15194 DataRoute::Client(DataRouteState::Bound(_))
15195 )
15196 }
15197 }
15198
15199 struct Relayed {
15200 target: String,
15201 client: RouteCtx,
15202 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15203 task: tokio::task::JoinHandle<Vec<Frame>>,
15204 bind: Frame,
15205 }
15206
15207 struct Bound {
15208 target: String,
15209 client: RouteCtx,
15210 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15211 channel: u16,
15212 epoch: u32,
15213 bind: Frame,
15214 }
15215
15216 impl Bound {
15217 fn closed_reason(&mut self) -> RouteCloseReason {
15220 let mut reason = None;
15221 let mut goodbye = false;
15222 while let Ok(outbound) = self.client_rx.try_recv() {
15223 let frame = outbound.frame;
15224 match frame.header.ty {
15225 FrameType::Goodbye => {
15226 assert_eq!(
15227 (frame.header.channel, frame.header.epoch),
15228 (self.channel, self.epoch)
15229 );
15230 goodbye = true;
15231 }
15232 FrameType::Push => {
15233 let ClientControlPush::RouteClosed {
15234 reason: r,
15235 module_id,
15236 ..
15237 } = serde_json::from_slice(&frame.body).unwrap()
15238 else {
15239 panic!("unexpected push");
15240 };
15241 assert_eq!(module_id, self.target);
15242 reason = Some(r);
15243 }
15244 other => panic!("unexpected frame {other:?}"),
15245 }
15246 }
15247 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
15248 reason.expect("the client is told why the route closed")
15249 }
15250
15251 fn untouched(&mut self) -> bool {
15252 self.client_rx.try_recv().is_err()
15253 }
15254
15255 fn stamp(&self) -> Option<ScopeStamp> {
15256 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
15257 ModuleControlRequest::RouteBind { scope, .. } => scope,
15258 other => panic!("expected a route.bind, got {other:?}"),
15259 }
15260 }
15261 }
15262
15263 #[tokio::test]
15264 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
15265 ) {
15266 let mut rig = rig().await;
15267 let mut record = session(1);
15268 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
15269 record.child_owners = vec![Principal::Reserved {
15270 module_id: MAGIC.to_string(),
15271 }];
15272 rig.sync(vec![record]).await;
15273 let scope = || Some(rig_selector("s", Some(1)));
15274
15275 rig.bound(Some(OWNER), PLEXUS, scope()).await;
15278 rig.bound(Some(AFT), OTHER, scope()).await;
15279 rig.bound(Some(BROCA), PLEXUS, scope()).await;
15280
15281 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
15285 assert_eq!(
15286 rig.refused(opener, target, scope()).await,
15287 error_codes::SCOPE_NOT_CARRIER,
15288 "{opener:?} -> {target}"
15289 );
15290 }
15291 }
15292
15293 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15294 ScopeSelector::new(
15295 Principal::Reserved {
15296 module_id: OWNER.to_string(),
15297 },
15298 scope_ref.to_string(),
15299 scope_epoch,
15300 )
15301 }
15302
15303 #[tokio::test]
15304 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
15305 let mut rig = rig().await;
15306 rig.sync(vec![session(1)]).await;
15307 for opener in [OWNER, AFT] {
15308 assert_eq!(
15309 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
15310 .await,
15311 error_codes::SCOPE_EPOCH_REQUIRED,
15312 "{opener}"
15313 );
15314 }
15315 }
15316
15317 #[tokio::test]
15318 async fn admission_separates_not_synced_not_live_and_ended() {
15319 let mut rig = rig().await;
15320 let code = rig
15322 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15323 .await;
15324 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
15325 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
15326
15327 let ghost = ScopeSelector::new(
15329 Principal::Reserved {
15330 module_id: "ghost".to_string(),
15331 },
15332 "s".to_string(),
15333 Some(1),
15334 );
15335 assert_eq!(
15336 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
15337 error_codes::SCOPE_NOT_LIVE
15338 );
15339
15340 rig.sync(vec![session(2)]).await;
15341 assert_eq!(
15342 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
15343 .await,
15344 error_codes::SCOPE_NOT_LIVE
15345 );
15346 for epoch in [1, 3] {
15347 assert_eq!(
15348 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
15349 .await,
15350 error_codes::SCOPE_ENDED,
15351 "epoch {epoch}"
15352 );
15353 }
15354 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
15356 .await;
15357 }
15358
15359 #[tokio::test]
15360 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
15361 let mut rig = rig().await;
15362 rig.sync(vec![session(1)]).await;
15363 let route = rig
15364 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15365 .await;
15366 let stamp = route.stamp().expect("a scoped bind carries the stamp");
15367 assert_eq!(stamp.scope_ref, "s");
15368 assert_eq!(stamp.scope_epoch, 1);
15369 assert_eq!(stamp.kind, ScopeKind::Head);
15370 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
15371 assert!(stamp.attributes.delegates);
15372 assert!(stamp.owner_authorized);
15373 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15374 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
15375
15376 let (broca, mut broca_rx) = wide_ctx(50);
15379 hello_via_sink(
15380 &rig.handler,
15381 &broca,
15382 &mut broca_rx,
15383 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
15384 )
15385 .await;
15386 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
15387 .await
15388 .unwrap();
15389 let own = ScopeSelector::new(
15390 Principal::Reserved {
15391 module_id: BROCA.to_string(),
15392 },
15393 "b".to_string(),
15394 Some(1),
15395 );
15396 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
15397 assert!(!route.stamp().unwrap().owner_authorized);
15398 }
15399
15400 #[tokio::test]
15401 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
15402 let mut rig = rig().await;
15403 let mut record = head("s", 1);
15404 record.carriers = vec![carrier(AFT, None)];
15405 let flow_id = "Flow:run-7/step_2!~";
15406 record.attributes.flow_id = Some(flow_id.to_string());
15407 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
15408 .await
15409 .unwrap();
15410 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
15411 panic!("not a sync reply");
15412 };
15413 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
15414 let route = rig
15415 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15416 .await;
15417 assert!(rig.live(&route), "the stamped bind committed");
15418 let stamp = route.stamp().expect("a flow scope carries a stamp");
15419 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
15420 assert_eq!(stamp.attributes.agent_id, None);
15421 assert!(!stamp.attributes.delegates);
15422 assert!(stamp.owner_authorized);
15423 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15424 assert_eq!(unscoped.stamp(), None);
15425 }
15426
15427 #[tokio::test]
15428 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
15429 let mut rig = rig_with_flow_support(false).await;
15430 let mut record = session(1);
15431 record.attributes.flow_id = Some("flow:7".to_string());
15432 rig.sync(vec![record]).await;
15433 for opener in [OWNER, AFT] {
15434 let body = rig
15435 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
15436 .await;
15437 assert_eq!(body["code"], "target_flow_unsupported");
15438 let message = body["message"].as_str().unwrap();
15439 for required in [PLEXUS, "flow-scopes/v1"] {
15440 assert!(message.contains(required), "{message}");
15441 }
15442 }
15443 }
15444
15445 #[tokio::test]
15446 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
15447 let mut rig = rig_with_flow_support(true).await;
15448 let mut record = session(1);
15449 record.attributes.flow_id = Some("flow:7".to_string());
15450 rig.sync(vec![record]).await;
15451 for opener in [OWNER, AFT] {
15452 let route = rig
15453 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
15454 .await;
15455 assert!(rig.live(&route));
15456 assert_eq!(
15457 route.stamp().unwrap().attributes.flow_id.as_deref(),
15458 Some("flow:7")
15459 );
15460 }
15461 }
15462
15463 #[tokio::test]
15464 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
15465 use std::future::Future;
15466
15467 let mut rig = rig_with_flow_support(true).await;
15468 let mut record = session(1);
15469 record.attributes.flow_id = Some("flow:7".to_string());
15470 rig.sync(vec![record]).await;
15471 let (client, mut client_rx, frame) =
15472 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
15473 for _ in 0..64 {
15476 client.egress.try_send(route_bind_ack(1)).unwrap();
15477 }
15478 let handler = rig.handler.clone();
15479 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15480 std::future::poll_fn(|cx| {
15481 assert!(open.as_mut().poll(cx).is_pending());
15482 std::task::Poll::Ready(())
15483 })
15484 .await;
15485 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15486
15487 let old_connection = rig.modules[PLEXUS].0.connection_id;
15488 rig.handler.cleanup_connection(old_connection).unwrap();
15489 let (replacement, mut replacement_rx) = wide_ctx(200);
15490 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
15491 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
15492 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
15493 let hello = Frame::build(
15494 FrameType::Hello,
15495 control_flags(),
15496 0,
15497 0,
15498 200,
15499 serde_json::to_vec(&body).unwrap(),
15500 )
15501 .unwrap();
15502 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
15503
15504 client_rx.try_recv().unwrap();
15505 let replies = tokio::time::timeout(Duration::from_secs(2), open)
15506 .await
15507 .expect("the replacement is refused without waiting for a bind ack")
15508 .unwrap();
15509 assert_eq!(replies.len(), 1);
15510 let body = parse_error(&replies[0]);
15511 assert_eq!(body["code"], "target_flow_unsupported");
15512 for required in [PLEXUS, "flow-scopes/v1"] {
15513 assert!(body["message"].as_str().unwrap().contains(required));
15514 }
15515 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
15516 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
15517 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15518 assert!(rig
15519 .handler
15520 .registry
15521 .get_module_by_connection(replacement.connection_id)
15522 .unwrap()
15523 .is_some());
15524 }
15525
15526 #[tokio::test]
15527 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
15528 use std::future::Future;
15529 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
15530 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15531 const NEXT_HASH: &str =
15532 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
15533 let mut rig = rig().await;
15534 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15535 .await;
15536 let (client, mut rx, frame) = rig.open_frame(
15537 Some(AFT),
15538 PLEXUS,
15539 Some(
15540 rig.selector("s", Some(1))
15541 .with_bind_proof(Some(PROOF.into())),
15542 ),
15543 );
15544 for _ in 0..64 {
15547 client.egress.try_send(route_bind_ack(1)).unwrap();
15548 }
15549 let handler = rig.handler.clone();
15550 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15551 std::future::poll_fn(|cx| {
15552 assert!(open.as_mut().poll(cx).is_pending());
15553 std::task::Poll::Ready(())
15554 })
15555 .await;
15556 rig.sync(vec![
15557 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
15558 ])
15559 .await;
15560 rx.try_recv().unwrap();
15561 let replies = tokio::time::timeout(Duration::from_secs(2), open)
15562 .await
15563 .unwrap()
15564 .unwrap();
15565 assert_eq!(
15566 parse_error(&replies[0])["code"],
15567 error_codes::SCOPE_BIND_PROOF_MISMATCH
15568 );
15569 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
15570 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
15571 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15572 }
15573
15574 #[tokio::test]
15575 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
15576 {
15577 let mut rig = rig_with_flow_support(false).await;
15578 rig.sync(vec![session(1)]).await;
15579 let route = rig
15580 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15581 .await;
15582 assert!(rig.live(&route));
15583 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
15584 let mut record = session(1);
15585 record.attributes.flow_id = Some("flow:7".to_string());
15586 rig.sync(vec![record]).await;
15587 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
15588 assert!(rig.live(&unscoped));
15589 assert_eq!(unscoped.stamp(), None);
15590 }
15591
15592 #[tokio::test]
15593 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
15594 let mut rig = rig().await;
15595 let mut record = session(1);
15596 record.attributes.flow_id = Some("flow:7".to_string());
15597 rig.sync(vec![record.clone()]).await;
15598 let before = rig.forwarding.published_scope_tag(OWNER, "s");
15599 let mut owner_route = rig
15600 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
15601 .await;
15602 let mut carrier_route = rig
15603 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
15604 .await;
15605 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15606 rig.sync(vec![record.clone()]).await;
15607 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
15608 assert!(rig.live(&owner_route) && owner_route.untouched());
15609 assert!(rig.live(&carrier_route) && carrier_route.untouched());
15610
15611 record.attributes.flow_id = Some("flow:8".to_string());
15612 rig.sync(vec![record]).await;
15613 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
15614 let before = before.unwrap();
15615 assert_eq!(after.scope_epoch, before.scope_epoch);
15616 assert!(after.version > before.version);
15617 for route in [&mut owner_route, &mut carrier_route] {
15618 assert!(!rig.live(route));
15619 assert_eq!(
15620 route.closed_reason(),
15621 RouteCloseReason::ScopeDelegationChanged
15622 );
15623 }
15624 assert!(rig.live(&unscoped) && unscoped.untouched());
15625 for target in [PLEXUS, OTHER] {
15628 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
15629 let goodbye = module_rx
15630 .try_recv()
15631 .expect("the provider sees the drain")
15632 .frame;
15633 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15634 assert!(module_rx.try_recv().is_err());
15635 }
15636 let rebound = rig
15637 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15638 .await;
15639 assert_eq!(
15640 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
15641 Some("flow:8")
15642 );
15643 }
15644
15645 #[tokio::test]
15650 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
15651 let mut rig = rig().await;
15652 rig.sync(vec![session(1)]).await;
15653 let mut cotenant = rig
15654 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
15655 .await;
15656
15657 let mut changed = session(1);
15658 changed.child_owners.push(Principal::Reserved {
15659 module_id: MAGIC.to_string(),
15660 });
15661 let mut ended = None;
15662 for (code, next) in [
15663 (error_codes::SCOPE_CHANGED, vec![changed]),
15664 (error_codes::SCOPE_ENDED, Vec::new()),
15665 ] {
15666 let relayed = rig
15667 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15668 .await;
15669 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
15670 ended = Some(next.is_empty());
15671 rig.sync(next).await;
15672 rig.ack(&relayed).await;
15673 let replies = relayed.task.await.unwrap();
15674 assert_eq!(replies.len(), 1, "{replies:?}");
15675 assert_eq!(parse_error(&replies[0])["code"], code);
15676 assert_eq!(
15677 subc_protocol::error_codes::is_retryable_route_open(code),
15678 code == error_codes::SCOPE_CHANGED
15679 );
15680 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15681 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15683 let mut goodbyes = Vec::new();
15686 while let Ok(outbound) = plexus_rx.try_recv() {
15687 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
15688 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
15689 }
15690 assert!(
15691 goodbyes.contains(&(bind_channel, bind_epoch)),
15692 "{goodbyes:?}"
15693 );
15694 assert!(rig
15695 .handler
15696 .registry
15697 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
15698 .unwrap()
15699 .is_some());
15700 }
15701 assert_eq!(ended, Some(true));
15702 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
15705 }
15706
15707 #[tokio::test]
15710 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
15711 struct Case {
15712 name: &'static str,
15713 change: fn(&mut ScopeRecord),
15714 closed: [Option<RouteCloseReason>; 4],
15717 }
15718 use RouteCloseReason::*;
15719 let cases = [
15720 Case {
15721 name: "a carrier entry removed",
15722 change: |r| {
15723 r.carriers.retain(|c| {
15724 c.principal
15725 != Principal::Reserved {
15726 module_id: AFT.to_string(),
15727 }
15728 })
15729 },
15730 closed: [None, Some(ScopeCarrierRemoved), None, None],
15731 },
15732 Case {
15733 name: "a target removed from a carrier",
15734 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
15735 closed: [None, None, None, Some(ScopeCarrierRemoved)],
15736 },
15737 Case {
15738 name: "a bare carrier narrowed to targets",
15739 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
15740 closed: [None, Some(ScopeCarrierRemoved), None, None],
15741 },
15742 Case {
15743 name: "delegates turned off",
15744 change: |r| r.attributes.delegates = false,
15745 closed: [Some(ScopeDelegationChanged); 4],
15746 },
15747 Case {
15748 name: "agent_id changed",
15749 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
15750 closed: [Some(ScopeDelegationChanged); 4],
15751 },
15752 Case {
15753 name: "a carrier added, child owners changed, the record re-sent",
15754 change: |r| {
15755 r.carriers.push(carrier(MAGIC, None));
15756 r.child_owners.push(Principal::Reserved {
15757 module_id: MAGIC.to_string(),
15758 });
15759 },
15760 closed: [None; 4],
15761 },
15762 Case {
15763 name: "a target added",
15764 change: |r| {
15765 r.carriers[1]
15766 .targets
15767 .as_mut()
15768 .unwrap()
15769 .push("third".to_string())
15770 },
15771 closed: [None; 4],
15772 },
15773 Case {
15774 name: "delegates turned on",
15775 change: |r| r.attributes.delegates = true,
15776 closed: [None; 4],
15777 },
15778 ];
15779 for case in cases {
15780 let mut rig = rig().await;
15781 rig.sync(vec![session(1)]).await;
15782 let scope = || Some(rig_selector("s", Some(1)));
15783 let mut routes = [
15784 rig.bound(Some(OWNER), PLEXUS, scope()).await,
15785 rig.bound(Some(AFT), PLEXUS, scope()).await,
15786 rig.bound(Some(BROCA), PLEXUS, scope()).await,
15787 rig.bound(Some(BROCA), OTHER, scope()).await,
15788 ];
15789 let mut record = session(1);
15790 (case.change)(&mut record);
15791 rig.sync(vec![record]).await;
15792 for (index, expected) in case.closed.iter().enumerate() {
15793 let route = &mut routes[index];
15794 match expected {
15795 Some(reason) => {
15796 assert!(!rig.live(route), "{}: route {index} still live", case.name);
15797 assert_eq!(
15798 route.closed_reason(),
15799 *reason,
15800 "{}: route {index}",
15801 case.name
15802 );
15803 }
15804 None => {
15805 assert!(rig.live(route), "{}: route {index} closed", case.name);
15806 assert!(
15807 route.untouched(),
15808 "{}: route {index} was told something",
15809 case.name
15810 );
15811 }
15812 }
15813 }
15814 }
15815 }
15816
15817 #[tokio::test]
15818 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
15819 for next in [Vec::new(), vec![session(2)]] {
15821 let mut rig = rig().await;
15822 rig.sync(vec![session(1)]).await;
15823 let mut route = rig
15824 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15825 .await;
15826 rig.sync(next).await;
15827 assert!(!rig.live(&route));
15828 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
15829 }
15830
15831 let mut rig = rig().await;
15834 let mut child = session(1);
15835 child.scope_ref = "child".to_string();
15836 child.kind = ScopeKind::Worker;
15837 child.parent = Some(ScopeParent::new(
15838 Principal::Reserved {
15839 module_id: OWNER.to_string(),
15840 },
15841 "s".to_string(),
15842 1,
15843 ));
15844 rig.sync(vec![session(1), child.clone()]).await;
15845 let mut child_route = rig
15846 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
15847 .await;
15848 assert_eq!(
15849 child_route.stamp().unwrap().parent_state,
15850 Some(ParentState::Linked)
15851 );
15852 rig.sync(vec![child]).await;
15853 assert!(!rig.live(&child_route));
15854 assert_eq!(
15855 child_route.closed_reason(),
15856 RouteCloseReason::ScopeParentEnded
15857 );
15858 }
15859
15860 #[tokio::test]
15861 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
15862 ) {
15863 let mut rig = rig().await;
15864 rig.sync(vec![session(1)]).await;
15865 let mut route = rig
15866 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15867 .await;
15868 let before = rig.forwarding.published_scope_tag(OWNER, "s");
15869 rig.sync(vec![session(1)]).await;
15870 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
15871 assert!(rig.live(&route) && route.untouched());
15872
15873 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
15877 .forwarding
15878 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
15879 .unwrap()
15880 else {
15881 panic!("the route is bound");
15882 };
15883 binding.flow.acquire_tagged(9, false).await.unwrap();
15884 let mut widened = session(1);
15885 widened.carriers.push(carrier(MAGIC, None));
15886 rig.sync(vec![widened]).await;
15887 assert!(rig.live(&route) && route.untouched());
15888 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15889 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
15890 assert_eq!(binding.flow.in_flight(), 1);
15893 binding
15894 .flow
15895 .acquire_tagged(10, false)
15896 .await
15897 .expect("the flow is still open");
15898 }
15899
15900 #[tokio::test]
15903 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
15904 let mut rig = rig().await;
15905 rig.sync(vec![session(1)]).await;
15906 let mut on_incumbent = rig
15907 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15908 .await;
15909
15910 let (candidate, _candidate_rx) = wide_ctx(9);
15913 let registration = rig
15914 .handler
15915 .registry
15916 .register_candidate_with_control_ops(
15917 manifest(PLEXUS, PROTOCOL_VERSION),
15918 PROTOCOL_VERSION,
15919 candidate.connection_id,
15920 module_baseline_control_ops(),
15921 )
15922 .unwrap();
15923 rig.forwarding
15924 .register_candidate_module_connection(
15925 candidate.connection_id,
15926 PLEXUS.to_string(),
15927 PROTOCOL_VERSION,
15928 manifest_concurrency(®istration.manifest),
15929 candidate.egress.clone(),
15930 )
15931 .unwrap();
15932 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
15933 rig.handler
15934 .registry
15935 .promote_candidate(PLEXUS)
15936 .unwrap()
15937 .unwrap();
15938 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
15939
15940 rig.sync(Vec::new()).await;
15941 assert!(!rig.live(&on_incumbent));
15942 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
15943 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15944 let goodbye = incumbent_rx
15945 .try_recv()
15946 .expect("the superseded endpoint is told")
15947 .frame;
15948 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15949 }
15950 }
15951}
15952
15953#[cfg(test)]
15954mod concurrency_default_exposure_tests {
15955 use super::*;
15956
15957 fn hello_body(role_json: &str) -> Vec<u8> {
15958 format!(
15959 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
15960 )
15961 .into_bytes()
15962 }
15963
15964 fn manifest_from(body: &[u8]) -> ModuleManifest {
15965 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
15966 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
15967 .expect("manifest parses")
15968 }
15969
15970 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
15971
15972 #[test]
15973 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
15974 let body = hello_body(&format!(
15975 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
15976 ));
15977 let manifest = manifest_from(&body);
15978 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
15981 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
15982 }
15983
15984 #[test]
15985 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
15986 let body = hello_body(&format!(
15987 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
15988 ));
15989 let manifest = manifest_from(&body);
15990 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
15991 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
15992 }
15993
15994 #[test]
15995 fn non_management_roles_are_never_reported() {
15996 let body = hello_body(
15997 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
15998 );
15999 let manifest = manifest_from(&body);
16000 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16001 }
16002}