1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex, RwLock},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 error_codes,
21 manifest::{
22 validate_hello_capability_grammar, validate_hello_event_declarations,
23 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
24 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
25 },
26 scope::{
27 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
28 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
29 SCOPE_SYNC_OP,
30 },
31 session::{
32 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
33 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
34 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
35 ROLE_VERSIONS_FIELD,
36 },
37 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
38 Priority, RouteTarget, PROTOCOL_VERSION,
39};
40use tokio::time::{timeout_at, Instant};
41use tracing::{debug, info, warn};
42
43use crate::{
44 capability_requirements::{
45 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
46 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
47 RuntimeModule,
48 },
49 daemon_config::RestartRequiredSection,
50 forwarding::{
51 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
52 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
53 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
54 },
55 observability::{
56 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
57 },
58 provenance::{
59 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
60 },
61 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
62 router::{RouteCtx, RouterError},
63 scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
64 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
65 stderr_tail::{CaptureState, TailEntry},
66 supervise::{
67 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
68 SupervisorHandle, SwapHelloAdmission,
69 },
70 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
71};
72
73pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
79
80const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
81const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
82const CAP_PING_PONG: &str = "ping_pong_v1";
83const CAP_SESSION_ATTACH: &str = "session_attach_v1";
84const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
85
86const SUBC_CONTROL_OPS: &[&str] = &[
87 ops::ACTIVITY_RECORD,
88 ops::SERVER_DESCRIBE,
89 ops::CATALOG_LIST,
90 ops::ROUTE_OPEN,
91 ops::ROUTE_POLL,
92 ops::ROUTE_CLOSING,
93 ops::ROUTE_CLOSED,
94 ops::SUPERVISOR_LIST,
95 ops::SUPERVISOR_RESTART,
96 ops::SUPERVISOR_SWAP,
97 ops::SUPERVISOR_RELOAD,
98 ops::SUPERVISOR_RESCAN,
99 ops::SUPERVISOR_RELEASE_RESERVED,
100 ops::SUPERVISOR_SET_ENABLED,
101 ops::SUPERVISOR_HEALTH_PROBE,
102 ops::SUPERVISOR_HEALTH,
103 ops::SUPERVISOR_STDERR_TAIL,
104 ops::SUPERVISOR_TERMINALS,
105 ops::SUPERVISOR_ROUTES,
106 ops::SUPERVISOR_PROVENANCE,
107 ops::SUPERVISOR_SPAWN_SNAPSHOT,
108 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
109];
110
111const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
112 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
113 "supervisor.live_roots",
114 SCOPE_SYNC_OP,
115 SCOPE_APPLY_OP,
116 SCOPE_DESCRIBE_OP,
117 "operator.confirm",
118];
119
120const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
124
125const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
126
127pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
135
136pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
151
152pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
170
171const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
172const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
173
174fn reload_verdict(
175 configured: &Path,
176 spawned_from: Option<&Path>,
177 image: subc_control::RunningImageAgreement,
178) -> PendingReloadVerdict {
179 let path = match spawned_from {
180 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
181 ReloadPathAgreement::Match
182 }
183 Some(spawned_from) => ReloadPathAgreement::Mismatch {
184 configured: configured.to_path_buf(),
185 spawned_from: spawned_from.to_path_buf(),
186 },
187 None => ReloadPathAgreement::Unavailable {
188 reason: if matches!(
189 image,
190 subc_control::RunningImageAgreement::Unavailable {
191 reason: subc_control::RunningImageUnavailableReason::NotRunning
192 }
193 ) {
194 ReloadPathUnavailableReason::NotRunning
195 } else {
196 ReloadPathUnavailableReason::SpawnedPathUnavailable
197 },
198 },
199 };
200 PendingReloadVerdict { path, image }
201}
202
203fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
204 if configured == spawned_from {
205 return true;
206 }
207 #[cfg(windows)]
208 {
209 match (
215 std::fs::canonicalize(configured),
216 std::fs::canonicalize(spawned_from),
217 ) {
218 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
219 _ => false,
220 }
221 }
222 #[cfg(not(windows))]
223 {
224 false
225 }
226}
227
228#[derive(Clone)]
229struct DaemonProvenanceFacts {
230 build: DaemonBuildProvenance,
231 pid: Option<u32>,
232 started_at_ms: Option<u64>,
233 start_clock: Option<crate::clock::StartClock>,
234 executable_path: Option<PathBuf>,
235 executable_identity: Option<SpawnedFileIdentity>,
236 process_start_time: Option<u64>,
237 probe: ExecutableIdentityProbe,
238}
239
240impl Default for DaemonProvenanceFacts {
241 fn default() -> Self {
242 Self {
243 build: DaemonBuildProvenance {
244 build_git_sha: None,
245 build_lock_digest: None,
246 },
247 pid: None,
248 started_at_ms: None,
249 start_clock: None,
250 executable_path: None,
251 executable_identity: None,
252 process_start_time: None,
253 probe: ExecutableIdentityProbe::default(),
254 }
255 }
256}
257
258#[derive(Debug, Clone)]
259struct SupervisorRescanContext {
260 supervisor: Supervisor,
261 config_path: PathBuf,
262 configured_port: Option<u16>,
263 storage_config: Option<crate::daemon_config::StorageConfig>,
264 admission_facts_carrier_module_id: Option<String>,
265 admission_facts_targets: Option<Vec<String>>,
266 scope_authority_owners: Vec<String>,
267}
268
269const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
290 "reloading",
291 "supervisor_not_live",
292 "registration_not_active",
293 "no_forwarding_connection",
294 "relay_send_failed",
295];
296
297#[derive(Clone)]
299pub struct ControlHandler {
300 registry: Arc<Registry>,
301 activity: Option<crate::activity::ActivityLedger>,
302 forwarding: Arc<ForwardingTable>,
303 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
304 supervisor: SupervisorHandle,
305 subc_capabilities: Arc<[String]>,
306 route_bind_relay_timeout: Duration,
310 route_bind_relay_timeouts: BTreeMap<String, Duration>,
314 route_bind_breakers: RouteBindBreakers,
317 route_bind_concurrency: RouteBindConcurrency,
320 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
325 route_bind_breaker_threshold: u32,
327 route_bind_breaker_cooldown: Duration,
329 health_probe_timeout: Duration,
330 storage_config: Option<crate::daemon_config::StorageConfig>,
333 machine_id: Option<crate::machine_id::MachineId>,
337 admission_facts_carrier_module_id: Option<String>,
338 admission_facts_targets: Option<Vec<String>>,
339 scopes: Arc<RwLock<ScopeTable>>,
343 scope_deadline_added: Arc<tokio::sync::Notify>,
347 scope_authority_owners: Vec<String>,
350 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
353 rescan: Option<SupervisorRescanContext>,
354 connected_clients: ConnectedClients,
355 counters: DaemonCounters,
356 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
357 daemon_provenance: DaemonProvenanceFacts,
358 #[cfg(test)]
359 control_dispatch_delay: Option<Duration>,
360 #[cfg(test)]
361 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
362 #[cfg(test)]
366 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
367}
368
369impl fmt::Debug for ControlHandler {
370 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
371 f.debug_struct("ControlHandler")
372 .field("registry", &self.registry)
373 .field("forwarding", &self.forwarding)
374 .field("process_liveness", &self.process_liveness.is_some())
375 .field("supervisor", &self.supervisor)
376 .field("subc_capabilities", &self.subc_capabilities)
377 .finish()
378 }
379}
380
381struct RouteOpenRequest {
382 target: RouteTarget,
383 identity: BindIdentity,
384 consumer_identity: Option<ConsumerIdentity>,
385 consumer_capabilities: Option<Vec<String>>,
386 role_versions: Option<BTreeMap<String, String>>,
387 admission_facts: Option<serde_json::Value>,
388 scope: Option<ScopeSelector>,
389}
390
391struct RouteBindReservationGuard {
392 forwarding: Arc<ForwardingTable>,
393 endpoint: ModuleEndpointId,
394 relay_corr: u64,
395 armed: bool,
396}
397
398struct ModuleControlRpcGuard {
399 forwarding: Arc<ForwardingTable>,
400 endpoint: ModuleEndpointId,
401 corr: u64,
402 armed: bool,
403}
404
405impl ModuleControlRpcGuard {
406 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
407 Self {
408 forwarding,
409 endpoint,
410 corr,
411 armed: true,
412 }
413 }
414
415 fn disarm(&mut self) {
416 self.armed = false;
417 }
418}
419
420impl Drop for ModuleControlRpcGuard {
421 fn drop(&mut self) {
422 if self.armed {
423 let _ = self
424 .forwarding
425 .cancel_module_control_rpc(self.endpoint, self.corr);
426 }
427 }
428}
429
430impl RouteBindReservationGuard {
431 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
432 Self {
433 forwarding,
434 endpoint,
435 relay_corr,
436 armed: true,
437 }
438 }
439
440 fn release_and_disarm(&mut self) {
441 if !self.armed {
442 return;
443 }
444 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
445 self.endpoint,
446 self.relay_corr,
447 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
448 ) {
449 send_goodbye_target_best_effort(
450 &self.forwarding.counters(),
451 &target,
452 "canceled route.bind",
453 );
454 }
455 self.armed = false;
456 }
457
458 fn disarm(&mut self) {
459 self.armed = false;
460 }
461}
462
463impl Drop for RouteBindReservationGuard {
464 fn drop(&mut self) {
465 self.release_and_disarm();
466 }
467}
468
469#[derive(Debug, Clone, Default)]
499pub(crate) struct RouteBindBreakers {
500 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
501}
502
503#[derive(Debug, Clone, Default)]
504pub(crate) struct RouteBindConcurrency {
505 modules: Arc<Mutex<HashMap<String, usize>>>,
506}
507
508struct RouteBindConcurrencyGuard {
509 concurrency: RouteBindConcurrency,
510 module_id: String,
511}
512
513impl RouteBindConcurrency {
514 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
517 let mut modules = self
518 .modules
519 .lock()
520 .expect("route.bind concurrency mutex poisoned");
521 let in_flight = modules.entry(module_id.to_string()).or_default();
522 if *in_flight >= limit {
523 return Err(*in_flight);
524 }
525 *in_flight += 1;
526 Ok(RouteBindConcurrencyGuard {
527 concurrency: self.clone(),
528 module_id: module_id.to_string(),
529 })
530 }
531}
532
533impl Drop for RouteBindConcurrencyGuard {
534 fn drop(&mut self) {
535 let mut modules = self
536 .concurrency
537 .modules
538 .lock()
539 .expect("route.bind concurrency mutex poisoned");
540 let remove = {
541 let in_flight = modules
542 .get_mut(&self.module_id)
543 .expect("admitted route.bind has a concurrency entry");
544 *in_flight -= 1;
545 *in_flight == 0
546 };
547 if remove {
548 modules.remove(&self.module_id);
549 }
550 }
551}
552
553#[derive(Debug, Default)]
554struct ModuleBreakerState {
555 consecutive_timeouts: u32,
557 cooldown_until: Option<Instant>,
560 probe_in_flight: Option<Arc<()>>,
565}
566
567enum RouteBindAdmission<'a> {
569 Admitted {
570 guard: RouteBindBreakerGuard<'a>,
571 probe: bool,
574 },
575 Refused {
576 consecutive_timeouts: u32,
577 retry_in: Duration,
581 probe_in_flight: bool,
582 },
583}
584
585struct RouteBindBreakerGuard<'a> {
592 breakers: RouteBindBreakers,
593 module_id: &'a str,
594 probe_token: Option<Arc<()>>,
595 settled: bool,
596}
597
598impl RouteBindBreakerGuard<'_> {
599 fn record_accepted(&mut self) -> bool {
603 self.settled = true;
604 self.breakers.record_accepted(self.module_id)
605 }
606
607 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
610 self.settled = true;
611 self.breakers.record_timeout(
612 self.module_id,
613 self.probe_token.as_ref(),
614 threshold,
615 cooldown,
616 )
617 }
618
619 fn record_inconclusive(&mut self) {
628 self.settled = true;
629 self.breakers
630 .record_inconclusive(self.module_id, self.probe_token.as_ref());
631 }
632}
633
634impl Drop for RouteBindBreakerGuard<'_> {
635 fn drop(&mut self) {
636 if !self.settled {
637 self.breakers
638 .record_inconclusive(self.module_id, self.probe_token.as_ref());
639 }
640 }
641}
642
643struct BreakerOpened {
647 consecutive_timeouts: u32,
648 reopened_after_probe: bool,
651}
652
653impl RouteBindBreakers {
654 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
655 self.modules
656 .lock()
657 .expect("route.bind breaker mutex poisoned")
658 }
659
660 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
663 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
664 probe: probe_token.is_some(),
665 guard: RouteBindBreakerGuard {
666 breakers: self.clone(),
667 module_id,
668 probe_token,
669 settled: false,
670 },
671 };
672
673 let mut modules = self.lock();
674 let Some(state) = modules.get_mut(module_id) else {
675 return admitted(None);
676 };
677 let Some(cooldown_until) = state.cooldown_until else {
678 return admitted(None);
679 };
680 if state.probe_in_flight.is_some() {
681 return RouteBindAdmission::Refused {
682 consecutive_timeouts: state.consecutive_timeouts,
683 retry_in: Duration::ZERO,
684 probe_in_flight: true,
685 };
686 }
687 let now = Instant::now();
688 if now < cooldown_until {
689 return RouteBindAdmission::Refused {
690 consecutive_timeouts: state.consecutive_timeouts,
691 retry_in: cooldown_until - now,
692 probe_in_flight: false,
693 };
694 }
695 let token = Arc::new(());
696 state.probe_in_flight = Some(Arc::clone(&token));
697 admitted(Some(token))
698 }
699
700 fn record_accepted(&self, module_id: &str) -> bool {
701 self.lock()
702 .remove(module_id)
703 .is_some_and(|state| state.cooldown_until.is_some())
704 }
705
706 fn record_timeout(
707 &self,
708 module_id: &str,
709 probe_token: Option<&Arc<()>>,
710 threshold: u32,
711 cooldown: Duration,
712 ) -> Option<BreakerOpened> {
713 let mut modules = self.lock();
714 let state = modules.entry(module_id.to_string()).or_default();
715 let was_open = state.cooldown_until.is_some();
716 let was_probe = Self::owns_probe(state, probe_token);
717 if was_probe {
718 state.probe_in_flight = None;
719 }
720 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
721 if state.consecutive_timeouts < threshold {
722 return None;
723 }
724 state.cooldown_until = Some(Instant::now() + cooldown);
725 Some(BreakerOpened {
726 consecutive_timeouts: state.consecutive_timeouts,
727 reopened_after_probe: was_open && was_probe,
728 })
729 }
730
731 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
732 state
736 .probe_in_flight
737 .as_ref()
738 .zip(token)
739 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
740 }
741
742 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
743 if let Some(state) = self.lock().get_mut(module_id) {
744 if Self::owns_probe(state, probe_token) {
745 state.probe_in_flight = None;
746 }
747 }
748 }
749
750 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
765 self.lock()
766 .remove(module_id)
767 .map(|state| state.consecutive_timeouts)
768 .filter(|discarded| *discarded > 0)
769 }
770
771 fn open_snapshot(&self) -> Option<serde_json::Value> {
778 let now = Instant::now();
779 let modules = self.lock();
780 let open = modules
781 .iter()
782 .filter_map(|(module_id, state)| {
783 let cooldown_until = state.cooldown_until?;
784 Some((
785 module_id.clone(),
786 serde_json::json!({
787 "consecutive_timeouts": state.consecutive_timeouts,
788 "cooldown_remaining_ms":
789 cooldown_until.saturating_duration_since(now).as_millis() as u64,
790 "probe_in_flight": state.probe_in_flight.is_some(),
791 }),
792 ))
793 })
794 .collect::<serde_json::Map<String, serde_json::Value>>();
795 (!open.is_empty()).then_some(serde_json::Value::Object(open))
796 }
797}
798
799impl ControlHandler {
800 pub fn new(registry: Arc<Registry>) -> Self {
801 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
802 }
803
804 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
805 let counters = forwarding.counters();
806 let route_bind_breakers = forwarding.route_bind_breakers();
810 let route_bind_concurrency = forwarding.route_bind_concurrency();
811 let route_outages = forwarding.route_outages();
812 Self {
813 registry,
814 activity: None,
815 forwarding,
816 process_liveness: None,
817 supervisor: SupervisorHandle::new(),
818 subc_capabilities: Arc::from([
819 CAP_MANIFEST_REGISTRATION.to_string(),
820 CAP_CHANNEL_LIFECYCLE.to_string(),
821 CAP_PING_PONG.to_string(),
822 CAP_SESSION_ATTACH.to_string(),
823 CAP_ADMISSION_FACTS_RELAY.to_string(),
824 CAP_SCOPES_V1.to_string(),
825 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
826 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
827 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
828 ]),
829 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
830 route_bind_relay_timeouts: BTreeMap::new(),
831 route_bind_breakers,
832 route_bind_concurrency,
833 route_outages,
834 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
835 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
836 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
837 storage_config: None,
838 machine_id: None,
839 admission_facts_carrier_module_id: None,
840 admission_facts_targets: None,
841 scopes: Arc::new(RwLock::new(ScopeTable::new(
842 crate::daemon_config::default_scope_authority_owners(),
843 ))),
844 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
845 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
846 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
847 rescan: None,
848 connected_clients: ConnectedClients::new(),
849 counters,
850 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
851 daemon_provenance: DaemonProvenanceFacts::default(),
852 #[cfg(test)]
853 control_dispatch_delay: None,
854 #[cfg(test)]
855 provenance_probe_override: None,
856 #[cfg(test)]
857 route_bind_relay_sent_hold: None,
858 }
859 }
860
861 pub fn with_storage_config(
864 mut self,
865 storage_config: Option<crate::daemon_config::StorageConfig>,
866 ) -> Self {
867 self.storage_config = storage_config;
868 self
869 }
870
871 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
874 self.activity = Some(crate::activity::ActivityLedger::open(
875 run_dir.join("activity.jsonl"),
876 incarnation,
877 ));
878 self
879 }
880
881 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
884 self.machine_id = machine_id;
885 self
886 }
887
888 pub fn with_admission_facts_config(
892 mut self,
893 carrier_module_id: Option<String>,
894 targets: Option<Vec<String>>,
895 ) -> Self {
896 self.admission_facts_carrier_module_id = carrier_module_id;
897 self.admission_facts_targets = targets;
898 self
899 }
900
901 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
905 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
906 self.scope_authority_owners = owners;
907 self
908 }
909
910 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
913 self.route_bind_relay_timeout = timeout;
914 self
915 }
916
917 pub fn with_route_bind_relay_timeouts(
923 mut self,
924 timeouts: impl IntoIterator<Item = (String, Duration)>,
925 ) -> Self {
926 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
927 self
928 }
929
930 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
936 self.route_bind_relay_timeouts
937 .get(module_id)
938 .copied()
939 .unwrap_or(self.route_bind_relay_timeout)
940 }
941
942 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
950 self.route_bind_breaker_threshold = threshold.max(1);
951 self.route_bind_breaker_cooldown = cooldown;
952 self
953 }
954
955 #[cfg(test)]
956 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
957 self.health_probe_timeout = timeout;
958 self
959 }
960
961 #[cfg(test)]
962 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
963 self.control_dispatch_delay = Some(delay);
964 self
965 }
966
967 #[cfg(test)]
971 pub(crate) fn with_route_bind_relay_sent_hold(
972 mut self,
973 hold: Arc<tokio::sync::Barrier>,
974 ) -> Self {
975 self.route_bind_relay_sent_hold = Some(hold);
976 self
977 }
978
979 pub fn with_process_liveness(
980 mut self,
981 process_liveness: Arc<dyn ModuleProcessLiveness>,
982 ) -> Self {
983 self.process_liveness = Some(process_liveness);
984 self
985 }
986
987 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
988 self.supervisor = supervisor;
989 self
990 }
991
992 pub fn with_daemon_provenance(
993 mut self,
994 pid: u32,
995 started_at_ms: u64,
996 executable_path: Option<PathBuf>,
997 build_git_sha: Option<String>,
998 build_lock_digest: Option<String>,
999 ) -> Self {
1000 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1001 let process_start_time = process_start_time(pid);
1002 self.daemon_provenance = DaemonProvenanceFacts {
1003 build: DaemonBuildProvenance {
1004 build_git_sha,
1005 build_lock_digest,
1006 },
1007 pid: Some(pid),
1008 started_at_ms: Some(started_at_ms),
1009 start_clock: None,
1010 executable_path,
1011 executable_identity,
1012 process_start_time,
1013 probe: ExecutableIdentityProbe::default(),
1014 };
1015 self
1016 }
1017
1018 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1019 self.daemon_provenance.start_clock = Some(clock);
1020 self
1021 }
1022
1023 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1025 self.scopes
1026 .write()
1027 .unwrap_or_else(|poisoned| poisoned.into_inner())
1028 .set_wall_clock(Arc::new(clock));
1029 self
1030 }
1031
1032 #[cfg(test)]
1033 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1034 self.provenance_probe_override = Some(result);
1035 self
1036 }
1037
1038 pub fn with_capability_config(
1042 self,
1043 modules: impl IntoIterator<Item = (String, bool)>,
1044 reserved_capabilities: BTreeMap<String, String>,
1045 ) -> Self {
1046 self.capability_evaluator
1047 .configure(modules, reserved_capabilities);
1048 self
1049 }
1050
1051 pub fn with_supervisor_rescan(
1052 mut self,
1053 supervisor: Supervisor,
1054 config_path: impl Into<PathBuf>,
1055 configured_port: Option<u16>,
1056 ) -> Self {
1057 self.rescan = Some(SupervisorRescanContext {
1058 supervisor,
1059 config_path: config_path.into(),
1060 configured_port,
1061 storage_config: self.storage_config.clone(),
1062 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1063 admission_facts_targets: self.admission_facts_targets.clone(),
1064 scope_authority_owners: self.scope_authority_owners.clone(),
1065 });
1066 self
1067 }
1068
1069 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1070 self.connected_clients = connected_clients;
1071 self
1072 }
1073
1074 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1075 Arc::clone(&self.forwarding)
1076 }
1077
1078 pub(crate) fn counters(&self) -> DaemonCounters {
1079 self.counters.clone()
1080 }
1081
1082 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1085 tokio::spawn(async move {
1086 loop {
1087 self.capability_evaluator
1088 .wait_for_change_or_deadline()
1089 .await;
1090 self.refresh_capability_requirements();
1091 }
1092 });
1093 }
1094
1095 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1101 tokio::spawn(async move {
1102 loop {
1103 let has_deadlines = self
1104 .scopes
1105 .read()
1106 .unwrap_or_else(|poisoned| poisoned.into_inner())
1107 .has_deadlines();
1108 if !has_deadlines {
1109 self.scope_deadline_added.notified().await;
1110 continue;
1111 }
1112 tokio::time::sleep(Duration::from_secs(1)).await;
1113 if let Err(error) = self.sweep_expired_scopes() {
1114 warn!(%error, "scope expiry sweep failed");
1115 }
1116 }
1117 });
1118 }
1119
1120 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1121 let mut table = self
1122 .scopes
1123 .write()
1124 .unwrap_or_else(|poisoned| poisoned.into_inner());
1125 let swept = table.sweep_expired();
1126 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1127 drop(table);
1128 self.close_scope_drained_routes(drained);
1129 Ok(())
1130 }
1131
1132 fn publish_scope_changes(
1133 &self,
1134 changes: &[crate::scopes::ScopeTagChange],
1135 expired: &[crate::scopes::ScopeExpired],
1136 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1137 if changes.is_empty() {
1138 return Ok(Vec::new());
1139 }
1140 let drained = self
1141 .forwarding
1142 .publish_scope_changes(changes)
1143 .map_err(RouterError::Forwarding)?;
1144 let mut counts = HashMap::new();
1145 for route in &drained {
1146 *counts
1147 .entry((
1148 route.scope.owner.as_str(),
1149 route.scope.scope_ref.as_str(),
1150 route.scope.tag.scope_epoch,
1151 ))
1152 .or_insert(0usize) += 1;
1153 }
1154 for scope in expired {
1155 let routes_closed = counts
1156 .get(&(
1157 scope.owner.as_str(),
1158 scope.scope_ref.as_str(),
1159 scope.scope_epoch,
1160 ))
1161 .copied()
1162 .unwrap_or(0);
1163 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1164 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1165 routes_closed, "scope expired");
1166 }
1167 Ok(drained)
1168 }
1169
1170 fn runtime_capability_snapshot(
1171 &self,
1172 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1173 let runtime = self
1174 .supervisor
1175 .list()
1176 .into_iter()
1177 .map(|module| {
1178 let status = module.status().map_err(|err| {
1179 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1180 })?;
1181 Ok(RuntimeModule {
1182 module_id: status.module_id,
1183 state: status.state,
1184 enabled: status.enabled,
1185 })
1186 })
1187 .collect::<Result<Vec<_>, RouterError>>()?;
1188 let (_, registrations) = self.registry.list_modules().map_err(|err| {
1189 RouterError::backend(
1190 0,
1191 0,
1192 format!("failed to list capability registrations: {err}"),
1193 )
1194 })?;
1195 let registrations = registrations
1196 .into_iter()
1197 .map(|registration| RegisteredModule {
1198 module_id: registration.manifest.module_id,
1199 module_version: registration.manifest.module_version,
1200 capabilities: registration.manifest.capabilities,
1201 })
1202 .collect();
1203 Ok((runtime, registrations))
1204 }
1205
1206 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1213 let cached_registration = RegisteredModule {
1214 module_id: registration.manifest.module_id.clone(),
1215 module_version: registration.manifest.module_version.clone(),
1216 capabilities: registration.manifest.capabilities.clone(),
1217 };
1218 if self.capability_evaluator.record_hello(&cached_registration) {
1219 warn!(
1220 module_id = %cached_registration.module_id,
1221 "capability claims drifted from the cached manifest"
1222 );
1223 }
1224 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1225 self.enforce_capability_denies();
1226 }
1227 self.refresh_capability_requirements();
1228 }
1229
1230 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1234 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1235 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1236 self.supervisor.set_swap_promotion_observer(observer);
1237 }
1238
1239 pub fn refresh_capability_requirements(&self) {
1240 match self.runtime_capability_snapshot() {
1241 Ok((runtime, registrations)) => {
1242 log_requirement_events(
1243 self.capability_evaluator
1244 .evaluate_now(&runtime, ®istrations),
1245 );
1246 }
1247 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1248 }
1249 }
1250
1251 fn enforce_capability_denies(&self) {
1255 let (_, registrations) = match self.registry.list_modules() {
1256 Ok(snapshot) => snapshot,
1257 Err(err) => {
1258 warn!(error = %err, "failed to read registrations for capability deny census");
1259 return;
1260 }
1261 };
1262 let manifests = registrations
1263 .into_iter()
1264 .map(|registration| {
1265 (
1266 registration.manifest.module_id.clone(),
1267 registration.manifest,
1268 )
1269 })
1270 .collect::<BTreeMap<_, _>>();
1271 let census = match self.forwarding.route_census(None) {
1272 Ok(census) => census,
1273 Err(err) => {
1274 warn!(error = %err, "failed to read route census for capability deny enforcement");
1275 return;
1276 }
1277 };
1278
1279 for (target_module_id, routes) in census {
1280 let Some(target_manifest) = manifests.get(&target_module_id) else {
1281 continue;
1282 };
1283 let mut closed_routes = Vec::new();
1284 let mut module_goodbyes = Vec::new();
1285 for route in routes {
1286 let Principal::Reserved {
1287 module_id: opening_module_id,
1288 } = &route.principal
1289 else {
1290 continue;
1291 };
1292 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1293 continue;
1294 };
1295 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1296 continue;
1297 };
1298
1299 match self.forwarding.release_client_route(
1300 route.goodbye_target.connection_id,
1301 route.goodbye_target.channel,
1302 route.goodbye_target.epoch,
1303 ) {
1304 Ok(RouteRelease::Removed(module_goodbye)) => {
1305 warn!(
1306 opening_module_id,
1307 target_module_id,
1308 capability,
1309 "force-closing route because an attested capability deny edge now matches"
1310 );
1311 closed_routes.push(route);
1312 module_goodbyes.push(module_goodbye);
1313 }
1314 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1315 Err(err) => warn!(
1316 opening_module_id,
1317 target_module_id,
1318 capability,
1319 error = %err,
1320 "failed to force-close capability-denied route"
1321 ),
1322 }
1323 }
1324
1325 if closed_routes.is_empty() {
1326 continue;
1327 }
1328 send_route_control_pushes(
1329 &self.forwarding,
1330 closed_routes,
1331 ClientControlPush::RouteClosed {
1332 module_id: target_module_id,
1333 channels: Vec::new(),
1334 reason: RouteCloseReason::CapabilityDenied,
1335 drained: false,
1336 abandoned: 0,
1337 excluded_subscriptions: 0,
1338 terminal: Some(false),
1339 },
1340 );
1341 self.emit_route_goodbyes(module_goodbyes);
1342 }
1343 }
1344
1345 fn not_ready_reason(
1351 &self,
1352 registration: &crate::registry::ModuleRegistration,
1353 ) -> Option<NotReadyReason> {
1354 if !registration.ready {
1355 return Some(NotReadyReason {
1356 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1357 capability: None,
1358 });
1359 }
1360 self.first_unprovided_required_capability(registration)
1361 .map(|capability| NotReadyReason {
1362 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1363 capability: Some(capability),
1364 })
1365 }
1366
1367 fn first_unprovided_required_capability(
1385 &self,
1386 registration: &crate::registry::ModuleRegistration,
1387 ) -> Option<String> {
1388 let required = registration
1389 .manifest
1390 .capabilities
1391 .iter()
1392 .flat_map(|declarations| declarations.requires.iter())
1393 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1394 .map(|requirement| requirement.capability.as_str())
1395 .collect::<BTreeSet<_>>();
1396 if required.is_empty() {
1397 return None;
1398 }
1399 let module_id = registration.manifest.module_id.as_str();
1400 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1401 if required
1402 .iter()
1403 .any(|capability| verdict(capability).is_none())
1404 {
1405 self.refresh_capability_requirements();
1406 }
1407 required
1408 .into_iter()
1409 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1410 .map(str::to_string)
1411 }
1412
1413 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1414 self.capability_evaluator
1415 .statuses()
1416 .into_iter()
1417 .map(capability_requirement_status)
1418 .collect()
1419 }
1420
1421 fn deregister_connection(
1428 &self,
1429 connection_id: ConnectionId,
1430 reason: RegistrationEndReason,
1431 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1432 self.registry
1433 .deregister_connection_with_reason(connection_id, reason)
1434 }
1435
1436 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1437 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1438 return None;
1439 }
1440 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1441 parse_client_control_request(&frame.body)
1442 else {
1443 return None;
1444 };
1445 Some(target_module_id(&target).to_string())
1446 }
1447
1448 pub(crate) fn route_open_capacity_refusal(
1449 &self,
1450 ctx: &RouteCtx,
1451 frame: &Frame,
1452 target_module_id: &str,
1453 in_flight: usize,
1454 limit: usize,
1455 ) -> Result<Frame, RouterError> {
1456 self.route_open_admission_refusal_frame(
1457 ctx,
1458 frame,
1459 target_module_id,
1460 "open_admission_full",
1461 (in_flight, limit),
1462 format!(
1463 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1464 ),
1465 )
1466 }
1467
1468 fn route_open_target_capacity_refusal(
1469 &self,
1470 ctx: &RouteCtx,
1471 frame: &Frame,
1472 target_module_id: &str,
1473 in_flight: usize,
1474 ) -> Result<Frame, RouterError> {
1475 self.route_open_admission_refusal_frame(
1476 ctx,
1477 frame,
1478 target_module_id,
1479 "target_binds_full",
1480 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1481 format!(
1482 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1483 ),
1484 )
1485 }
1486
1487 fn route_open_admission_refusal_frame(
1495 &self,
1496 ctx: &RouteCtx,
1497 frame: &Frame,
1498 target_module_id: &str,
1499 reason: &'static str,
1500 (in_flight, limit): (usize, usize),
1501 message: impl Into<String>,
1502 ) -> Result<Frame, RouterError> {
1503 let code = error_codes::TARGET_UNAVAILABLE;
1504 self.counters.increment_route_open_refused(code);
1505 info!(
1506 target: "control",
1507 code,
1508 reason,
1509 module_id = ?target_module_id,
1510 connection_id = ctx.connection_id.get(),
1511 in_flight,
1512 limit,
1513 "route.open refused"
1514 );
1515 control_error_frame(frame, code, message.into())
1516 }
1517
1518 #[cfg(test)]
1524 pub fn handle_control(
1525 &self,
1526 connection_id: ConnectionId,
1527 frame: Frame,
1528 ) -> Result<Vec<Frame>, RouterError> {
1529 match frame.header.ty {
1530 FrameType::Ping => Ok(vec![pong(&frame)?]),
1531 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1532 FrameType::Goodbye => self.handle_goodbye(connection_id),
1533 ty => Ok(vec![control_error_frame(
1534 &frame,
1535 "unsupported_control_frame",
1536 format!("unsupported channel-0 frame {ty:?}"),
1537 )?]),
1538 }
1539 }
1540
1541 pub async fn handle_control_frame(
1542 &self,
1543 ctx: &RouteCtx,
1544 frame: Frame,
1545 ) -> Result<Vec<Frame>, RouterError> {
1546 self.handle_control_frame_timed(ctx, frame, None).await
1547 }
1548
1549 pub(crate) async fn handle_control_frame_timed(
1550 &self,
1551 ctx: &RouteCtx,
1552 frame: Frame,
1553 dispatch_started_at: Option<StdInstant>,
1554 ) -> Result<Vec<Frame>, RouterError> {
1555 match frame.header.ty {
1556 FrameType::Ping => Ok(vec![pong(&frame)?]),
1557 FrameType::Hello => {
1558 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1559 }
1560 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1561 FrameType::Cancel => {
1562 if self
1565 .forwarding
1566 .operator_confirms()
1567 .cancel(ctx.connection_id, frame.header.corr)
1568 || self
1569 .supervisor
1570 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1571 {
1572 Ok(Vec::new())
1573 } else {
1574 Ok(vec![control_error_frame(
1575 &frame,
1576 "unknown_subscription",
1577 "no supervisor spawn subscription has this correlation id",
1578 )?])
1579 }
1580 }
1581 FrameType::Request => {
1582 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1585 if op
1586 .as_ref()
1587 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1588 {
1589 return self.handle_activity_record(ctx, frame).await;
1590 }
1591 if op
1592 .as_ref()
1593 .is_some_and(|probe| probe.op == "operator.confirm")
1594 {
1595 return self.handle_operator_confirm(ctx, frame);
1596 }
1597 if self
1598 .forwarding
1599 .module_endpoint_for_connection(ctx.connection_id)
1600 .map_err(RouterError::Forwarding)?
1601 .is_some()
1602 {
1603 if !is_known_module_request_op(&frame.body) {
1604 return Ok(vec![control_error_frame(
1605 &frame,
1606 "unsupported_control_frame",
1607 "module-originated channel-0 REQUEST is not supported",
1608 )?]);
1609 }
1610 let request = match parse_module_control_request_from_module(&frame.body) {
1611 Ok(request) => request,
1612 Err((err, ControlRequestBodyError::UnknownOp)) => {
1613 return Ok(vec![control_error_frame(
1614 &frame,
1615 "unsupported_control_frame",
1616 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1617 )?])
1618 }
1619 Err((err, ControlRequestBodyError::InvalidBody)) => {
1620 return Ok(vec![control_error_frame(
1621 &frame,
1622 "invalid_control_body",
1623 format!("malformed module control body: {err}"),
1624 )?])
1625 }
1626 };
1627 let op = module_control_request_op(&request);
1628 let corr = frame.header.corr;
1629 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1630 let result =
1631 self.handle_module_control_request(ctx.connection_id, frame, request);
1632 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1633 return result;
1634 }
1635
1636 if is_known_module_request_op(&frame.body) {
1637 return Ok(vec![control_error_frame(
1638 &frame,
1639 "not_registered",
1640 "catalog.update requires an active module registration owned by this connection",
1641 )?]);
1642 }
1643
1644 let request = match parse_client_control_request(&frame.body) {
1645 Ok(request) => request,
1646 Err((err, ControlRequestBodyError::UnknownOp)) => {
1647 return Ok(vec![control_error_frame(
1648 &frame,
1649 "unknown_control_op",
1650 format!("unknown client control op: {err}"),
1651 )?])
1652 }
1653 Err((err, ControlRequestBodyError::InvalidBody)) => {
1654 return Ok(vec![control_error_frame(
1655 &frame,
1656 "invalid_control_body",
1657 format!("malformed client control body: {err}"),
1658 )?])
1659 }
1660 };
1661 let op = client_control_request_op(&request);
1662 let corr = frame.header.corr;
1663 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1664 #[cfg(test)]
1665 if let Some(delay) = self.control_dispatch_delay {
1666 tokio::time::sleep(delay).await;
1667 }
1668 let result = self
1669 .handle_client_control_request(ctx, frame, request)
1670 .await;
1671 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1672 result
1673 }
1674 FrameType::Push => {
1675 let Some(endpoint) = self
1676 .forwarding
1677 .module_endpoint_for_connection(ctx.connection_id)
1678 .map_err(RouterError::Forwarding)?
1679 else {
1680 return Ok(vec![control_error_frame(
1681 &frame,
1682 "unsupported_control_frame",
1683 "client-originated channel-0 PUSH is not supported",
1684 )?]);
1685 };
1686 self.handle_status_update(endpoint, frame)
1687 }
1688 FrameType::Response | FrameType::Error
1689 if self
1690 .forwarding
1691 .module_endpoint_for_connection(ctx.connection_id)
1692 .map_err(RouterError::Forwarding)?
1693 .is_some() =>
1694 {
1695 self.handle_module_relay_response(ctx.connection_id, frame)
1696 }
1697 ty => Ok(vec![control_error_frame(
1698 &frame,
1699 "unsupported_control_frame",
1700 format!("unsupported channel-0 frame {ty:?}"),
1701 )?]),
1702 }
1703 }
1704
1705 pub fn cleanup_connection(
1706 &self,
1707 connection_id: ConnectionId,
1708 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1709 self.cleanup_connection_with_end_reason(
1710 connection_id,
1711 RegistrationEndReason::ConnectionClosed,
1712 )
1713 }
1714
1715 fn cleanup_connection_with_end_reason(
1716 &self,
1717 connection_id: ConnectionId,
1718 requested_reason: RegistrationEndReason,
1719 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1720 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1721 self.registry
1722 .get_module_by_connection(connection_id)?
1723 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
1724 .and_then(|module| module.registration_end_reason().ok().flatten())
1725 .unwrap_or(requested_reason)
1726 } else {
1727 requested_reason
1728 };
1729 let crash_closed = self
1730 .registry
1731 .get_module_by_connection(connection_id)?
1732 .and_then(|registration| {
1733 self.forwarding
1734 .module_endpoint_for_connection(connection_id)
1735 .ok()
1736 .flatten()
1737 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1738 .map(|routes| (registration.manifest.module_id, routes))
1739 });
1740 let crash_closed = crash_closed.map(|(module_id, routes)| {
1741 let terminal = match self.supervisor.get(&module_id) {
1742 None => false,
1743 Some(module) => match module.will_recover_after_connection_loss() {
1744 Ok(will_recover) => !will_recover,
1745 Err(err) => {
1746 warn!(
1747 %module_id,
1748 error = %err,
1749 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1750 );
1751 false
1752 }
1753 },
1754 };
1755 let reason = match self.forwarding.is_daemon_draining() {
1759 Ok(true) => RouteCloseReason::Restart,
1760 Ok(false) => RouteCloseReason::Crash,
1761 Err(err) => {
1762 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1763 RouteCloseReason::Crash
1764 }
1765 };
1766 (module_id, routes, reason, terminal)
1767 });
1768 let registrations = self.deregister_connection(connection_id, end_reason);
1769 let cleanup = if crash_closed.is_some() {
1770 self.forwarding.cleanup_connection_counted(connection_id)
1771 } else {
1772 self.forwarding
1778 .cleanup_connection(connection_id)
1779 .map(|released| crate::forwarding::ConnectionCleanup {
1780 released,
1781 abandoned_relays: 0,
1782 })
1783 };
1784 if let Some((module_id, routes, reason, terminal)) = crash_closed {
1789 let abandoned = cleanup
1790 .as_ref()
1791 .map_or(0, |cleanup| cleanup.abandoned_relays);
1792 send_route_control_pushes(
1793 &self.forwarding,
1794 routes,
1795 ClientControlPush::RouteClosed {
1796 module_id,
1797 channels: Vec::new(),
1798 reason,
1799 drained: false,
1800 abandoned,
1801 excluded_subscriptions: 0,
1802 terminal: Some(terminal),
1803 },
1804 );
1805 }
1806 if let Ok(cleanup) = cleanup {
1807 self.emit_route_goodbyes(cleanup.released);
1808 }
1809 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1813 crate::supervise::notify_registration_release();
1814 self.capability_evaluator.wake_deadline_loop();
1815 self.refresh_capability_requirements();
1816 }
1817 self.supervisor.remove_spawn_subscribers(connection_id);
1818 self.hello_launch_nonces
1821 .lock()
1822 .unwrap_or_else(|poisoned| poisoned.into_inner())
1823 .forget(connection_id);
1824 self.scopes
1825 .write()
1826 .unwrap_or_else(|poisoned| poisoned.into_inner())
1827 .release_connection(connection_id);
1828 registrations
1829 }
1830
1831 pub(crate) fn handle_route_goodbye(
1832 &self,
1833 connection_id: ConnectionId,
1834 route_channel: u16,
1835 route_epoch: u32,
1836 ) -> Result<bool, RouterError> {
1837 debug!(
1838 connection_id = connection_id.get(),
1839 route_channel, route_epoch, "handling route GOODBYE"
1840 );
1841 let RouteRelease::Removed(released_route) = self
1842 .forwarding
1843 .release_client_route(connection_id, route_channel, route_epoch)
1844 .map_err(RouterError::Forwarding)?
1845 else {
1846 return Ok(false);
1847 };
1848 self.emit_route_goodbyes(vec![released_route]);
1849 Ok(true)
1850 }
1851
1852 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1853 for released in released_routes {
1854 let frame = match Frame::build_with_version(
1855 released.negotiated_ver,
1856 FrameType::Goodbye,
1857 control_flags(),
1858 released.channel,
1859 released.epoch,
1860 0,
1861 Vec::new(),
1862 ) {
1863 Ok(frame) => frame,
1864 Err(err) => {
1865 warn!(
1866 route_channel = released.channel,
1867 error = %err,
1868 "failed to build route GOODBYE frame"
1869 );
1870 continue;
1871 }
1872 };
1873 if !released.close_on_delivery_failure() {
1874 crate::forwarding::send_module_route_goodbye(
1875 &self.counters,
1876 &released.sink,
1877 frame,
1878 released.module_id.as_deref(),
1879 "client route released",
1880 );
1881 continue;
1882 }
1883 if let Err(err) = released.sink.try_send(frame) {
1884 warn!(
1885 target_connection_id = released.connection_id.get(),
1886 route_channel = released.channel,
1887 error = %err,
1888 "route GOODBYE was not delivered to client; closing target connection"
1889 );
1890 if self
1891 .forwarding
1892 .escalate_client_delivery_failure(
1893 released.connection_id,
1894 released.channel,
1895 released.epoch,
1896 CloseReason::new(
1897 "route_goodbye_delivery_failed",
1898 format!(
1899 "failed to enqueue route GOODBYE for channel {}: {err}",
1900 released.channel
1901 ),
1902 ),
1903 crate::forwarding::UndeliveredFrame {
1904 module_id: released.module_id.as_deref(),
1905 sink: &released.sink,
1906 },
1907 )
1908 .unwrap_or(false)
1909 {
1910 self.counters.increment_goodbye_relay_client_failed();
1911 }
1912 }
1913 }
1914 }
1915
1916 fn send_abandoned_route_bind_goodbye(
1928 &self,
1929 module_sink: &crate::FrameSink,
1930 negotiated_ver: u8,
1931 module_channel: u16,
1932 module_epoch: u32,
1933 ) {
1934 let frame = match Frame::build_with_version(
1935 negotiated_ver,
1936 FrameType::Goodbye,
1937 control_flags(),
1938 module_channel,
1939 module_epoch,
1940 0,
1941 Vec::new(),
1942 ) {
1943 Ok(frame) => frame,
1944 Err(err) => {
1945 warn!(
1946 route_channel = module_channel,
1947 error = %err,
1948 "failed to build GOODBYE for abandoned route.bind"
1949 );
1950 return;
1951 }
1952 };
1953 crate::forwarding::send_module_route_goodbye(
1954 &self.counters,
1955 module_sink,
1956 frame,
1957 None,
1958 "abandoned route.bind",
1959 );
1960 }
1961
1962 fn handle_hello(
1963 &self,
1964 connection_id: ConnectionId,
1965 sink: Option<crate::FrameSink>,
1966 frame: Frame,
1967 ) -> Result<Vec<Frame>, RouterError> {
1968 debug!(
1969 connection_id = connection_id.get(),
1970 corr = frame.header.corr,
1971 "handling HELLO"
1972 );
1973 if self
1977 .registry
1978 .get_module_by_connection(connection_id)
1979 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
1980 .is_some()
1981 {
1982 return Ok(vec![control_error_frame(
1983 &frame,
1984 "invalid_hello",
1985 "connection is already registered as a module",
1986 )?]);
1987 }
1988 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1989 Ok(value) => value,
1990 Err(err) => {
1991 return Ok(vec![control_error_frame(
1992 &frame,
1993 "invalid_hello",
1994 format!("malformed HELLO body: {err}"),
1995 )?])
1996 }
1997 };
1998 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1999 return Ok(vec![control_error_frame(
2000 &frame,
2001 "invalid_capability_grammar",
2002 err.to_string(),
2003 )?]);
2004 }
2005 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2006 return Ok(vec![control_error_frame(
2007 &frame,
2008 "invalid_manifest",
2009 err.to_string(),
2010 )?]);
2011 }
2012 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2013 return Ok(vec![control_error_frame(
2014 &frame,
2015 "invalid_event_declaration",
2016 err.to_string(),
2017 )?]);
2018 }
2019 if let Some(provenance) = hello_value
2020 .get("manifest")
2021 .and_then(|manifest| manifest.get("provenance"))
2022 {
2023 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2024 return Ok(vec![control_error_frame(
2025 &frame,
2026 "invalid_manifest",
2027 format!("malformed manifest provenance: {err}"),
2028 )?]);
2029 }
2030 }
2031 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2032 Ok(hello) => hello,
2033 Err(err) => {
2034 return Ok(vec![control_error_frame(
2035 &frame,
2036 "invalid_hello",
2037 format!("malformed HELLO body: {err}"),
2038 )?])
2039 }
2040 };
2041
2042 if hello.protocol_ver != hello.manifest.protocol_ver {
2043 return Ok(vec![control_error_frame(
2044 &frame,
2045 "invalid_manifest",
2046 format!(
2047 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2048 hello.protocol_ver, hello.manifest.protocol_ver
2049 ),
2050 )?]);
2051 }
2052
2053 if hello.manifest.module_id.trim().is_empty() {
2054 return Ok(vec![control_error_frame(
2055 &frame,
2056 "invalid_manifest",
2057 "manifest module_id must not be empty",
2058 )?]);
2059 }
2060
2061 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2062 Ok(negotiated_ver) => negotiated_ver,
2063 Err(message) => {
2064 return Ok(vec![control_error_frame(
2065 &frame,
2066 "version_unsupported",
2067 message,
2068 )?])
2069 }
2070 };
2071
2072 let swap_admission = self
2080 .supervisor
2081 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2082 if swap_admission == SwapHelloAdmission::Refused {
2083 warn!(
2084 module_id = %hello.manifest.module_id,
2085 connection_id = connection_id.get(),
2086 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2087 );
2088 return Ok(vec![control_error_frame(
2089 &frame,
2090 "swap_token_invalid",
2091 format!(
2092 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2093 hello.manifest.module_id
2094 ),
2095 )?]);
2096 }
2097 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2098
2099 if let Some(rejection) = (!swap_candidate)
2107 .then(|| {
2108 self.supervisor.reserved_hello_rejection(
2109 &hello.manifest.module_id,
2110 hello.launch_nonce.as_deref(),
2111 )
2112 })
2113 .flatten()
2114 {
2115 let message = match rejection {
2116 ReservedHelloRejection::Exact { module_id } => format!(
2117 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2118 ),
2119 ReservedHelloRejection::Prefix {
2120 prefix,
2121 owner_module_id,
2122 } => format!(
2123 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2124 hello.manifest.module_id
2125 ),
2126 };
2127 return Ok(vec![control_error_frame(
2128 &frame,
2129 "reserved_module",
2130 message,
2131 )?]);
2132 }
2133
2134 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2135 &hello.manifest.module_id,
2136 hello.manifest.capabilities.as_ref(),
2137 );
2138 if let Some(refusal) = reserved_capability_refusals.first() {
2139 let capability = refusal.capability.clone();
2140 let bound_module = refusal.claimants[0].clone();
2141 log_duplicate_claim_events(reserved_capability_refusals);
2142 return Ok(vec![control_error_frame(
2143 &frame,
2144 "reserved_capability",
2145 format!(
2146 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2147 capability, bound_module, hello.manifest.module_id
2148 ),
2149 )?]);
2150 }
2151
2152 if self
2155 .forwarding
2156 .connection_has_client_routes(connection_id)
2157 .map_err(RouterError::Forwarding)?
2158 {
2159 return Ok(vec![control_error_frame(
2160 &frame,
2161 "invalid_hello",
2162 "connection has open client routes and cannot also register as a module",
2163 )?]);
2164 }
2165
2166 self.hello_launch_nonces
2171 .lock()
2172 .unwrap_or_else(|poisoned| poisoned.into_inner())
2173 .record(connection_id, hello.launch_nonce.as_deref());
2174 let control_ops = effective_module_control_ops(hello.control_ops);
2175 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2178 if swap_candidate {
2179 return self.register_swap_candidate(
2180 connection_id,
2181 sink,
2182 &frame,
2183 hello.manifest,
2184 negotiated_ver,
2185 control_ops,
2186 hello_ack,
2187 );
2188 }
2189 let registration = match self.registry.register_with_control_ops(
2190 hello.manifest,
2191 negotiated_ver,
2192 connection_id,
2193 control_ops,
2194 ) {
2195 Ok(registration) => registration,
2196 Err(RegistryError::DuplicateModuleId { module_id }) => {
2197 return Ok(vec![control_error_frame(
2198 &frame,
2199 "duplicate_module_id",
2200 format!(
2201 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2202 ),
2203 )?])
2204 }
2205 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2206 return Ok(vec![control_error_frame(
2207 &frame,
2208 "invalid_module_id",
2209 err.to_string(),
2210 )?])
2211 }
2212 Err(err) => {
2213 return Ok(vec![control_error_frame(
2214 &frame,
2215 "registry_error",
2216 err.to_string(),
2217 )?])
2218 }
2219 };
2220
2221 let reply = if let Some(sink) = sink {
2222 let concurrency = manifest_concurrency(®istration.manifest);
2238 if let Err(err) = self.forwarding.register_module_connection_acked(
2239 connection_id,
2240 registration.manifest.module_id.clone(),
2241 negotiated_ver,
2242 concurrency,
2243 sink,
2244 hello_ack,
2245 ) {
2246 if matches!(
2250 self.deregister_connection(
2251 connection_id,
2252 RegistrationEndReason::RegistrationFailed,
2253 ),
2254 Ok(r) if !r.is_empty()
2255 ) {
2256 crate::supervise::notify_registration_release();
2257 }
2258 return Ok(vec![control_error_frame(
2259 &frame,
2260 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2261 "invalid_hello"
2262 } else {
2263 forwarding_error_code(&err)
2264 },
2265 err.to_string(),
2266 )?]);
2267 }
2268 Vec::new()
2269 } else {
2270 vec![hello_ack]
2273 };
2274
2275 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2284 info!(
2285 module_id = %registration.manifest.module_id,
2286 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2287 );
2288 }
2289
2290 self.apply_registration_capabilities(®istration);
2291
2292 info!(
2293 module_id = %registration.manifest.module_id,
2294 module_version = %registration.manifest.module_version,
2295 negotiated_ver,
2296 routable_provider = manifest_provides_routable_role(®istration.manifest),
2297 connection_id = connection_id.get(),
2298 "module registered"
2299 );
2300
2301 Ok(reply)
2302 }
2303
2304 #[allow(clippy::too_many_arguments)]
2314 fn register_swap_candidate(
2315 &self,
2316 connection_id: ConnectionId,
2317 sink: Option<crate::FrameSink>,
2318 frame: &Frame,
2319 manifest: ModuleManifest,
2320 negotiated_ver: u8,
2321 control_ops: Vec<String>,
2322 hello_ack: Frame,
2323 ) -> Result<Vec<Frame>, RouterError> {
2324 let module_id = manifest.module_id.clone();
2325 let registration = match self.registry.register_candidate_with_control_ops(
2326 manifest,
2327 negotiated_ver,
2328 connection_id,
2329 control_ops,
2330 ) {
2331 Ok(registration) => registration,
2332 Err(RegistryError::DuplicateModuleId { module_id }) => {
2333 return Ok(vec![control_error_frame(
2334 frame,
2335 "duplicate_module_id",
2336 format!(
2337 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2338 ),
2339 )?])
2340 }
2341 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2342 return Ok(vec![control_error_frame(
2343 frame,
2344 "invalid_module_id",
2345 err.to_string(),
2346 )?])
2347 }
2348 Err(err) => {
2349 return Ok(vec![control_error_frame(
2350 frame,
2351 "registry_error",
2352 err.to_string(),
2353 )?])
2354 }
2355 };
2356 let reply = if let Some(sink) = sink {
2357 let concurrency = manifest_concurrency(®istration.manifest);
2361 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2362 connection_id,
2363 module_id.clone(),
2364 negotiated_ver,
2365 concurrency,
2366 sink,
2367 hello_ack,
2368 ) {
2369 if matches!(
2370 self.deregister_connection(
2371 connection_id,
2372 RegistrationEndReason::RegistrationFailed,
2373 ),
2374 Ok(r) if !r.is_empty()
2375 ) {
2376 crate::supervise::notify_registration_release();
2377 }
2378 return Ok(vec![control_error_frame(
2379 frame,
2380 forwarding_error_code(&err),
2381 err.to_string(),
2382 )?]);
2383 }
2384 Vec::new()
2385 } else {
2386 vec![hello_ack]
2387 };
2388 self.supervisor.mark_swap_candidate_admitted(&module_id);
2389 info!(
2390 module_id = %module_id,
2391 module_version = %registration.manifest.module_version,
2392 negotiated_ver,
2393 ready = registration.ready,
2394 connection_id = connection_id.get(),
2395 "swap candidate registered; not routable until cutover"
2396 );
2397 Ok(reply)
2398 }
2399
2400 fn build_hello_ack(
2401 &self,
2402 frame: &Frame,
2403 negotiated_ver: u8,
2404 module_id: &str,
2405 ) -> Result<Frame, RouterError> {
2406 let ack = ModuleHelloAckBody {
2407 negotiated_ver,
2408 subc_ops: module_subc_ops(),
2409 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2410 storage: self
2411 .storage_config
2412 .as_ref()
2413 .map(|cfg| cfg.descriptor_for(module_id)),
2414 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2415 };
2416 let body = serde_json::to_vec(&ack).map_err(|err| {
2417 RouterError::backend(
2418 0,
2419 frame.header.corr,
2420 format!("failed to encode HELLO_ACK: {err}"),
2421 )
2422 })?;
2423
2424 Frame::build_with_version(
2425 negotiated_ver,
2426 FrameType::HelloAck,
2427 control_flags(),
2428 0,
2429 0,
2430 frame.header.corr,
2431 body,
2432 )
2433 .map_err(RouterError::FrameBuild)
2434 }
2435
2436 async fn handle_client_control_request(
2437 &self,
2438 ctx: &RouteCtx,
2439 frame: Frame,
2440 request: ClientControlRequest,
2441 ) -> Result<Vec<Frame>, RouterError> {
2442 match request {
2443 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2444 ClientControlRequest::CatalogList { module_id } => {
2445 self.handle_catalog_list(frame, module_id)
2446 }
2447 ClientControlRequest::RouteOpen {
2448 target,
2449 identity,
2450 consumer_identity,
2451 consumer_capabilities,
2452 role_versions,
2453 admission_facts,
2454 scope,
2455 } => {
2456 self.handle_route_open(
2457 ctx,
2458 frame,
2459 RouteOpenRequest {
2460 target,
2461 identity,
2462 consumer_identity,
2463 consumer_capabilities,
2464 role_versions,
2465 admission_facts,
2466 scope,
2467 },
2468 )
2469 .await
2470 }
2471 ClientControlRequest::RoutePoll {
2472 route_channel,
2473 route_epoch,
2474 kind,
2475 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2476 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2477 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2478 self.handle_supervisor_spawn_snapshot(frame)
2479 }
2480 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2481 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2482 }
2483 ClientControlRequest::SupervisorRestart {
2484 module_id,
2485 drain_timeout_ms,
2486 } => {
2487 self.log_supervisor_request_received(
2488 ctx,
2489 frame.header.corr,
2490 ops::SUPERVISOR_RESTART,
2491 Some(&module_id),
2492 None,
2493 )?;
2494 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2495 .await
2496 }
2497 ClientControlRequest::SupervisorSwap {
2498 module_id,
2499 ready_timeout_ms,
2500 } => {
2501 self.log_supervisor_request_received(
2502 ctx,
2503 frame.header.corr,
2504 ops::SUPERVISOR_SWAP,
2505 Some(&module_id),
2506 None,
2507 )?;
2508 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2509 .await
2510 }
2511 ClientControlRequest::SupervisorReload { module_id } => {
2512 self.log_supervisor_request_received(
2513 ctx,
2514 frame.header.corr,
2515 ops::SUPERVISOR_RELOAD,
2516 Some(&module_id),
2517 None,
2518 )?;
2519 self.handle_supervisor_reload(frame, module_id).await
2520 }
2521 ClientControlRequest::SupervisorRescan { preview } => {
2522 if !preview {
2523 self.log_supervisor_request_received(
2524 ctx,
2525 frame.header.corr,
2526 ops::SUPERVISOR_RESCAN,
2527 None,
2528 None,
2529 )?;
2530 }
2531 self.handle_supervisor_rescan(frame, preview).await
2532 }
2533 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2534 self.log_supervisor_request_received(
2535 ctx,
2536 frame.header.corr,
2537 ops::SUPERVISOR_RELEASE_RESERVED,
2538 Some(&module_id),
2539 None,
2540 )?;
2541 self.handle_supervisor_release_reserved(frame, module_id)
2542 .await
2543 }
2544 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2545 self.log_supervisor_request_received(
2546 ctx,
2547 frame.header.corr,
2548 ops::SUPERVISOR_SET_ENABLED,
2549 Some(&module_id),
2550 Some(enabled),
2551 )?;
2552 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2553 .await
2554 }
2555 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2556 self.handle_supervisor_health_probe(frame, module_id).await
2557 }
2558 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2559 ClientControlRequest::SupervisorRoutes { module_id } => {
2560 self.handle_supervisor_routes(frame, module_id)
2561 }
2562 ClientControlRequest::SupervisorProvenance { module_id } => {
2563 self.handle_supervisor_provenance(frame, module_id).await
2564 }
2565 ClientControlRequest::SupervisorStderrTail {
2566 module_id,
2567 max_lines,
2568 max_bytes,
2569 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2570 ClientControlRequest::SupervisorTerminals { module_id } => {
2571 self.handle_supervisor_terminals(frame, module_id).await
2572 }
2573 }
2574 }
2575
2576 fn log_supervisor_request_received(
2577 &self,
2578 ctx: &RouteCtx,
2579 corr: u64,
2580 op: &'static str,
2581 module_id: Option<&str>,
2582 enabled: Option<bool>,
2583 ) -> Result<(), RouterError> {
2584 let caller = self
2585 .registry
2586 .get_module_by_connection(ctx.connection_id)
2587 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2588 .map(|registration| Principal::Reserved {
2589 module_id: registration.manifest.module_id,
2590 })
2591 .unwrap_or(Principal::Direct);
2592 let caller = principal_label(&caller);
2593 info!(
2596 target: "control",
2597 op,
2598 module_id,
2599 enabled,
2600 connection_id = ctx.connection_id.get(),
2601 caller = %caller,
2602 "supervisor request received"
2603 );
2604 Ok(())
2605 }
2606
2607 fn handle_module_control_request(
2608 &self,
2609 connection_id: ConnectionId,
2610 frame: Frame,
2611 request: ModuleControlRequestFromModule,
2612 ) -> Result<Vec<Frame>, RouterError> {
2613 match request {
2614 ModuleControlRequestFromModule::CatalogUpdate {
2615 provides,
2616 capabilities,
2617 ready,
2618 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2619 ModuleControlRequestFromModule::LiveRoots {} => {
2620 let registered = self
2621 .registry
2622 .get_module_by_connection(connection_id)
2623 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2624 let Some(registration) = registered else {
2625 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2626 };
2627 let response = self
2628 .forwarding
2629 .live_roots(®istration.manifest.module_id)
2630 .map_err(RouterError::Forwarding)?;
2631 Ok(vec![control_response_body_frame(
2632 &frame,
2633 &response,
2634 "ModuleControlResponseToModule::LiveRoots",
2635 )?])
2636 }
2637 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2638 self.handle_scope_sync(connection_id, frame, generation, scopes)
2639 }
2640 ModuleControlRequestFromModule::ScopeApply {
2641 generation,
2642 upsert,
2643 end,
2644 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
2645 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2646 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2647 }
2648 }
2649 }
2650
2651 fn handle_operator_confirm(
2652 &self,
2653 ctx: &RouteCtx,
2654 frame: Frame,
2655 ) -> Result<Vec<Frame>, RouterError> {
2656 use crate::operator_confirm::{audit, Outcome};
2657 let registration = self
2658 .registry
2659 .get_module_by_connection(ctx.connection_id)
2660 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2661 let Some(registration) = registration else {
2662 let outcome = Outcome::refusal("not_registered");
2663 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2664 return Ok(vec![outcome.frame(&frame)]);
2665 };
2666 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
2667 Ok(request) => request,
2668 Err(_) => {
2669 let outcome = Outcome::refusal("invalid_control_body");
2670 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2671 return Ok(vec![outcome.frame(&frame)]);
2672 }
2673 };
2674 let module_id = registration.manifest.module_id;
2675 let nonce = self
2679 .hello_launch_nonces
2680 .lock()
2681 .unwrap_or_else(|p| p.into_inner())
2682 .nonce(ctx.connection_id)
2683 .map(str::to_owned);
2684 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
2685 self.supervisor
2686 .spawned_consumer_authorized(&module_id, nonce)
2687 });
2688 let confirms = self.forwarding.operator_confirms();
2689 self.forwarding
2690 .with_operator_route(
2691 ctx.connection_id,
2692 request.route_channel,
2693 request.route_epoch,
2694 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
2695 )
2696 .map_err(RouterError::Forwarding)
2697 }
2698
2699 fn handle_scope_sync(
2703 &self,
2704 connection_id: ConnectionId,
2705 frame: Frame,
2706 generation: u64,
2707 scopes: Vec<ScopeRecord>,
2708 ) -> Result<Vec<Frame>, RouterError> {
2709 self.handle_scope_change(connection_id, frame, generation, scopes, None)
2710 }
2711
2712 fn handle_scope_change(
2713 &self,
2714 connection_id: ConnectionId,
2715 frame: Frame,
2716 generation: u64,
2717 scopes: Vec<ScopeRecord>,
2718 end: Option<Vec<ScopeEnd>>,
2719 ) -> Result<Vec<Frame>, RouterError> {
2720 let is_apply = end.is_some();
2721 let Some(registration) = self
2722 .registry
2723 .get_module_by_connection(connection_id)
2724 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2725 else {
2726 return Ok(vec![control_error_frame(
2727 &frame,
2728 "not_registered",
2729 "scope.sync and scope.apply require an active module registration owned by this connection",
2730 )?]);
2731 };
2732 let owner = registration.manifest.module_id;
2733 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2734 let is_current_launch = |connection: ConnectionId| {
2735 self.hello_launch_nonces
2736 .lock()
2737 .unwrap_or_else(|poisoned| poisoned.into_inner())
2738 .presented(connection, current_nonce.as_deref())
2739 };
2740 let mut table = self
2745 .scopes
2746 .write()
2747 .unwrap_or_else(|poisoned| poisoned.into_inner());
2748 let outcome = match end {
2749 Some(end) => table.apply(
2750 &owner,
2751 connection_id,
2752 is_current_launch,
2753 generation,
2754 scopes,
2755 end,
2756 ),
2757 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
2758 };
2759 let drained = match &outcome {
2760 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
2761 Err(_) => Vec::new(),
2762 };
2763 let has_deadlines = outcome.is_ok() && table.has_deadlines();
2764 drop(table);
2765 if has_deadlines {
2766 self.scope_deadline_added.notify_one();
2767 }
2768 match outcome {
2769 Ok(applied) => {
2770 let counts = ScopeOutcomeCounts::of(&applied.results);
2771 info!(
2772 owner = %owner,
2773 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
2774 generation,
2775 records = applied.results.len(),
2776 created = counts.created,
2777 replaced = counts.replaced,
2778 updated = counts.updated,
2779 unchanged = counts.unchanged,
2780 refused = counts.refused,
2781 ended = applied.ended.len(),
2782 tag_changes = applied.tag_changes.len(),
2783 routes_closed = drained.len(),
2784 "scope change accepted"
2785 );
2786 for refused in applied
2792 .results
2793 .iter()
2794 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
2795 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
2796 {
2797 warn!(
2798 owner = %owner,
2799 generation,
2800 scope_ref = %refused.scope_ref,
2801 scope_epoch = refused.scope_epoch,
2802 code = refused.code.as_deref().unwrap_or(""),
2803 "scope record refused"
2804 );
2805 }
2806 self.close_scope_drained_routes(drained);
2807 let response = if is_apply {
2808 ModuleControlResponseToModule::ScopeApply {
2809 generation,
2810 results: applied.results,
2811 end_results: applied.end_results,
2812 ended: applied.ended,
2813 }
2814 } else {
2815 ModuleControlResponseToModule::ScopeSync {
2816 generation,
2817 results: applied.results,
2818 ended: applied.ended,
2819 }
2820 };
2821 Ok(vec![control_response_body_frame(
2822 &frame,
2823 &response,
2824 "scope change response",
2825 )?])
2826 }
2827 Err(refusal) => {
2828 info!(
2829 owner = %owner,
2830 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
2831 generation,
2832 code = refusal.code,
2833 "scope change refused"
2834 );
2835 Ok(vec![control_error_frame(
2836 &frame,
2837 refusal.code,
2838 refusal.message,
2839 )?])
2840 }
2841 }
2842 }
2843
2844 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2850 if drained.is_empty() {
2851 return;
2852 }
2853 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2854 BTreeMap::new();
2855 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2856 for route in drained {
2857 warn!(
2858 module_id = %route.module_id,
2859 reason = ?route.reason,
2860 client_connection_id = route.client.connection_id.get(),
2861 route_channel = route.client.channel,
2862 "closing route because its scope changed"
2863 );
2864 pushes
2865 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2866 .or_insert_with(|| (route.reason, Vec::new()))
2867 .1
2868 .push(EndpointRoute {
2869 goodbye_target: route.client.clone(),
2870 principal: Principal::Unverified,
2871 bound_at: Instant::now(),
2872 draining: false,
2873 drain_reason: None,
2874 });
2875 goodbyes.push(route.module);
2876 goodbyes.push(route.client);
2877 }
2878 for ((module_id, _), (reason, routes)) in pushes {
2879 send_route_control_pushes(
2880 &self.forwarding,
2881 routes,
2882 ClientControlPush::RouteClosed {
2883 module_id,
2884 channels: Vec::new(),
2885 reason,
2886 drained: false,
2887 abandoned: 0,
2888 excluded_subscriptions: 0,
2889 terminal: Some(false),
2890 },
2891 );
2892 }
2893 self.emit_route_goodbyes(goodbyes);
2894 }
2895
2896 fn handle_scope_describe(
2899 &self,
2900 connection_id: ConnectionId,
2901 frame: Frame,
2902 owner: Principal,
2903 scope_ref: String,
2904 ) -> Result<Vec<Frame>, RouterError> {
2905 let registered = self
2906 .registry
2907 .get_module_by_connection(connection_id)
2908 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2909 if registered.is_none() {
2910 return Ok(vec![control_error_frame(
2911 &frame,
2912 "not_registered",
2913 "scope.describe requires an active module registration owned by this connection",
2914 )?]);
2915 }
2916 let description = self
2917 .scopes
2918 .read()
2919 .unwrap_or_else(|poisoned| poisoned.into_inner())
2920 .describe(&owner, &scope_ref);
2921 let owner_configured = match &owner {
2922 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
2927 _ => false,
2928 };
2929 let response = ModuleControlResponseToModule::ScopeDescribe {
2930 status: description.status,
2931 scope_epoch: description.scope_epoch,
2932 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2933 owner_synced: description.owner_synced,
2934 owner_configured,
2935 scope: description.stamp,
2936 };
2937 Ok(vec![control_response_body_frame(
2938 &frame,
2939 &response,
2940 "ModuleControlResponseToModule::ScopeDescribe",
2941 )?])
2942 }
2943
2944 fn handle_catalog_update(
2945 &self,
2946 connection_id: ConnectionId,
2947 frame: Frame,
2948 provides: Vec<ProviderRole>,
2949 capabilities: Option<CapabilityDeclarations>,
2950 ready: Option<bool>,
2951 ) -> Result<Vec<Frame>, RouterError> {
2952 self.refresh_capability_requirements();
2953 let Some(registration) = self
2954 .registry
2955 .get_module_by_connection(connection_id)
2956 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2957 else {
2958 return Ok(vec![control_error_frame(
2959 &frame,
2960 "not_registered",
2961 "catalog.update requires an active module registration owned by this connection",
2962 )?]);
2963 };
2964
2965 if let Some(message) =
2966 catalog_update_frozen_field_message(®istration.manifest, &provides)
2967 {
2968 return Ok(vec![control_error_frame(
2969 &frame,
2970 "catalog_update_frozen_field",
2971 message,
2972 )?]);
2973 }
2974
2975 let mut candidate = registration.manifest.clone();
2976 candidate.provides = provides.clone();
2977 candidate.capabilities = capabilities
2978 .clone()
2979 .or_else(|| registration.manifest.capabilities.clone());
2980 if let Err(err) = candidate.validate_capability_grammar() {
2981 return Ok(vec![control_error_frame(
2982 &frame,
2983 "invalid_capability_grammar",
2984 err.to_string(),
2985 )?]);
2986 }
2987
2988 let mut conflicts = self
2991 .capability_evaluator
2992 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
2993 if let Some(conflict) = conflicts.first() {
2994 let message = format!(
2995 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2996 conflict.capability, conflict.claimants[0], candidate.module_id
2997 );
2998 for conflict in &mut conflicts {
2999 conflict.source = DuplicateClaimSource::CatalogUpdate;
3000 }
3001 log_duplicate_claim_events(conflicts);
3002 return Ok(vec![control_error_frame(
3003 &frame,
3004 "reserved_capability",
3005 message,
3006 )?]);
3007 }
3008
3009 let updated = self
3010 .registry
3011 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3012 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3013 if updated.is_none() {
3014 return Ok(vec![control_error_frame(
3015 &frame,
3016 "not_registered",
3017 "catalog.update requires an active module registration owned by this connection",
3018 )?]);
3019 }
3020 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3021 log_duplicate_claim_events(
3022 self.capability_evaluator
3023 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3024 );
3025 }
3026 if capability_census_trigger(
3027 registration.manifest.capabilities.as_ref(),
3028 updated
3029 .as_ref()
3030 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3031 ) {
3032 self.enforce_capability_denies();
3033 }
3034 self.refresh_capability_requirements();
3035
3036 let response = ModuleControlResponseToModule::CatalogUpdate {};
3037 control_response_body_frame(
3038 &frame,
3039 &response,
3040 "ModuleControlResponseToModule::CatalogUpdate",
3041 )
3042 .map(|frame| vec![frame])
3043 }
3044
3045 async fn handle_activity_record(
3046 &self,
3047 ctx: &RouteCtx,
3048 frame: Frame,
3049 ) -> Result<Vec<Frame>, RouterError> {
3050 use subc_control::{
3051 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3052 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3053 };
3054 let result = async {
3055 let module = self
3056 .forwarding
3057 .module_id_for_connection(ctx.connection_id)
3058 .ok()
3059 .flatten()
3060 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3061 let request: ActivityRecordRequest =
3062 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3063 crate::activity::validate(&module, &request)?;
3064 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3065 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3066 let at_ms = std::time::SystemTime::now()
3067 .duration_since(std::time::UNIX_EPOCH)
3068 .unwrap_or_default()
3069 .as_millis() as u64;
3070 ledger.record(module, request, actor, at_ms).await
3071 }
3072 .await;
3073 match result {
3074 Ok(entry) => Ok(vec![control_response_body_frame(
3075 &frame,
3076 &ActivityRecordResponse { seq: entry.seq },
3077 "activity.record",
3078 )?]),
3079 Err(code) => Ok(vec![control_error_frame(
3080 &frame,
3081 code,
3082 "activity recording refused",
3083 )?]),
3084 }
3085 }
3086
3087 fn activity_actor(
3088 &self,
3089 connection: ConnectionId,
3090 module: &str,
3091 on: Option<&subc_control::ActivityOn>,
3092 ) -> Result<subc_control::ActivityActor, &'static str> {
3093 use crate::forwarding::{DataRoute, DataRouteState};
3094 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3095 let principal = |id: &str| Principal::Reserved {
3096 module_id: id.to_owned(),
3097 };
3098 let mut actor = ActivityActor {
3099 principal: principal(module),
3100 scope_owner: None,
3101 agent_id: None,
3102 run_id: None,
3103 flow_id: None,
3104 project_root: None,
3105 };
3106 let scope = match on {
3107 None => return Ok(actor),
3108 Some(ActivityOn::Scope(scope)) => {
3109 let selector = subc_protocol::scope::ScopeSelector {
3110 owner: principal(module),
3111 scope_ref: scope.scope_ref.clone(),
3112 scope_epoch: Some(scope.epoch),
3113 };
3114 self.scopes
3115 .read()
3116 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3117 .admit(&principal(module), module, &selector, true)
3118 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3119 }
3120 Some(ActivityOn::Route(route)) => {
3121 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3122 .forwarding
3123 .lookup_data_route(connection, route.channel, route.epoch)
3124 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3125 else {
3126 return Err(ACTIVITY_ACTOR_UNKNOWN);
3127 };
3128 if binding.module_id != module {
3129 return Err(ACTIVITY_ACTOR_UNKNOWN);
3130 }
3131 actor.principal = binding.principal.clone();
3132 actor.project_root = binding
3133 .project_root
3134 .as_ref()
3135 .map(|root| root.as_path().to_path_buf());
3136 let Some(bound) = &binding.scope else {
3137 return Ok(actor);
3138 };
3139 let owner = principal(&bound.owner);
3140 let selector = subc_protocol::scope::ScopeSelector {
3141 owner: owner.clone(),
3142 scope_ref: bound.scope_ref.clone(),
3143 scope_epoch: Some(bound.tag.scope_epoch),
3144 };
3145 let admitted = self
3146 .scopes
3147 .read()
3148 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3149 .admit(&owner, module, &selector, true)
3150 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3151 if admitted.tag != bound.tag {
3152 return Err(ACTIVITY_ACTOR_UNKNOWN);
3153 }
3154 admitted
3155 }
3156 };
3157 actor.scope_owner = Some(scope.stamp.owner);
3158 actor.agent_id = scope.stamp.attributes.agent_id;
3159 actor.run_id = scope.stamp.attributes.run_id;
3160 actor.flow_id = scope.stamp.attributes.flow_id;
3161 Ok(actor)
3162 }
3163
3164 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3165 self.refresh_capability_requirements();
3166 let mut counters = self.counters.snapshot();
3172 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3173 obj.insert(
3174 "activity_unparseable_lines".into(),
3175 ledger.read_errors().into(),
3176 );
3177 }
3178 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3179 self.forwarding.client_route_concentration(),
3180 counters.as_object_mut(),
3181 ) {
3182 obj.insert(
3183 "client_connections_with_routes".into(),
3184 connections_with_routes.into(),
3185 );
3186 obj.insert("max_routes_on_one_connection".into(), max.into());
3187 }
3188 if let (Some(open_breakers), Some(obj)) = (
3194 self.route_bind_breakers.open_snapshot(),
3195 counters.as_object_mut(),
3196 ) {
3197 obj.insert("route_bind_breakers_open".into(), open_breakers);
3198 }
3199 let response = ClientControlResponse::ServerDescribe {
3200 protocol_ver: PROTOCOL_VERSION,
3201 subc_ops: subc_ops(),
3202 capabilities: self.subc_capabilities.as_ref().to_vec(),
3203 connected_clients: self.connected_clients.count(),
3204 counters: Some(counters),
3205 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3206 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3207 capability_requirements: self.capability_requirement_statuses(),
3208 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3209 };
3210 Ok(vec![control_response_body_frame(
3211 &frame,
3212 &response,
3213 "ClientControlResponse::ServerDescribe",
3214 )?])
3215 }
3216
3217 fn handle_catalog_list(
3218 &self,
3219 frame: Frame,
3220 module_id: Option<String>,
3221 ) -> Result<Vec<Frame>, RouterError> {
3222 let (generation, modules) = self.registry.list_modules().map_err(|err| {
3223 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3224 })?;
3225 let entries = modules
3226 .into_iter()
3227 .filter(|registration| {
3228 module_id
3229 .as_deref()
3230 .map(|wanted| registration.manifest.module_id == wanted)
3231 .unwrap_or(true)
3232 })
3233 .map(|registration| {
3234 let not_ready = self.not_ready_reason(®istration);
3235 let roles = registration.manifest.provides;
3236 CatalogEntry::new(
3237 registration.manifest.module_id,
3238 roles,
3239 registration.control_ops,
3240 )
3241 .with_ready(not_ready.is_none())
3242 .with_not_ready(not_ready)
3243 .with_module_version(Some(registration.manifest.module_version))
3244 .with_capabilities(registration.manifest.capabilities)
3245 .with_self_signals(registration.manifest.self_signals)
3246 .with_events(registration.manifest.events)
3247 })
3248 .collect();
3249 let response = ClientControlResponse::CatalogList {
3250 generation,
3251 modules: entries,
3252 subc_ops: subc_ops(),
3253 };
3254 Ok(vec![control_response_body_frame(
3255 &frame,
3256 &response,
3257 "ClientControlResponse::CatalogList",
3258 )?])
3259 }
3260
3261 fn route_open_principal(
3262 &self,
3263 frame: &Frame,
3264 consumer_identity: Option<ConsumerIdentity>,
3265 ) -> Result<Result<Principal, Frame>, RouterError> {
3266 let Some(consumer_identity) = consumer_identity else {
3267 return Ok(Ok(Principal::Direct));
3268 };
3269
3270 if self.supervisor.spawned_consumer_authorized(
3271 &consumer_identity.module_id,
3272 &consumer_identity.launch_nonce,
3273 ) {
3274 return Ok(Ok(Principal::Reserved {
3275 module_id: consumer_identity.module_id,
3276 }));
3277 }
3278
3279 Ok(Err(control_error_frame(
3280 frame,
3281 "bad_consumer_identity",
3282 format!(
3283 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3284 consumer_identity.module_id
3285 ),
3286 )?))
3287 }
3288
3289 fn route_open_refusal_frame(
3296 &self,
3297 ctx: &RouteCtx,
3298 frame: &Frame,
3299 module_id: &str,
3300 reason: &'static str,
3301 code: &'static str,
3302 message: impl Into<String>,
3303 ) -> Result<Frame, RouterError> {
3304 self.observe_route_open_refusal(ctx, module_id, reason, code);
3305 control_error_frame(frame, code, message.into())
3306 }
3307
3308 fn route_open_breaker_refusal_frame(
3326 &self,
3327 ctx: &RouteCtx,
3328 frame: &Frame,
3329 module_id: &str,
3330 consecutive_timeouts: u32,
3331 retry_in: Duration,
3332 probe_in_flight: bool,
3333 ) -> Result<Frame, RouterError> {
3334 self.counters
3335 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
3336 debug!(
3337 target: "control",
3338 code = "module_timeout",
3339 module_id = ?module_id,
3340 connection_id = ctx.connection_id.get(),
3341 consecutive_timeouts,
3342 retry_in_ms = retry_in.as_millis() as u64,
3343 probe_in_flight,
3344 "route.open refused by open bind-relay breaker"
3345 );
3346 let detail = if probe_in_flight {
3353 "one test route is already being tried; retry once it settles".to_string()
3354 } else {
3355 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
3356 };
3357 control_error_frame(
3358 frame,
3359 "module_timeout",
3360 format!(
3361 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
3362 timed out in a row); {detail}; its established routes are unaffected"
3363 ),
3364 )
3365 }
3366
3367 fn observe_route_open_refusal(
3380 &self,
3381 ctx: &RouteCtx,
3382 module_id: &str,
3383 reason: &'static str,
3384 code: &'static str,
3385 ) {
3386 self.counters.increment_route_open_refused(code);
3387 info!(
3388 target: "control",
3389 code,
3390 reason,
3391 module_id = ?module_id,
3392 connection_id = ctx.connection_id.get(),
3393 "route.open refused"
3394 );
3395 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
3396 self.route_outages.record_not_serving(module_id, reason);
3397 }
3398 }
3399
3400 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
3445 self.route_outages.record_accepted(module_id);
3446 self.counters.increment_route_open_accepted(principal);
3447 info!(
3448 target: "control",
3449 principal,
3450 module_id,
3451 connection_id = ctx.connection_id.get(),
3452 "route.open accepted"
3453 );
3454 }
3455
3456 fn supervised_absent_route_open_refusal_frame(
3457 &self,
3458 ctx: &RouteCtx,
3459 frame: &Frame,
3460 module_id: &str,
3461 code: &'static str,
3462 status: &crate::supervise::ModuleStatus,
3463 ) -> Result<Frame, RouterError> {
3464 self.counters.increment_route_open_refused(code);
3465 info!(
3466 target: "control",
3467 code,
3468 reason = "supervised_not_registered",
3469 module_id = ?module_id,
3470 connection_id = ctx.connection_id.get(),
3471 state = %status.state,
3472 enabled = status.enabled,
3473 live = status.live,
3474 "route.open refused"
3475 );
3476 self.route_outages
3480 .record_not_serving(module_id, "supervised_not_registered");
3481 let message = if let Some(pid) = status.waiting_for_previous_pid {
3482 format!("waiting: previous instance (pid {pid}) is still running but hidden; it cannot be identified")
3483 } else {
3484 format!(
3485 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
3486 status.state, status.enabled, status.live
3487 )
3488 };
3489 control_error_frame(frame, code, message)
3490 }
3491
3492 async fn handle_route_open(
3493 &self,
3494 ctx: &RouteCtx,
3495 frame: Frame,
3496 request: RouteOpenRequest,
3497 ) -> Result<Vec<Frame>, RouterError> {
3498 let RouteOpenRequest {
3499 target,
3500 mut identity,
3501 consumer_identity,
3502 consumer_capabilities,
3503 role_versions,
3504 admission_facts,
3505 scope,
3506 } = request;
3507 let target_module_id = target_module_id(&target).to_string();
3508 if self
3509 .registry
3510 .get_module_by_connection(ctx.connection_id)
3511 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3512 .is_some()
3513 {
3514 return Ok(vec![control_error_frame(
3515 &frame,
3516 "invalid_request",
3517 "module connections cannot open client routes",
3518 )?]);
3519 }
3520 debug!(
3521 connection_id = ctx.connection_id.get(),
3522 corr = frame.header.corr,
3523 module_id = %target_module_id,
3524 "handling route.open"
3525 );
3526
3527 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
3533 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
3534 self.observe_route_open_refusal(
3535 ctx,
3536 &target_module_id,
3537 "invalid_role_versions",
3538 error_codes::INVALID_REQUEST,
3539 );
3540 return Ok(vec![control_error_body_frame(
3541 &frame,
3542 ErrorBody {
3543 code: error_codes::INVALID_REQUEST.to_string(),
3544 message: error.to_string(),
3545 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
3546 },
3547 )?]);
3548 }
3549
3550 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
3551 if status.waiting_for_previous_pid.is_some() {
3552 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3553 ctx,
3554 &frame,
3555 &target_module_id,
3556 "module_warming",
3557 &status,
3558 )?]);
3559 }
3560 }
3561
3562 let Some(registration) = self
3581 .registry
3582 .get_module(&target_module_id)
3583 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3584 else {
3585 if let Some((status, warming)) =
3586 self.supervisor_status(&target_module_id, frame.header.corr)?
3587 {
3588 if status.protocol == ModuleProtocol::None {
3595 return Ok(vec![self.route_open_refusal_frame(
3596 ctx,
3597 &frame,
3598 &target_module_id,
3599 "protocol_none",
3600 error_codes::MODULE_NO_PROTOCOL,
3601 format!(
3602 "module_id '{target_module_id}' is declared protocol: none; \
3603 it speaks no subc wire and serves no routes"
3604 ),
3605 )?]);
3606 }
3607 let code = if warming {
3608 "module_warming"
3609 } else {
3610 "target_unavailable"
3611 };
3612 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3613 ctx,
3614 &frame,
3615 &target_module_id,
3616 code,
3617 &status,
3618 )?]);
3619 }
3620 if let Some(removed_ago_ms) =
3621 self.supervisor.removal_tombstone_age_ms(&target_module_id)
3622 {
3623 return Ok(vec![self.route_open_refusal_frame(
3624 ctx,
3625 &frame,
3626 &target_module_id,
3627 "removed",
3628 error_codes::MODULE_REMOVED,
3629 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
3630 )?]);
3631 }
3632 return Ok(vec![self.route_open_refusal_frame(
3633 ctx,
3634 &frame,
3635 &target_module_id,
3636 "not_registered",
3637 error_codes::UNKNOWN_MODULE,
3638 format!("module_id '{target_module_id}' is not registered"),
3639 )?]);
3640 };
3641
3642 if !registration.ready {
3646 self.counters
3647 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
3648 info!(
3649 target: "control",
3650 code = error_codes::MODULE_WARMING,
3651 module_id = ?target_module_id,
3652 connection_id = ctx.connection_id.get(),
3653 reason = "declared_not_ready",
3654 "route.open refused"
3655 );
3656 self.route_outages
3659 .record_not_serving(&target_module_id, "declared_not_ready");
3660 return Ok(vec![control_error_body_frame(
3661 &frame,
3662 ErrorBody {
3663 code: error_codes::MODULE_WARMING.to_string(),
3664 message: format!(
3665 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
3666 ),
3667 detail: Some(serde_json::json!({
3668 "reason": "declared_not_ready"
3669 })),
3670 },
3671 )?]);
3672 }
3673
3674 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
3693 self.counters
3694 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3695 info!(
3696 target: "control",
3697 code = error_codes::MODULE_WARMING,
3698 module_id = ?target_module_id,
3699 connection_id = ctx.connection_id.get(),
3700 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3701 capability = %capability,
3702 "route.open refused"
3703 );
3704 return Ok(vec![control_error_body_frame(
3705 &frame,
3706 ErrorBody {
3707 code: error_codes::MODULE_WARMING.to_string(),
3708 message: format!(
3709 "module_id '{target_module_id}' requires capability '{capability}', \
3710 which no registered module provides; retry"
3711 ),
3712 detail: Some(serde_json::json!({
3713 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3714 "capability": capability,
3715 })),
3716 },
3717 )?]);
3718 }
3719
3720 if !target_has_required_role(&target, ®istration.manifest.provides) {
3721 return Ok(vec![self.route_open_refusal_frame(
3722 ctx,
3723 &frame,
3724 &target_module_id,
3725 "role_not_provided",
3726 "target_unavailable",
3727 format!("module_id '{target_module_id}' does not provide the requested target"),
3728 )?]);
3729 }
3730
3731 if registration.state != ChannelState::Active {
3732 return Ok(vec![self.route_open_refusal_frame(
3733 ctx,
3734 &frame,
3735 &target_module_id,
3736 "registration_not_active",
3737 "target_unavailable",
3738 format!("module_id '{target_module_id}' is not active"),
3739 )?]);
3740 }
3741
3742 if self
3743 .forwarding
3744 .module_is_draining(&target_module_id)
3745 .map_err(RouterError::Forwarding)?
3746 {
3747 return Ok(vec![self.route_open_refusal_frame(
3748 ctx,
3749 &frame,
3750 &target_module_id,
3751 "reloading",
3752 "module_reloading",
3753 format!("module_id '{target_module_id}' is reloading"),
3754 )?]);
3755 }
3756
3757 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
3758 process_liveness.process_live(&target_module_id) == Some(false)
3759 }) {
3760 if process_liveness.process_replacing(&target_module_id) {
3767 return Ok(vec![self.route_open_refusal_frame(
3768 ctx,
3769 &frame,
3770 &target_module_id,
3771 "reloading",
3772 "module_reloading",
3773 format!("module_id '{target_module_id}' is reloading"),
3774 )?]);
3775 }
3776 return Ok(vec![self.route_open_refusal_frame(
3777 ctx,
3778 &frame,
3779 &target_module_id,
3780 "supervisor_not_live",
3781 "target_unavailable",
3782 format!("module_id '{target_module_id}' is not live"),
3783 )?]);
3784 }
3785
3786 if !self
3787 .forwarding
3788 .has_live_module_connection(&target_module_id)
3789 .map_err(RouterError::Forwarding)?
3790 {
3791 return Ok(vec![self.route_open_refusal_frame(
3792 ctx,
3793 &frame,
3794 &target_module_id,
3795 "no_forwarding_connection",
3796 "target_unavailable",
3797 format!("module_id '{target_module_id}' has no live forwarding connection"),
3798 )?]);
3799 }
3800
3801 if let Some(error) =
3802 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
3803 {
3804 self.observe_route_open_refusal(
3805 ctx,
3806 &target_module_id,
3807 "op_not_allowed",
3808 "op_not_allowed",
3809 );
3810 return Ok(vec![error]);
3811 }
3812
3813 let principal = match self.route_open_principal(&frame, consumer_identity)? {
3814 Ok(principal) => principal,
3815 Err(error) => {
3816 self.observe_route_open_refusal(
3817 ctx,
3818 &target_module_id,
3819 "bad_consumer_identity",
3820 "bad_consumer_identity",
3821 );
3822 return Ok(vec![error]);
3823 }
3824 };
3825
3826 if let Principal::Reserved {
3830 module_id: opening_module_id,
3831 } = &principal
3832 {
3833 if let Some(opening_registration) = self
3834 .registry
3835 .get_module(opening_module_id)
3836 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3837 {
3838 if let Some(capability) =
3839 denied_capability(&opening_registration.manifest, ®istration.manifest)
3840 {
3841 warn!(
3842 opening_module_id,
3843 target_module_id,
3844 capability,
3845 "refusing route.open because an attested capability deny edge matches"
3846 );
3847 return Ok(vec![self.route_open_refusal_frame(
3848 ctx,
3849 &frame,
3850 &target_module_id,
3851 "capability_deny_edge",
3852 "capability_forbidden",
3853 format!(
3854 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
3855 ),
3856 )?]);
3857 }
3858 }
3859 }
3860
3861 if admission_facts.is_some() {
3862 let carrier_matches = matches!(
3863 &principal,
3864 Principal::Reserved { module_id }
3865 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
3866 );
3867 if !carrier_matches {
3868 return Ok(vec![self.route_open_refusal_frame(
3869 ctx,
3870 &frame,
3871 &target_module_id,
3872 "admission_facts_carrier_not_permitted",
3873 "admission_facts_not_permitted",
3874 "admission facts may only be carried by the configured reserved module",
3875 )?]);
3876 }
3877
3878 let target_allowed = self
3879 .admission_facts_targets
3880 .as_ref()
3881 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
3882 if !target_allowed {
3883 return Ok(vec![self.route_open_refusal_frame(
3884 ctx,
3885 &frame,
3886 &target_module_id,
3887 "admission_facts_target_not_listed",
3888 "admission_facts_target_not_allowed",
3889 format!(
3890 "admission facts are not permitted for target module_id '{target_module_id}'"
3891 ),
3892 )?]);
3893 }
3894
3895 }
3899
3900 let (bound_scope, scope_stamp) = match scope {
3906 None => (None, None),
3907 Some(selector) => {
3908 let owner_configured = match &selector.owner {
3909 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3915 _ => false,
3916 };
3917 let admitted = self
3918 .scopes
3919 .read()
3920 .unwrap_or_else(|poisoned| poisoned.into_inner())
3921 .admit(&principal, &target_module_id, &selector, owner_configured)
3922 .and_then(|admission| {
3923 crate::scopes::check_target_flow_support(
3924 &admission.stamp,
3925 &target_module_id,
3926 registration.manifest.capabilities.as_ref(),
3927 )?;
3928 crate::scopes::check_target_agent_run_support(
3929 &admission.stamp,
3930 &target_module_id,
3931 registration.manifest.capabilities.as_ref(),
3932 )?;
3933 Ok(admission)
3934 });
3935 match admitted {
3936 Ok(admission) => (
3937 Some(BoundScope {
3938 owner: admission.owner,
3939 scope_ref: admission.stamp.scope_ref.clone(),
3940 tag: admission.tag,
3941 }),
3942 Some(admission.stamp),
3943 ),
3944 Err(refusal) => {
3945 return Ok(vec![self.route_open_refusal_frame(
3946 ctx,
3947 &frame,
3948 &target_module_id,
3949 refusal.code,
3950 refusal.code,
3951 refusal.message,
3952 )?]);
3953 }
3954 }
3955 }
3956 };
3957
3958 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
3980 Ok(project_root) => project_root,
3981 Err(err) => {
3982 return Ok(vec![control_error_frame(
3983 &frame,
3984 "invalid_project_root",
3985 err.to_string(),
3986 )?])
3987 }
3988 };
3989 identity.project_root = project_root.as_path().to_path_buf();
3990
3991 let _concurrency_guard = match self
4002 .route_bind_concurrency
4003 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4004 {
4005 Ok(guard) => guard,
4006 Err(in_flight) => {
4007 return Ok(vec![self.route_open_target_capacity_refusal(
4008 ctx,
4009 &frame,
4010 &target_module_id,
4011 in_flight,
4012 )?]);
4013 }
4014 };
4015
4016 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4019 RouteBindAdmission::Admitted { guard, probe } => {
4020 if probe {
4021 info!(
4022 module_id = %target_module_id,
4023 connection_id = ctx.connection_id.get(),
4024 "route.bind breaker half-open: admitting one probe"
4025 );
4026 }
4027 guard
4028 }
4029 RouteBindAdmission::Refused {
4030 consecutive_timeouts,
4031 retry_in,
4032 probe_in_flight,
4033 } => {
4034 return Ok(vec![self.route_open_breaker_refusal_frame(
4035 ctx,
4036 &frame,
4037 &target_module_id,
4038 consecutive_timeouts,
4039 retry_in,
4040 probe_in_flight,
4041 )?]);
4042 }
4043 };
4044
4045 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4050 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4051 let pending = match self
4052 .forwarding
4053 .begin_route_bind_relay_for(
4054 ctx.connection_id,
4055 ctx.egress.clone(),
4056 response_version(&frame),
4057 frame.header.corr,
4058 &target_module_id,
4059 principal.clone(),
4060 bound_scope,
4061 Some(project_root),
4062 relay_deadline,
4063 )
4064 .await
4065 {
4066 Ok(pending) => pending,
4067 Err(err) => {
4068 return Ok(vec![self.route_open_refusal_frame(
4069 ctx,
4070 &frame,
4071 &target_module_id,
4072 "relay_reservation_failed",
4073 forwarding_error_code(&err),
4074 err.to_string(),
4075 )?])
4076 }
4077 };
4078 let crate::forwarding::PendingRouteBindRelay {
4079 endpoint,
4080 module_sink,
4081 negotiated_ver,
4082 client_channel,
4083 client_epoch,
4084 module_channel,
4085 module_epoch,
4086 corr: relay_corr,
4087 receiver,
4088 } = pending;
4089 let mut reservation =
4090 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4091
4092 if let Some(stamp) = scope_stamp
4097 .as_ref()
4098 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4099 {
4100 let relay_registration = self
4101 .registry
4102 .get_module_by_connection(endpoint.connection_id)
4103 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4104 if let Err(refusal) = crate::scopes::check_target_flow_support(
4105 stamp,
4106 &target_module_id,
4107 relay_registration
4108 .as_ref()
4109 .and_then(|registration| registration.manifest.capabilities.as_ref()),
4110 )
4111 .and_then(|()| {
4112 crate::scopes::check_target_agent_run_support(
4113 stamp,
4114 &target_module_id,
4115 relay_registration
4116 .as_ref()
4117 .and_then(|registration| registration.manifest.capabilities.as_ref()),
4118 )
4119 }) {
4120 reservation.release_and_disarm();
4121 return Ok(vec![self.route_open_refusal_frame(
4122 ctx,
4123 &frame,
4124 &target_module_id,
4125 refusal.code,
4126 refusal.code,
4127 refusal.message,
4128 )?]);
4129 }
4130 }
4131
4132 debug!(
4133 connection_id = ctx.connection_id.get(),
4134 client_channel,
4135 client_epoch,
4136 module_channel,
4137 module_epoch,
4138 "reserved route handle pair"
4139 );
4140 let principal_label = principal_label(&principal);
4143 let relay = ModuleControlRequest::RouteBind {
4144 route_channel: module_channel,
4145 epoch: module_epoch,
4146 target,
4147 identity,
4148 principal: Some(principal),
4149 consumer_capabilities,
4150 role_versions,
4151 admission_facts,
4152 scope: scope_stamp,
4153 };
4154 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4155 RouterError::backend(
4156 0,
4157 frame.header.corr,
4158 format!("failed to encode route.bind request: {err}"),
4159 )
4160 })?;
4161 let relay_frame = Frame::build_with_version(
4162 negotiated_ver,
4163 FrameType::Request,
4164 control_flags(),
4165 0,
4166 0,
4167 relay_corr,
4168 relay_body,
4169 )
4170 .map_err(RouterError::FrameBuild)?;
4171
4172 if let Err(err) = module_sink.send(relay_frame).await {
4173 reservation.release_and_disarm();
4174 return Ok(vec![self.route_open_refusal_frame(
4175 ctx,
4176 &frame,
4177 &target_module_id,
4178 "relay_send_failed",
4179 "target_unavailable",
4180 err.to_string(),
4181 )?]);
4182 }
4183
4184 #[cfg(test)]
4185 if let Some(hold) = &self.route_bind_relay_sent_hold {
4186 hold.wait().await;
4190 hold.wait().await;
4191 }
4192
4193 if self
4198 .forwarding
4199 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4200 .map_err(RouterError::Forwarding)?
4201 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4202 {
4203 self.send_abandoned_route_bind_goodbye(
4204 &module_sink,
4205 negotiated_ver,
4206 module_channel,
4207 module_epoch,
4208 );
4209 }
4210
4211 match timeout_at(relay_deadline, receiver).await {
4212 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4213 reservation.disarm();
4214 if breaker.record_accepted() {
4215 info!(
4216 module_id = %target_module_id,
4217 "route.bind breaker closed: the probe was accepted"
4218 );
4219 }
4220 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
4221 Ok(Vec::new())
4222 }
4223 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4224 reservation.release_and_disarm();
4225 breaker.record_inconclusive();
4229 let scope_code = match body.code.as_str() {
4233 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4234 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4235 _ => None,
4236 };
4237 if let Some(code) = scope_code {
4238 self.observe_route_open_refusal(
4239 ctx,
4240 &target_module_id,
4241 "scope_changed_before_commit",
4242 code,
4243 );
4244 return Ok(vec![control_error_body_frame(&frame, body)?]);
4245 }
4246 self.counters
4247 .increment_route_open_refused("module_rejected");
4248 info!(
4249 target: "control",
4250 code = "module_rejected",
4251 module_code = ?body.code,
4252 module_id = ?target_module_id,
4253 connection_id = ctx.connection_id.get(),
4254 "route.open refused"
4255 );
4256 Ok(vec![control_error_body_frame(&frame, body)?])
4257 }
4258 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
4259 reservation.release_and_disarm();
4260 breaker.record_inconclusive();
4261 tracing::warn!(
4268 module_id = %target_module_id,
4269 "route.bind relay abandoned: {message}"
4270 );
4271 Ok(vec![self.route_open_refusal_frame(
4272 ctx,
4273 &frame,
4274 &target_module_id,
4275 "relay_abandoned",
4276 "target_unavailable",
4277 message,
4278 )?])
4279 }
4280 Ok(Err(_)) => {
4281 reservation.release_and_disarm();
4282 breaker.record_inconclusive();
4283 Ok(vec![self.route_open_refusal_frame(
4284 ctx,
4285 &frame,
4286 &target_module_id,
4287 "relay_waiter_canceled",
4288 "target_unavailable",
4289 "route.bind relay waiter was canceled before the module responded",
4290 )?])
4291 }
4292 Err(_) => {
4293 reservation.release_and_disarm();
4294 if let Some(opened) = breaker.record_timeout(
4298 self.route_bind_breaker_threshold,
4299 self.route_bind_breaker_cooldown,
4300 ) {
4301 warn!(
4302 module_id = %target_module_id,
4303 consecutive_timeouts = opened.consecutive_timeouts,
4304 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
4305 reopened_after_probe = opened.reopened_after_probe,
4306 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
4307 );
4308 }
4309 tracing::warn!(
4317 module_id = %target_module_id,
4318 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
4319 "route.bind relay timed out: module did not ack within budget"
4320 );
4321 Ok(vec![self.route_open_refusal_frame(
4322 ctx,
4323 &frame,
4324 &target_module_id,
4325 "relay_timed_out",
4326 "module_timeout",
4327 format!(
4328 "module_id '{target_module_id}' did not answer route.bind within {:?}",
4329 route_bind_relay_timeout
4330 ),
4331 )?])
4332 }
4333 }
4334 }
4335
4336 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4337 let response = ClientControlResponse::SupervisorSpawnSnapshot {
4338 snapshot: self.supervisor.spawn_snapshot(),
4339 };
4340 Ok(vec![control_response_body_frame(
4341 &frame,
4342 &response,
4343 "ClientControlResponse::SupervisorSpawnSnapshot",
4344 )?])
4345 }
4346
4347 fn handle_supervisor_spawn_subscribe(
4348 &self,
4349 ctx: &RouteCtx,
4350 frame: Frame,
4351 since: Option<SpawnCursor>,
4352 ) -> Result<Vec<Frame>, RouterError> {
4353 match self.supervisor.subscribe_spawns(
4354 ctx.connection_id,
4355 frame.header.corr,
4356 response_version(&frame),
4357 since,
4358 ctx.egress.clone(),
4359 ) {
4360 Ok(()) => Ok(Vec::new()),
4361 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
4362 Ok(vec![control_error_body_frame(
4363 &frame,
4364 ErrorBody {
4365 code: "spawn_cursor_incarnation_mismatch".to_string(),
4366 message: "spawn cursor belongs to a different daemon incarnation"
4367 .to_string(),
4368 detail: Some(serde_json::json!({
4369 "current_daemon_incarnation": current
4370 })),
4371 },
4372 )?])
4373 }
4374 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
4375 &frame,
4376 ErrorBody {
4377 code: "spawn_cursor_too_old".to_string(),
4378 message: "spawn cursor predates the retained event ring".to_string(),
4379 detail: Some(serde_json::json!({
4380 "oldest_retained_cursor": oldest
4381 })),
4382 },
4383 )?]),
4384 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
4385 0,
4386 frame.header.corr,
4387 format!("failed to open supervisor spawn subscription: {error}"),
4388 )),
4389 }
4390 }
4391
4392 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4393 let generation = self
4394 .registry
4395 .generation()
4396 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4397 let mut modules = Vec::new();
4398 for module in self.supervisor.list() {
4399 let status = module.status_for_control("list").map_err(|err| {
4400 RouterError::backend(
4401 0,
4402 frame.header.corr,
4403 format!("failed to read supervisor status: {err}"),
4404 )
4405 })?;
4406 let (configured, _) = module.configuration().map_err(|err| {
4407 RouterError::backend(
4408 0,
4409 frame.header.corr,
4410 format!("failed to read module configuration: {err}"),
4411 )
4412 })?;
4413 let image = module.running_image_agreement().await;
4415 let resources = Some(module.child_resource_usage());
4418 let pending_reload = Some(reload_verdict(
4419 &configured.program,
4420 status.spawned_from.as_deref(),
4421 image,
4422 ));
4423 modules.push(
4426 SupervisorEntry::new(
4427 status.module_id,
4428 status.state.to_string(),
4429 status.enabled,
4430 status.live,
4431 status.health.status,
4432 )
4433 .with_launch_nonce_env(Some(!cfg!(unix)))
4434 .with_launch_nonce_source(status.launch_nonce_source)
4435 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
4436 .with_protocol(status.protocol)
4437 .with_pending_reload(pending_reload)
4438 .with_last_probe_ms(status.health.last_probe_ms)
4439 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
4440 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
4441 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
4442 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
4443 .with_restart_count(Some(status.restart_count))
4444 .with_max_restarts(Some(status.max_restarts))
4445 .with_lifetime_restarts(Some(status.lifetime_restarts))
4446 .with_spawn_generation(Some(status.spawn_generation))
4447 .with_restart_window_secs(Some(status.restart_window.as_secs()))
4448 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
4449 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
4450 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
4451 .with_resources(resources),
4452 );
4453 }
4454 let response = ClientControlResponse::SupervisorList {
4455 generation,
4456 modules,
4457 };
4458 Ok(vec![control_response_body_frame(
4459 &frame,
4460 &response,
4461 "ClientControlResponse::SupervisorList",
4462 )?])
4463 }
4464
4465 fn handle_supervisor_stderr_tail(
4466 &self,
4467 frame: Frame,
4468 module_id: String,
4469 max_lines: Option<u32>,
4470 max_bytes: Option<u32>,
4471 ) -> Result<Vec<Frame>, RouterError> {
4472 let Some(module) = self.supervisor.get(&module_id) else {
4473 return Ok(vec![control_error_frame(
4474 &frame,
4475 "unknown_module",
4476 format!("module_id '{module_id}' is not supervised"),
4477 )?]);
4478 };
4479
4480 let snapshot = module.stderr_tail(
4481 max_lines.map(|value| value as usize),
4482 max_bytes.map(|value| value as usize),
4483 );
4484
4485 let response = ClientControlResponse::SupervisorStderrTail {
4486 module_id,
4487 tail: StderrTail {
4488 capture: match snapshot.capture {
4489 CaptureState::Captured => StderrCaptureState::Captured,
4490 CaptureState::Incomplete { reason } => {
4491 StderrCaptureState::Incomplete { reason }
4492 }
4493 CaptureState::NotCaptured { reason } => {
4494 StderrCaptureState::NotCaptured { reason }
4495 }
4496 },
4497 entries: snapshot
4498 .entries
4499 .into_iter()
4500 .map(|entry| match entry {
4501 TailEntry::Line {
4502 text,
4503 truncated,
4504 at_ms,
4505 } => StderrTailEntry::Line {
4506 text,
4507 truncated,
4508 at_ms,
4509 },
4510 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
4511 })
4512 .collect(),
4513 dropped_lines: snapshot.dropped_lines,
4514 },
4515 };
4516 Ok(vec![control_response_body_frame(
4517 &frame,
4518 &response,
4519 "ClientControlResponse::SupervisorStderrTail",
4520 )?])
4521 }
4522
4523 async fn handle_supervisor_terminals(
4524 &self,
4525 frame: Frame,
4526 module_id: String,
4527 ) -> Result<Vec<Frame>, RouterError> {
4528 let Some(module) = self.supervisor.get(&module_id) else {
4529 return Ok(vec![control_error_frame(
4530 &frame,
4531 "unknown_module",
4532 format!("module_id '{module_id}' is not supervised"),
4533 )?]);
4534 };
4535
4536 let terminals = module
4539 .read_durable_terminal_history()
4540 .await
4541 .map_err(|error| {
4542 RouterError::backend(
4543 0,
4544 frame.header.corr,
4545 format!("failed to read terminal history: {error}"),
4546 )
4547 })?;
4548 let response = ClientControlResponse::SupervisorTerminals {
4549 module_id,
4550 terminals,
4551 };
4552 Ok(vec![control_response_body_frame(
4553 &frame,
4554 &response,
4555 "ClientControlResponse::SupervisorTerminals",
4556 )?])
4557 }
4558
4559 fn handle_supervisor_routes(
4560 &self,
4561 frame: Frame,
4562 module_id: Option<String>,
4563 ) -> Result<Vec<Frame>, RouterError> {
4564 let modules = self
4565 .forwarding
4566 .route_census(module_id.as_deref())
4567 .map_err(RouterError::Forwarding)?
4568 .into_iter()
4569 .map(|(module_id, routes)| SupervisorRouteModule {
4570 module_id,
4571 routes: routes
4572 .into_iter()
4573 .map(|route| SupervisorRoute {
4574 consumer: match route.principal {
4575 Principal::Reserved { module_id } => {
4576 SupervisorRouteConsumer::Reserved { module_id }
4577 }
4578 Principal::Direct | Principal::Unverified => {
4579 SupervisorRouteConsumer::Direct {
4580 connection_id: route.goodbye_target.connection_id.get(),
4581 }
4582 }
4583 },
4584 age_ms: Instant::now()
4585 .saturating_duration_since(route.bound_at)
4586 .as_millis()
4587 .try_into()
4588 .unwrap_or(u64::MAX),
4589 draining: route.draining,
4590 drain_reason: route.drain_reason,
4591 })
4592 .collect(),
4593 })
4594 .collect();
4595 let response = ClientControlResponse::SupervisorRoutes { modules };
4596 Ok(vec![control_response_body_frame(
4597 &frame,
4598 &response,
4599 "ClientControlResponse::SupervisorRoutes",
4600 )?])
4601 }
4602
4603 async fn handle_supervisor_provenance(
4604 &self,
4605 frame: Frame,
4606 module_id: Option<String>,
4607 ) -> Result<Vec<Frame>, RouterError> {
4608 let mut selected = if let Some(module_id) = module_id {
4609 let Some(module) = self.supervisor.get(&module_id) else {
4610 return Ok(vec![control_error_frame(
4611 &frame,
4612 "unknown_module",
4613 format!("module_id '{module_id}' is not supervised"),
4614 )?]);
4615 };
4616 vec![module]
4617 } else {
4618 self.supervisor.list()
4619 };
4620
4621 let mut modules = Vec::with_capacity(selected.len());
4622 for module in selected.drain(..) {
4623 let (status, observed_image) = module
4624 .status_and_running_image_agreement()
4625 .await
4626 .map_err(|err| {
4627 RouterError::backend(
4628 0,
4629 frame.header.corr,
4630 format!("failed to read supervisor status: {err}"),
4631 )
4632 })?;
4633 let module_declared = self
4634 .registry
4635 .get_module(&status.module_id)
4636 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4637 .and_then(|registration| registration.manifest.provenance)
4638 .map(|build| ModuleDeclaredProvenance::Reported { build })
4639 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
4640 #[cfg(test)]
4641 let running_image = match &self.provenance_probe_override {
4642 Some(result) => result.clone(),
4643 None => observed_image,
4644 };
4645 #[cfg(not(test))]
4646 let running_image = observed_image;
4647 modules.push(SupervisorModuleProvenance {
4648 module_id: status.module_id,
4649 module_declared,
4650 daemon_observed: SupervisorObservedProcess {
4651 launch_nonce_source: status.launch_nonce_source,
4652 pid: status.pid,
4653 spawned_at_ms: status.spawned_at_ms,
4654 spawned_from: status.spawned_from,
4655 running_image,
4656 },
4657 });
4658 }
4659 let daemon = SupervisorDaemonProvenance {
4660 daemon_build: self.daemon_provenance.build.clone(),
4661 daemon_observed: DaemonObservedProcess {
4662 pid: self.daemon_provenance.pid,
4663 started_at_ms: self
4664 .daemon_provenance
4665 .start_clock
4666 .map(|clock| clock.started_at_ms())
4667 .or(self.daemon_provenance.started_at_ms),
4668 running_image: self
4669 .daemon_provenance
4670 .probe
4671 .observe(
4672 self.daemon_provenance.pid,
4673 self.daemon_provenance.executable_path.as_deref(),
4674 self.daemon_provenance.executable_identity.clone(),
4675 self.daemon_provenance.process_start_time,
4676 )
4677 .await,
4678 },
4679 };
4680 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
4681 Ok(vec![control_response_body_frame(
4682 &frame,
4683 &response,
4684 "ClientControlResponse::SupervisorProvenance",
4685 )?])
4686 }
4687
4688 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4689 self.refresh_capability_requirements();
4690 let generation = self
4691 .registry
4692 .generation()
4693 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4694 let modules = self
4695 .supervisor
4696 .list()
4697 .into_iter()
4698 .map(|module| {
4699 let status = module.status_for_control("health").map_err(|err| {
4700 RouterError::backend(
4701 0,
4702 frame.header.corr,
4703 format!("failed to read supervisor health: {err}"),
4704 )
4705 })?;
4706 let module_id = status.module_id;
4707 let capability_detail = self
4708 .capability_evaluator
4709 .required_problem_detail(&module_id);
4710 Ok(SupervisorHealthEntry {
4711 module_id,
4712 status: status.health.status,
4713 detail: append_capability_problem_detail(
4714 status.health.detail,
4715 capability_detail,
4716 ),
4717 metrics: status.health.metrics,
4718 consecutive_failures: status.health.consecutive_failures,
4719 late_answer_count: status.health.late_answer_count,
4720 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
4721 last_action: status.health.last_action,
4722 last_action_ms: status.health.last_action_ms,
4723 last_probe_ms: status.health.last_probe_ms,
4724 })
4725 })
4726 .collect::<Result<Vec<_>, RouterError>>()?;
4727 let response = ClientControlResponse::SupervisorHealth {
4728 generation,
4729 modules,
4730 };
4731 Ok(vec![control_response_body_frame(
4732 &frame,
4733 &response,
4734 "ClientControlResponse::SupervisorHealth",
4735 )?])
4736 }
4737
4738 async fn handle_supervisor_restart(
4739 &self,
4740 frame: Frame,
4741 module_id: String,
4742 drain_timeout_ms: Option<u64>,
4743 ) -> Result<Vec<Frame>, RouterError> {
4744 let operation_lock = self.supervisor.operation_lock();
4745 let _operation_guard = operation_lock.lock().await;
4746 let Some(module) = self.supervisor.get(&module_id) else {
4747 return Ok(vec![control_error_frame(
4748 &frame,
4749 "unknown_module",
4750 format!("module_id '{module_id}' is not supervised"),
4751 )?]);
4752 };
4753
4754 self.route_outages.mark_operator_action(&module_id);
4755 if let Err(err) = module.restart(drain_timeout_ms).await {
4756 self.route_outages
4757 .operator_action_ended_unrefused(&module_id);
4758 let (code, message) = match err {
4759 crate::supervise::SuperviseError::Disabled { .. } => {
4760 ("module_disabled", err.to_string())
4761 }
4762 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4763 ("swap_in_progress", err.to_string())
4764 }
4765 _ => (
4766 "target_unavailable",
4767 format!("failed to restart module_id '{module_id}': {err}"),
4768 ),
4769 };
4770 return Ok(vec![control_error_frame(&frame, code, message)?]);
4771 }
4772
4773 let response = ClientControlResponse::SupervisorAck {
4774 module_id,
4775 applied: true,
4776 };
4777 Ok(vec![control_response_body_frame(
4778 &frame,
4779 &response,
4780 "ClientControlResponse::SupervisorAck",
4781 )?])
4782 }
4783
4784 async fn handle_supervisor_swap(
4788 &self,
4789 frame: Frame,
4790 module_id: String,
4791 ready_timeout_ms: Option<u64>,
4792 ) -> Result<Vec<Frame>, RouterError> {
4793 let module = {
4800 let operation_lock = self.supervisor.operation_lock();
4801 let _operation_guard = operation_lock.lock().await;
4802 self.supervisor.get(&module_id)
4803 };
4804 let Some(module) = module else {
4805 return Ok(vec![control_error_frame(
4806 &frame,
4807 "unknown_module",
4808 format!("module_id '{module_id}' is not supervised"),
4809 )?]);
4810 };
4811
4812 self.route_outages.mark_operator_action(&module_id);
4813 if let Err(err) = module
4814 .swap(ready_timeout_ms.map(Duration::from_millis))
4815 .await
4816 {
4817 self.route_outages
4818 .operator_action_ended_unrefused(&module_id);
4819 use crate::supervise::SuperviseError;
4820 let message = err.to_string();
4821 let error = match err {
4822 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
4823 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
4824 code: "swap_refused".to_string(),
4825 message,
4826 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
4827 },
4828 SuperviseError::SwapFailed {
4829 arm,
4830 candidate_exit,
4831 ..
4832 } => ErrorBody {
4833 code: "swap_failed".to_string(),
4834 message,
4835 detail: Some(serde_json::json!({
4836 "arm": arm.as_str(),
4837 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
4838 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
4839 })),
4840 },
4841 _ => ErrorBody::new(
4842 "target_unavailable",
4843 format!("failed to swap module_id '{module_id}': {message}"),
4844 ),
4845 };
4846 return Ok(vec![control_error_body_frame(&frame, error)?]);
4847 }
4848 self.route_outages
4852 .operator_action_ended_unrefused(&module_id);
4853
4854 let response = ClientControlResponse::SupervisorAck {
4855 module_id,
4856 applied: true,
4857 };
4858 Ok(vec![control_response_body_frame(
4859 &frame,
4860 &response,
4861 "ClientControlResponse::SupervisorAck",
4862 )?])
4863 }
4864
4865 async fn handle_supervisor_reload(
4866 &self,
4867 frame: Frame,
4868 module_id: String,
4869 ) -> Result<Vec<Frame>, RouterError> {
4870 let operation_lock = self.supervisor.operation_lock();
4871 let _operation_guard = operation_lock.lock().await;
4872 let Some(module) = self.supervisor.get(&module_id) else {
4873 return Ok(vec![control_error_frame(
4874 &frame,
4875 "unknown_module",
4876 format!("module_id '{module_id}' is not supervised"),
4877 )?]);
4878 };
4879
4880 self.route_outages.mark_operator_action(&module_id);
4881 if let Err(err) = module.reload().await {
4882 self.route_outages
4883 .operator_action_ended_unrefused(&module_id);
4884 let (code, message) = match err {
4885 crate::supervise::SuperviseError::Disabled { .. } => {
4886 ("module_disabled", err.to_string())
4887 }
4888 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4889 ("swap_in_progress", err.to_string())
4890 }
4891 _ => (
4892 "reload_failed",
4893 format!("failed to reload module_id '{module_id}': {err}"),
4894 ),
4895 };
4896 return Ok(vec![control_error_frame(&frame, code, message)?]);
4897 }
4898
4899 let response = ClientControlResponse::SupervisorAck {
4900 module_id,
4901 applied: true,
4902 };
4903 Ok(vec![control_response_body_frame(
4904 &frame,
4905 &response,
4906 "ClientControlResponse::SupervisorAck",
4907 )?])
4908 }
4909
4910 async fn handle_supervisor_rescan(
4911 &self,
4912 frame: Frame,
4913 preview: bool,
4914 ) -> Result<Vec<Frame>, RouterError> {
4915 let Some(context) = self.rescan.clone() else {
4916 return Ok(vec![control_error_frame(
4917 &frame,
4918 "rescan_unavailable",
4919 "the daemon was not started with a reloadable config path".to_string(),
4920 )?]);
4921 };
4922
4923 let operation_lock = self.supervisor.operation_lock();
4924 let _operation_guard = operation_lock.lock().await;
4925 let loaded = match crate::daemon_config::load(&context.config_path) {
4926 Ok(config) => config,
4927 Err(err) => {
4928 return Ok(vec![control_error_frame(
4929 &frame,
4930 "invalid_daemon_config",
4931 format!("supervisor rescan rejected daemon config: {err}"),
4932 )?])
4933 }
4934 };
4935 let Some(config) = loaded else {
4943 return Ok(vec![control_error_frame(
4944 &frame,
4945 "invalid_daemon_config",
4946 format!(
4947 "daemon config not found at {}; refusing to rescan (an absent config would \
4948 retire every supervised module)",
4949 context.config_path.display()
4950 ),
4951 )?]);
4952 };
4953 let (
4954 configured_port,
4955 storage_config,
4956 admission_facts_carrier_module_id,
4957 admission_facts_targets,
4958 scope_authority_owners,
4959 modules,
4960 reserved_capabilities,
4961 ) = (
4962 config.port,
4963 config.storage,
4964 config.admission_facts_carrier_module_id,
4965 config.admission_facts_targets,
4966 config.scope_authority_owners,
4967 config.modules,
4968 config.reserved_capabilities,
4969 );
4970
4971 let mut restart_required = Vec::new();
4980 for section in RestartRequiredSection::ALL {
4981 let changed = match section {
4982 RestartRequiredSection::Port => configured_port != context.configured_port,
4983 RestartRequiredSection::Storage => storage_config != context.storage_config,
4984 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
4985 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
4986 }
4987 RestartRequiredSection::AdmissionFactsTargets => {
4988 admission_facts_targets != context.admission_facts_targets
4989 }
4990 RestartRequiredSection::ScopeAuthorityOwners => {
4991 scope_authority_owners != context.scope_authority_owners
4992 }
4993 };
4994 if changed {
4995 restart_required.push(section.label().to_string());
4996 }
4997 }
4998 if !restart_required.is_empty() {
4999 warn!(
5000 config_path = %context.config_path.display(),
5001 sections = %restart_required.join(", "),
5002 "daemon config changed outside the modules section; restart the daemon to apply those changes"
5003 );
5004 }
5005
5006 for configured in &modules {
5007 if let Err(err) = validate_spec(&configured.module_spec()) {
5008 return Ok(vec![control_error_frame(
5009 &frame,
5010 "invalid_daemon_config",
5011 format!("supervisor rescan rejected daemon config: {err}"),
5012 )?]);
5013 }
5014 }
5015
5016 let configured_capabilities = modules
5017 .iter()
5018 .map(|module| (module.module_id.clone(), module.enabled))
5019 .collect::<Vec<_>>();
5020 let preview_capability_warnings = if preview {
5021 let (_, registrations) = self.runtime_capability_snapshot()?;
5022 let current_modules = self
5023 .supervisor
5024 .list()
5025 .into_iter()
5026 .map(|module| module.module_id().to_string())
5027 .collect::<BTreeSet<_>>();
5028 let resulting_modules = configured_capabilities.clone();
5029 let removed = current_modules
5030 .into_iter()
5031 .filter(|module_id| {
5032 !resulting_modules
5033 .iter()
5034 .any(|(configured_id, _)| configured_id == module_id)
5035 })
5036 .collect::<Vec<_>>();
5037 self.capability_evaluator.preview_removal_warnings(
5038 resulting_modules,
5039 &removed,
5040 ®istrations,
5041 )
5042 } else {
5043 Vec::new()
5044 };
5045 let result = match self
5046 .reconcile_supervised_modules(&context.supervisor, modules, preview)
5047 .await
5048 {
5049 Ok(result) => result,
5050 Err(message) => {
5051 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5052 }
5053 };
5054 if !preview {
5055 self.capability_evaluator
5056 .configure(configured_capabilities, reserved_capabilities);
5057 self.capability_evaluator.wake_deadline_loop();
5058 self.refresh_capability_requirements();
5059 }
5060 let mut result = result;
5061 result.restart_required = restart_required;
5062 result.capability_warnings = preview_capability_warnings;
5063 let response = ClientControlResponse::SupervisorRescan { result };
5064 Ok(vec![control_response_body_frame(
5065 &frame,
5066 &response,
5067 "ClientControlResponse::SupervisorRescan",
5068 )?])
5069 }
5070
5071 async fn handle_supervisor_release_reserved(
5072 &self,
5073 frame: Frame,
5074 module_id: String,
5075 ) -> Result<Vec<Frame>, RouterError> {
5076 let Some(context) = self.rescan.clone() else {
5077 return Ok(vec![control_error_frame(
5078 &frame,
5079 "release_unavailable",
5080 "reserved-id release requires a daemon started with a reloadable config path",
5081 )?]);
5082 };
5083 let operation_lock = self.supervisor.operation_lock();
5084 let _operation_guard = operation_lock.lock().await;
5085 let loaded = match crate::daemon_config::load(&context.config_path) {
5086 Ok(Some(config)) => config,
5087 Ok(None) => {
5088 return Ok(vec![control_error_frame(
5089 &frame,
5090 "invalid_daemon_config",
5091 format!(
5092 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5093 context.config_path.display()
5094 ),
5095 )?])
5096 }
5097 Err(err) => {
5098 return Ok(vec![control_error_frame(
5099 &frame,
5100 "invalid_daemon_config",
5101 format!("unable to verify reserved-id release against daemon config: {err}"),
5102 )?])
5103 }
5104 };
5105 if loaded
5106 .modules
5107 .iter()
5108 .any(|configured| configured.module_id == module_id)
5109 {
5110 return Ok(vec![control_error_frame(
5111 &frame,
5112 "reserved_module_configured",
5113 format!(
5114 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5115 ),
5116 )?]);
5117 }
5118 if !self.supervisor.release_retained_reserved_gate(&module_id) {
5119 return Ok(vec![control_error_frame(
5120 &frame,
5121 "reserved_gate_not_retained",
5122 format!(
5123 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
5124 ),
5125 )?]);
5126 }
5127
5128 let response = ClientControlResponse::SupervisorAck {
5129 module_id,
5130 applied: true,
5131 };
5132 Ok(vec![control_response_body_frame(
5133 &frame,
5134 &response,
5135 "ClientControlResponse::SupervisorAck",
5136 )?])
5137 }
5138
5139 async fn reconcile_supervised_modules(
5148 &self,
5149 supervisor: &Supervisor,
5150 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
5151 preview: bool,
5152 ) -> Result<SupervisorRescanResult, String> {
5153 let mut current = BTreeMap::new();
5154 for module in self.supervisor.list() {
5155 let (spec, health) = module.configuration().map_err(|err| {
5156 format!(
5157 "failed to read configuration for module_id '{}': {err}",
5158 module.module_id()
5159 )
5160 })?;
5161 let enabled = module
5162 .status()
5163 .map_err(|err| {
5164 format!(
5165 "failed to read status for module_id '{}': {err}",
5166 module.module_id()
5167 )
5168 })?
5169 .enabled;
5170 current.insert(
5171 module.module_id().to_string(),
5172 (module, spec, health, enabled),
5173 );
5174 }
5175 let configured = configured_modules
5176 .into_iter()
5177 .map(|module| (module.module_id.clone(), module))
5178 .collect::<BTreeMap<_, _>>();
5179
5180 let added = configured
5181 .keys()
5182 .filter(|module_id| !current.contains_key(*module_id))
5183 .cloned()
5184 .collect::<Vec<_>>();
5185 let removed = current
5186 .keys()
5187 .filter(|module_id| !configured.contains_key(*module_id))
5188 .cloned()
5189 .collect::<Vec<_>>();
5190 let mut changed_pending_reload = Vec::new();
5191 let mut configuration_changes = BTreeSet::new();
5192 let mut enabled_changes = BTreeSet::new();
5193 let mut unchanged = 0_u32;
5194
5195 for (module_id, configured_module) in &configured {
5196 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5197 else {
5198 continue;
5199 };
5200 let launch_changed = *current_spec != configured_module.module_spec();
5204 let configuration_changed =
5205 launch_changed || *current_health != configured_module.health;
5206 let enabled_changed = *current_enabled != configured_module.enabled;
5207 if configuration_changed {
5208 configuration_changes.insert(module_id.clone());
5209 }
5210 if launch_changed {
5211 changed_pending_reload.push(module_id.clone());
5212 }
5213 if enabled_changed {
5214 enabled_changes.insert(module_id.clone());
5215 }
5216 if !configuration_changed && !enabled_changed {
5217 unchanged = unchanged.saturating_add(1);
5218 }
5219 }
5220
5221 if preview {
5227 return Ok(SupervisorRescanResult {
5228 added,
5229 removed,
5230 changed_pending_reload,
5231 enabled_changes: enabled_changes.iter().cloned().collect(),
5232 unchanged,
5233 preview: true,
5234 restart_required: Vec::new(),
5238 capability_warnings: Vec::new(),
5239 });
5240 }
5241
5242 for module_id in &removed {
5243 let module = ¤t
5244 .get(module_id)
5245 .expect("removed module came from current supervisor state")
5246 .0;
5247 module.retire().await.map_err(|err| {
5248 format!("failed to retire module_id '{module_id}' during rescan: {err}")
5249 })?;
5250 self.supervisor.record_rescan_removal(module_id);
5284 self.supervisor.retire(module_id);
5285 self.route_outages.forget(module_id);
5286 }
5287
5288 for module_id in configured.keys() {
5289 let Some((module, _, _, _)) = current.get(module_id) else {
5290 continue;
5291 };
5292 let configured_module = configured
5293 .get(module_id)
5294 .expect("configured module id came from configured map");
5295 if configuration_changes.contains(module_id) {
5296 module
5297 .update_configuration(
5298 configured_module.module_spec(),
5299 configured_module.health.clone(),
5300 configured_module.drain_timeout_ms,
5301 )
5302 .await
5303 .map_err(|err| {
5304 format!(
5305 "failed to update module_id '{module_id}' configuration during rescan: {err}"
5306 )
5307 })?;
5308 }
5309 if enabled_changes.contains(module_id) {
5310 self.route_outages.mark_operator_action(module_id);
5313 module
5314 .set_enabled(configured_module.enabled)
5315 .await
5316 .map_err(|err| {
5317 self.route_outages.operator_action_ended_unrefused(module_id);
5318 format!(
5319 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
5320 configured_module.enabled
5321 )
5322 })?;
5323 }
5324 }
5325
5326 for module_id in &added {
5327 let configured_module = configured
5328 .get(module_id)
5329 .expect("added module id came from configured map");
5330 supervisor
5331 .supervise_configured_with_health(
5332 configured_module.module_spec(),
5333 configured_module.enabled,
5334 configured_module.health.clone(),
5335 configured_module.drain_timeout_ms,
5336 configured_module.restart,
5337 )
5338 .map_err(|err| {
5339 format!("failed to add module_id '{module_id}' during rescan: {err}")
5340 })?;
5341 }
5342
5343 Ok(SupervisorRescanResult {
5344 added,
5345 removed,
5346 changed_pending_reload,
5347 enabled_changes: enabled_changes.iter().cloned().collect(),
5348 unchanged,
5349 preview: false,
5350 restart_required: Vec::new(),
5353 capability_warnings: Vec::new(),
5354 })
5355 }
5356
5357 async fn handle_supervisor_set_enabled(
5358 &self,
5359 frame: Frame,
5360 module_id: String,
5361 enabled: bool,
5362 ) -> Result<Vec<Frame>, RouterError> {
5363 let operation_lock = self.supervisor.operation_lock();
5364 let _operation_guard = operation_lock.lock().await;
5365 let Some(module) = self.supervisor.get(&module_id) else {
5366 return Ok(vec![control_error_frame(
5367 &frame,
5368 "unknown_module",
5369 format!("module_id '{module_id}' is not supervised"),
5370 )?]);
5371 };
5372
5373 self.route_outages.mark_operator_action(&module_id);
5376 let applied = match module.set_enabled(enabled).await {
5377 Ok(applied) => applied,
5378 Err(err) => {
5379 self.route_outages
5380 .operator_action_ended_unrefused(&module_id);
5381 return Ok(vec![control_error_frame(
5382 &frame,
5383 "target_unavailable",
5384 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
5385 )?]);
5386 }
5387 };
5388 if !applied {
5389 self.route_outages
5392 .operator_action_ended_unrefused(&module_id);
5393 }
5394
5395 self.capability_evaluator.wake_deadline_loop();
5396 self.refresh_capability_requirements();
5397 let response = ClientControlResponse::SupervisorAck { module_id, applied };
5398 Ok(vec![control_response_body_frame(
5399 &frame,
5400 &response,
5401 "ClientControlResponse::SupervisorAck",
5402 )?])
5403 }
5404
5405 async fn handle_supervisor_health_probe(
5406 &self,
5407 frame: Frame,
5408 module_id: String,
5409 ) -> Result<Vec<Frame>, RouterError> {
5410 self.refresh_capability_requirements();
5411 let Some(registration) = self
5412 .registry
5413 .get_module(&module_id)
5414 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5415 else {
5416 return Ok(vec![control_error_frame(
5417 &frame,
5418 "unknown_module",
5419 format!("module_id '{module_id}' is not registered"),
5420 )?]);
5421 };
5422
5423 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
5436 {
5437 return Ok(vec![control_error_frame(
5438 &frame,
5439 "health_not_advertised",
5440 format!("module_id '{module_id}' did not advertise health.check"),
5441 )?]);
5442 }
5443
5444 let deadline = Instant::now() + self.health_probe_timeout;
5445 let pending = match self.forwarding.begin_module_control_rpc_for(
5446 &module_id,
5447 MODULE_CONTROL_OP_HEALTH_CHECK,
5448 deadline,
5449 ) {
5450 Ok(pending) => pending,
5451 Err(err) => {
5452 return Ok(vec![control_error_frame(
5453 &frame,
5454 forwarding_error_code(&err),
5455 err.to_string(),
5456 )?])
5457 }
5458 };
5459
5460 let PendingModuleControlRpc {
5461 endpoint,
5462 module_sink,
5463 negotiated_ver,
5464 corr: probe_corr,
5465 receiver,
5466 } = pending;
5467 let mut guard =
5468 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
5469 let probe_body =
5470 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5471 RouterError::backend(
5472 0,
5473 frame.header.corr,
5474 format!("failed to encode health.check request: {err}"),
5475 )
5476 })?;
5477 let probe_frame = Frame::build_with_version(
5478 negotiated_ver,
5479 FrameType::Request,
5480 control_flags(),
5481 0,
5482 0,
5483 probe_corr,
5484 probe_body,
5485 )
5486 .map_err(RouterError::FrameBuild)?;
5487
5488 if let Err(err) = module_sink.send(probe_frame).await {
5489 return Ok(vec![control_error_frame(
5490 &frame,
5491 "target_unavailable",
5492 err.to_string(),
5493 )?]);
5494 }
5495
5496 match timeout_at(deadline, receiver).await {
5497 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5498 guard.disarm();
5499 let Some(report) = response.health_report() else {
5500 return Ok(vec![control_error_frame(
5501 &frame,
5502 "invalid_control_body",
5503 "health.check RPC returned a non-health response",
5504 )?]);
5505 };
5506 let HealthReport {
5511 status,
5512 detail,
5513 metrics,
5514 } = report;
5515 let capability_detail = self
5516 .capability_evaluator
5517 .required_problem_detail(&module_id);
5518 let response = ClientControlResponse::SupervisorHealthProbe {
5519 module_id,
5520 status,
5521 detail: append_capability_problem_detail(detail, capability_detail),
5522 metrics,
5523 };
5524 Ok(vec![control_response_body_frame(
5525 &frame,
5526 &response,
5527 "ClientControlResponse::SupervisorHealthProbe",
5528 )?])
5529 }
5530 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
5531 guard.disarm();
5532 Ok(vec![control_error_body_frame(&frame, body)?])
5533 }
5534 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5535 guard.disarm();
5536 Ok(vec![control_error_frame(
5537 &frame,
5538 "target_unavailable",
5539 message,
5540 )?])
5541 }
5542 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5543 guard.disarm();
5544 Ok(vec![control_error_frame(
5545 &frame,
5546 "invalid_control_body",
5547 message,
5548 )?])
5549 }
5550 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5551 guard.disarm();
5552 Ok(vec![control_error_frame(
5553 &frame,
5554 "invalid_control_body",
5555 format!("expected module-control op '{expected}', got '{actual}'"),
5556 )?])
5557 }
5558 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
5559 guard.disarm();
5560 Ok(vec![control_error_frame(
5561 &frame,
5562 "module_timeout",
5563 format!(
5564 "module_id '{module_id}' answered health.check after {:?}",
5565 self.health_probe_timeout
5566 ),
5567 )?])
5568 }
5569 Ok(Err(_)) => Ok(vec![control_error_frame(
5570 &frame,
5571 "target_unavailable",
5572 "health.check waiter was canceled before the module responded",
5573 )?]),
5574 Err(_) => Ok(vec![control_error_frame(
5575 &frame,
5576 "module_timeout",
5577 format!(
5578 "module_id '{module_id}' did not answer health.check within {:?}",
5579 self.health_probe_timeout
5580 ),
5581 )?]),
5582 }
5583 }
5584
5585 fn supervisor_status(
5586 &self,
5587 module_id: &str,
5588 corr: u64,
5589 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
5590 self.supervisor
5591 .get(module_id)
5592 .map(|module| {
5593 let warming = module.is_warming_for_control("status").map_err(|err| {
5594 RouterError::backend(
5595 0,
5596 corr,
5597 format!(
5598 "failed to read supervisor warming state for module_id '{module_id}': {err}"
5599 ),
5600 )
5601 })?;
5602 module.status_for_control("status").map_err(|err| {
5603 RouterError::backend(
5604 0,
5605 corr,
5606 format!(
5607 "failed to read supervisor status for module_id '{module_id}': {err}"
5608 ),
5609 )
5610 }).map(|status| (status, warming))
5611 })
5612 .transpose()
5613 }
5614
5615 fn guard_module_control_op(
5616 &self,
5617 frame: &Frame,
5618 module_id: &str,
5619 op: &str,
5620 ) -> Result<Option<Frame>, RouterError> {
5621 if self.module_grants_op(module_id, op, frame.header.corr)? {
5622 return Ok(None);
5623 }
5624
5625 Ok(Some(control_error_frame(
5626 frame,
5627 "op_not_allowed",
5628 format!("module_id '{module_id}' did not grant control op '{op}'"),
5629 )?))
5630 }
5631
5632 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
5633 let Some(registration) = self
5634 .registry
5635 .get_module(module_id)
5636 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
5637 else {
5638 return Ok(false);
5639 };
5640 Ok(module_registration_grants_op(®istration.control_ops, op))
5641 }
5642
5643 fn handle_status_update(
5644 &self,
5645 endpoint: ModuleEndpointId,
5646 frame: Frame,
5647 ) -> Result<Vec<Frame>, RouterError> {
5648 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
5649 Ok(update) => update,
5650 Err(err) => {
5651 if is_known_module_push_op(&frame.body) {
5656 return Ok(vec![control_error_frame(
5657 &frame,
5658 "invalid_control_body",
5659 format!("malformed module control push body: {err}"),
5660 )?]);
5661 }
5662 return Ok(Vec::new());
5663 }
5664 };
5665
5666 match update {
5667 ModuleControlPush::RouteStatus {
5668 route_channel,
5669 route_epoch,
5670 status,
5671 } => {
5672 self.forwarding
5673 .cache_status(endpoint, route_channel, route_epoch, status)
5674 .map_err(RouterError::Forwarding)?;
5675 }
5676 }
5677 Ok(Vec::new())
5678 }
5679
5680 fn handle_route_poll(
5681 &self,
5682 ctx: &RouteCtx,
5683 frame: Frame,
5684 route_channel: u16,
5685 route_epoch: u32,
5686 kind: PollKind,
5687 ) -> Result<Vec<Frame>, RouterError> {
5688 let snapshot = self
5689 .forwarding
5690 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
5691 .map_err(RouterError::Forwarding)?;
5692 let response = match (kind, snapshot) {
5693 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
5694 ClientControlResponse::RoutePoll {
5695 route_channel,
5696 route_epoch,
5697 status,
5698 live: None,
5699 }
5700 }
5701 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5702 route_channel,
5703 route_epoch,
5704 status: None,
5705 live: None,
5706 },
5707 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
5708 let live = self
5734 .process_liveness
5735 .as_ref()
5736 .and_then(|source| source.process_live(&module_id))
5737 .unwrap_or(true);
5738 ClientControlResponse::RoutePoll {
5739 route_channel,
5740 route_epoch,
5741 status: None,
5742 live: Some(live),
5743 }
5744 }
5745 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5746 route_channel,
5747 route_epoch,
5748 status: None,
5749 live: Some(false),
5750 },
5751 };
5752
5753 Ok(vec![control_response_body_frame(
5754 &frame,
5755 &response,
5756 "ClientControlResponse::RoutePoll",
5757 )?])
5758 }
5759
5760 pub(crate) fn observe_module_control_completion(
5761 &self,
5762 completion: ModuleControlRpcCompletion,
5763 ) -> bool {
5764 match completion {
5765 ModuleControlRpcCompletion::Unknown => false,
5766 ModuleControlRpcCompletion::Settled => true,
5767 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
5768 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
5769 info!(
5770 module_id = %module_id,
5771 latency_ms,
5772 "late health.check answer proves the module is alive"
5773 );
5774 match self
5775 .supervisor
5776 .record_late_health_answer(&module_id, latency_ms)
5777 {
5778 Ok(true) => {}
5779 Ok(false) => debug!(
5780 module_id = %module_id,
5781 latency_ms,
5782 "late health.check answer has no active supervisor snapshot"
5783 ),
5784 Err(err) => warn!(
5785 module_id = %module_id,
5786 latency_ms,
5787 error = %err,
5788 "failed to record late health.check answer"
5789 ),
5790 }
5791 true
5792 }
5793 }
5794 }
5795
5796 fn refuse_to_end_module_connection_for_a_client(
5817 &self,
5818 module_connection_id: ConnectionId,
5819 corr: u64,
5820 err: ForwardingError,
5821 ) -> Result<(), RouterError> {
5822 if let ForwardingError::ConnectionClosing { connection_id } = err {
5823 if connection_id != module_connection_id {
5824 warn!(
5825 module_connection_id = module_connection_id.get(),
5826 client_connection_id = connection_id.get(),
5827 corr,
5828 "dropping a route.bind response for a closing client; the module connection keeps serving"
5829 );
5830 return Ok(());
5831 }
5832 }
5833 Err(RouterError::Forwarding(err))
5834 }
5835
5836 fn handle_module_relay_response(
5837 &self,
5838 connection_id: ConnectionId,
5839 frame: Frame,
5840 ) -> Result<Vec<Frame>, RouterError> {
5841 let mut secondary_error = None;
5842 let outcome = match frame.header.ty {
5843 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
5844 Ok(probe) if probe.op == "route.bind" => {
5845 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
5846 Ok(ModuleControlResponse::RouteBindAck {}) => {
5847 RouteBindRelayOutcome::Accepted
5848 }
5849 Ok(other) => {
5850 let message =
5851 format!("route.bind response carried unexpected body: {other:?}");
5852 secondary_error = Some(control_error_frame(
5853 &frame,
5854 "invalid_control_body",
5855 message.clone(),
5856 )?);
5857 RouteBindRelayOutcome::ModuleGone(message)
5858 }
5859 Err(err) => {
5860 let message = format!("malformed route.bind response body: {err}");
5861 secondary_error = Some(control_error_frame(
5862 &frame,
5863 "invalid_control_body",
5864 message.clone(),
5865 )?);
5866 RouteBindRelayOutcome::ModuleGone(message)
5867 }
5868 }
5869 }
5870 Ok(probe) => {
5871 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
5872 {
5873 Ok(response) => ModuleControlRpcOutcome::Response(response),
5874 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5875 "malformed {} response body: {err}",
5876 probe.op
5877 )),
5878 };
5879 let completion = self
5880 .forwarding
5881 .complete_module_control_rpc(
5882 connection_id,
5883 frame.header.corr,
5884 Some(&probe.op),
5885 outcome,
5886 )
5887 .map_err(RouterError::Forwarding)?;
5888 if !self.observe_module_control_completion(completion) {
5889 debug!(
5890 connection_id = connection_id.get(),
5891 corr = frame.header.corr,
5892 op = %probe.op,
5893 "dropping late or unknown module-control RPC response"
5894 );
5895 }
5896 return Ok(Vec::new());
5897 }
5898 Err(err) => {
5899 if let Some(expected_op) = self
5900 .forwarding
5901 .pending_module_control_op(connection_id, frame.header.corr)
5902 .map_err(RouterError::Forwarding)?
5903 {
5904 let completion = self
5905 .forwarding
5906 .complete_module_control_rpc(
5907 connection_id,
5908 frame.header.corr,
5909 None,
5910 ModuleControlRpcOutcome::MalformedResponse(format!(
5911 "malformed {expected_op} response body: {err}"
5912 )),
5913 )
5914 .map_err(RouterError::Forwarding)?;
5915 if !self.observe_module_control_completion(completion) {
5916 debug!(
5917 connection_id = connection_id.get(),
5918 corr = frame.header.corr,
5919 "dropping late malformed module-control RPC response"
5920 );
5921 }
5922 return Ok(Vec::new());
5923 }
5924 let message = format!("malformed route.bind response body: {err}");
5925 secondary_error = Some(control_error_frame(
5926 &frame,
5927 "invalid_control_body",
5928 message.clone(),
5929 )?);
5930 RouteBindRelayOutcome::ModuleGone(message)
5931 }
5932 },
5933 FrameType::Error => {
5934 if self
5935 .forwarding
5936 .pending_module_control_op(connection_id, frame.header.corr)
5937 .map_err(RouterError::Forwarding)?
5938 .is_some()
5939 {
5940 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
5941 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
5942 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5943 "malformed module-control ERROR body: {err}"
5944 )),
5945 };
5946 let completion = self
5947 .forwarding
5948 .complete_module_control_rpc(
5949 connection_id,
5950 frame.header.corr,
5951 None,
5952 outcome,
5953 )
5954 .map_err(RouterError::Forwarding)?;
5955 if !self.observe_module_control_completion(completion) {
5956 debug!(
5957 connection_id = connection_id.get(),
5958 corr = frame.header.corr,
5959 "dropping late or unknown module-control RPC error"
5960 );
5961 }
5962 return Ok(Vec::new());
5963 }
5964 match serde_json::from_slice::<ErrorBody>(&frame.body) {
5965 Ok(body) => RouteBindRelayOutcome::Rejected(body),
5966 Err(err) => {
5967 let message = format!("malformed route.bind ERROR body: {err}");
5968 secondary_error = Some(control_error_frame(
5969 &frame,
5970 "invalid_control_body",
5971 message.clone(),
5972 )?);
5973 RouteBindRelayOutcome::ModuleGone(message)
5974 }
5975 }
5976 }
5977 ty => {
5978 return Ok(vec![control_error_frame(
5979 &frame,
5980 "unsupported_control_frame",
5981 format!("unsupported module channel-0 frame {ty:?}"),
5982 )?])
5983 }
5984 };
5985
5986 let settled =
5987 self.forwarding
5988 .complete_pending_relay(connection_id, frame.header.corr, outcome);
5989 let completion = match settled {
5990 Ok(completion) => completion,
5991 Err(err) => {
5992 self.refuse_to_end_module_connection_for_a_client(
5993 connection_id,
5994 frame.header.corr,
5995 err,
5996 )?;
5997 return Ok(secondary_error.into_iter().collect());
5998 }
5999 };
6000 if let Some(target) = completion.abandoned.as_ref() {
6001 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6002 }
6003 if !completion.settled {
6004 debug!(
6005 connection_id = connection_id.get(),
6006 corr = frame.header.corr,
6007 frame_type = ?frame.header.ty,
6008 "dropping late or unknown route.bind relay response"
6009 );
6010 }
6011 Ok(secondary_error.into_iter().collect())
6012 }
6013
6014 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6015 debug!(connection_id = connection_id.get(), "handling GOODBYE");
6016 self.cleanup_connection_with_end_reason(
6020 connection_id,
6021 RegistrationEndReason::ExplicitGoodbye,
6022 )
6023 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6024 Ok(Vec::new())
6025 }
6026}
6027
6028impl Default for ControlHandler {
6029 fn default() -> Self {
6030 Self::new(Arc::new(Registry::default()))
6031 }
6032}
6033
6034impl crate::supervise::SwapPromotionObserver for ControlHandler {
6035 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6036 self.apply_registration_capabilities(registration);
6037 }
6038}
6039
6040fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6041 CapabilityRequirementStatus {
6042 consumer: status.consumer,
6043 capability: status.capability,
6044 need: match status.need {
6045 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6046 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6047 },
6048 verdict: status.verdict.as_str().to_string(),
6049 episode_seq: status.episode_seq,
6050 config_satisfiable: status.config_satisfiable,
6051 runtime_available: status.runtime_available,
6052 detail: status.detail,
6053 }
6054}
6055
6056fn append_capability_problem_detail(
6057 detail: Option<String>,
6058 capability_detail: Option<String>,
6059) -> Option<String> {
6060 match (detail, capability_detail) {
6061 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
6062 (Some(detail), None) => Some(detail),
6063 (None, Some(capability_detail)) => Some(capability_detail),
6064 (None, None) => None,
6065 }
6066}
6067
6068fn subc_ops() -> Vec<String> {
6069 SUBC_CONTROL_OPS
6070 .iter()
6071 .map(|op| (*op).to_string())
6072 .collect()
6073}
6074
6075fn module_subc_ops() -> Vec<String> {
6076 SUBC_CONTROL_OPS
6077 .iter()
6078 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
6079 .map(|op| (*op).to_string())
6080 .collect()
6081}
6082
6083#[cfg(test)]
6084fn module_baseline_control_ops() -> Vec<String> {
6085 MODULE_BASELINE_CONTROL_OPS
6086 .iter()
6087 .map(|op| (*op).to_string())
6088 .collect()
6089}
6090
6091fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
6092 let mut seen = HashSet::new();
6093 let mut effective = Vec::new();
6094 for op in MODULE_BASELINE_CONTROL_OPS {
6095 if seen.insert((*op).to_string()) {
6096 effective.push((*op).to_string());
6097 }
6098 }
6099 for op in declared.unwrap_or_default() {
6100 if seen.insert(op.clone()) {
6101 effective.push(op);
6102 }
6103 }
6104 effective
6105}
6106
6107fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
6108 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
6109}
6110
6111fn target_module_id(target: &RouteTarget) -> &str {
6112 match target {
6113 RouteTarget::ToolProvider { module_id }
6114 | RouteTarget::ManagementSurface { module_id }
6115 | RouteTarget::InternalService { module_id, .. } => module_id,
6116 }
6117}
6118
6119fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
6120 roles.iter().any(|role| match (target, role) {
6121 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
6122 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
6123 (
6124 RouteTarget::InternalService { service_id, .. },
6125 ProviderRole::InternalService {
6126 service_id: provided,
6127 ..
6128 },
6129 ) => service_id == provided,
6130 _ => false,
6131 })
6132}
6133
6134fn is_routable_role(role: &ProviderRole) -> bool {
6135 matches!(
6136 role,
6137 ProviderRole::ToolProvider { .. }
6138 | ProviderRole::ManagementSurface { .. }
6139 | ProviderRole::InternalService { .. }
6140 )
6141}
6142
6143#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6144enum ControlRequestBodyError {
6145 UnknownOp,
6146 InvalidBody,
6147}
6148
6149#[derive(Debug, Deserialize)]
6150struct ControlOpProbe {
6151 op: String,
6152}
6153
6154const MODULE_PUSH_OPS: &[&str] = &["route.status"];
6157
6158fn is_known_module_push_op(body: &[u8]) -> bool {
6159 serde_json::from_slice::<ControlOpProbe>(body)
6160 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
6161 .unwrap_or(false)
6162}
6163
6164fn is_known_module_request_op(body: &[u8]) -> bool {
6165 serde_json::from_slice::<ControlOpProbe>(body)
6166 .map(|probe| is_module_to_subc_op(&probe.op))
6167 .unwrap_or(false)
6168}
6169
6170fn is_module_to_subc_op(op: &str) -> bool {
6171 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
6172}
6173
6174fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
6175 debug!(
6176 op = %op,
6177 connection_id = connection_id.get(),
6178 corr,
6179 "control dispatch"
6180 );
6181}
6182
6183fn log_slow_control_dispatch(
6184 dispatch_started_at: Option<StdInstant>,
6185 op: &'static str,
6186 connection_id: ConnectionId,
6187 corr: u64,
6188) {
6189 let Some(dispatch_started_at) = dispatch_started_at else {
6190 return;
6191 };
6192 let elapsed = dispatch_started_at.elapsed();
6193 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
6194 warn!(
6195 op = %op,
6196 connection_id = connection_id.get(),
6197 corr,
6198 elapsed_ms = elapsed.as_millis() as u64,
6199 "slow control dispatch"
6200 );
6201 }
6202}
6203
6204fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
6205 match request {
6206 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
6207 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
6208 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
6209 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
6210 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
6211 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
6212 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
6213 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
6214 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
6215 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
6216 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
6217 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
6218 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
6219 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
6220 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
6221 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
6222 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
6223 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
6224 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
6225 }
6226}
6227
6228fn principal_label(principal: &Principal) -> String {
6229 match principal {
6230 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
6231 Principal::Direct => "direct".to_string(),
6232 other => format!("{other:?}"),
6233 }
6234}
6235
6236fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
6237 match request {
6238 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
6239 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
6240 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
6241 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
6242 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
6243 }
6244}
6245
6246fn parse_client_control_request(
6247 body: &[u8],
6248) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
6249 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
6250 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6251 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
6252 ControlRequestBodyError::InvalidBody
6253 }
6254 Ok(_) => ControlRequestBodyError::UnknownOp,
6255 Err(_) => ControlRequestBodyError::InvalidBody,
6256 };
6257 (err, classification)
6258 })
6259}
6260
6261fn parse_module_control_request_from_module(
6262 body: &[u8],
6263) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
6264 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
6265 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6266 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
6267 Ok(_) => ControlRequestBodyError::UnknownOp,
6268 Err(_) => ControlRequestBodyError::InvalidBody,
6269 };
6270 (err, classification)
6271 })
6272}
6273
6274#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
6275enum ProviderRoleKind {
6276 ToolProvider,
6277 PipelineStage,
6278 ManagementSurface,
6279 InternalService,
6280}
6281
6282fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
6283 match role {
6284 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
6285 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
6286 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
6287 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
6288 }
6289}
6290
6291fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
6292 roles.iter().map(provider_role_kind).collect()
6293}
6294
6295const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
6298
6299#[derive(Debug, Default, PartialEq, Eq)]
6301struct ScopeOutcomeCounts {
6302 created: usize,
6303 replaced: usize,
6304 updated: usize,
6305 unchanged: usize,
6306 refused: usize,
6307}
6308
6309impl ScopeOutcomeCounts {
6310 fn of(results: &[ScopeRecordResult]) -> Self {
6311 let mut counts = Self::default();
6312 for result in results {
6313 let slot = match result.outcome {
6314 ScopeRecordOutcome::Created => &mut counts.created,
6315 ScopeRecordOutcome::Replaced => &mut counts.replaced,
6316 ScopeRecordOutcome::Updated => &mut counts.updated,
6317 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
6318 ScopeRecordOutcome::Refused => &mut counts.refused,
6319 };
6320 *slot += 1;
6321 }
6322 counts
6323 }
6324}
6325
6326#[cfg(test)]
6327mod scope_outcome_count_tests {
6328 use super::*;
6329
6330 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
6331 ScopeRecordResult {
6332 scope_ref: "r".to_string(),
6333 scope_epoch: 1,
6334 outcome,
6335 code: None,
6336 message: None,
6337 version: None,
6338 parent_state: None,
6339 }
6340 }
6341
6342 #[test]
6345 fn every_outcome_is_counted_in_its_own_field() {
6346 let results = [
6347 result(ScopeRecordOutcome::Created),
6348 result(ScopeRecordOutcome::Created),
6349 result(ScopeRecordOutcome::Replaced),
6350 result(ScopeRecordOutcome::Updated),
6351 result(ScopeRecordOutcome::Unchanged),
6352 result(ScopeRecordOutcome::Refused),
6353 result(ScopeRecordOutcome::Refused),
6354 result(ScopeRecordOutcome::Refused),
6355 ];
6356 assert_eq!(
6357 ScopeOutcomeCounts::of(&results),
6358 ScopeOutcomeCounts {
6359 created: 2,
6360 replaced: 1,
6361 updated: 1,
6362 unchanged: 1,
6363 refused: 3,
6364 }
6365 );
6366 }
6367}
6368
6369fn capability_census_trigger(
6373 old: Option<&CapabilityDeclarations>,
6374 new: Option<&CapabilityDeclarations>,
6375) -> bool {
6376 let old_provides = old
6377 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
6378 .unwrap_or_default();
6379 let old_denies = old
6380 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
6381 .unwrap_or_default();
6382 let new = new.cloned().unwrap_or(CapabilityDeclarations {
6383 provides: Vec::new(),
6384 requires: Vec::new(),
6385 must_never_reach: Vec::new(),
6386 });
6387
6388 new.provides
6389 .iter()
6390 .any(|capability| !old_provides.contains(capability))
6391 || new
6392 .must_never_reach
6393 .iter()
6394 .any(|capability| !old_denies.contains(capability))
6395}
6396
6397fn denied_capability<'a>(
6400 opening_manifest: &'a ModuleManifest,
6401 target_manifest: &ModuleManifest,
6402) -> Option<&'a str> {
6403 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
6404 let target_capabilities = target_manifest.capabilities.as_ref()?;
6405 opening_capabilities
6406 .must_never_reach
6407 .iter()
6408 .find(|denied| {
6409 target_capabilities
6410 .provides
6411 .iter()
6412 .any(|provided| provided == *denied)
6413 })
6414 .map(String::as_str)
6415}
6416
6417fn catalog_update_frozen_field_message(
6418 registered: &ModuleManifest,
6419 provides: &[ProviderRole],
6420) -> Option<String> {
6421 let old_has_provides = !registered.provides.is_empty();
6422 let new_has_provides = !provides.is_empty();
6423 if old_has_provides != new_has_provides {
6424 return Some(format!(
6425 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
6426 registered.module_id
6427 ));
6428 }
6429
6430 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
6431 return Some(format!(
6432 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
6433 registered.module_id
6434 ));
6435 }
6436
6437 let registered_concurrency = manifest_concurrency(registered);
6438 let mut candidate = registered.clone();
6439 candidate.provides = provides.to_vec();
6440 let candidate_concurrency = manifest_concurrency(&candidate);
6441 if candidate_concurrency != registered_concurrency {
6442 return Some(format!(
6443 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
6444 registered.module_id, registered_concurrency, candidate_concurrency
6445 ));
6446 }
6447
6448 None
6451}
6452
6453fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
6454 manifest.provides.iter().any(is_routable_role)
6455}
6456
6457fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
6463 manifest
6464 .provides
6465 .iter()
6466 .find_map(|provider| match provider {
6467 ProviderRole::ToolProvider { concurrency, .. }
6468 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
6469 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
6470 })
6471 .unwrap_or(Concurrency::ModuleManaged)
6472}
6473
6474fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
6481 let has_management_surface = manifest
6482 .provides
6483 .iter()
6484 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
6485 if !has_management_surface {
6486 return false;
6487 }
6488 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
6489 return false;
6490 };
6491 let Some(provides) = raw
6492 .get("manifest")
6493 .and_then(|manifest| manifest.get("provides"))
6494 .and_then(serde_json::Value::as_array)
6495 else {
6496 return false;
6497 };
6498 provides.iter().any(|role| {
6503 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
6504 && role.get("concurrency").is_none()
6505 })
6506}
6507
6508fn negotiate_version(peer_version: u8) -> Result<u8, String> {
6509 if peer_version != PROTOCOL_VERSION {
6510 return Err(format!(
6511 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
6512 ));
6513 }
6514 Ok(PROTOCOL_VERSION)
6515}
6516
6517fn pong(frame: &Frame) -> Result<Frame, RouterError> {
6518 Frame::build_with_version(
6519 response_version(frame),
6520 FrameType::Pong,
6521 frame.header.flags,
6522 0,
6523 0,
6524 frame.header.corr,
6525 Vec::new(),
6526 )
6527 .map_err(RouterError::FrameBuild)
6528}
6529
6530pub(crate) fn control_error_frame(
6531 frame: &Frame,
6532 code: &'static str,
6533 message: impl Into<String>,
6534) -> Result<Frame, RouterError> {
6535 control_error_body_frame(
6536 frame,
6537 ErrorBody {
6538 code: code.to_string(),
6539 message: message.into(),
6540 detail: None,
6541 },
6542 )
6543}
6544
6545fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
6546 let body = serde_json::to_vec(&error).map_err(|err| {
6547 RouterError::backend(
6548 0,
6549 frame.header.corr,
6550 format!("failed to encode control ERROR: {err}"),
6551 )
6552 })?;
6553
6554 Frame::build_with_version(
6555 response_version(frame),
6556 FrameType::Error,
6557 control_flags(),
6558 0,
6559 0,
6560 frame.header.corr,
6561 body,
6562 )
6563 .map_err(RouterError::FrameBuild)
6564}
6565
6566fn control_response_body_frame<T: Serialize>(
6567 frame: &Frame,
6568 reply: &T,
6569 label: &'static str,
6570) -> Result<Frame, RouterError> {
6571 let body = serde_json::to_vec(reply).map_err(|err| {
6572 RouterError::backend(
6573 0,
6574 frame.header.corr,
6575 format!("failed to encode {label}: {err}"),
6576 )
6577 })?;
6578
6579 Frame::build_with_version(
6580 response_version(frame),
6581 FrameType::Response,
6582 control_flags(),
6583 0,
6584 0,
6585 frame.header.corr,
6586 body,
6587 )
6588 .map_err(RouterError::FrameBuild)
6589}
6590
6591fn forwarding_error_code(err: &ForwardingError) -> &'static str {
6621 match err {
6622 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
6623 ForwardingError::NoModuleConnection => "target_unavailable",
6624 ForwardingError::ModuleReloading { .. } => "module_reloading",
6625 ForwardingError::ClientRouteChannelExhausted { .. }
6626 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
6627 ForwardingError::StaleModuleEndpoint
6628 | ForwardingError::UnknownReservation { .. }
6629 | ForwardingError::ConnectionClosing { .. }
6630 | ForwardingError::ClientEgressClosed { .. }
6631 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
6632 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
6635 ForwardingError::RelayCorrelationExhausted
6636 | ForwardingError::RouteOpenBuild(_)
6637 | ForwardingError::Poisoned => "forwarding_error",
6638 }
6639}
6640
6641fn response_version(frame: &Frame) -> u8 {
6642 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
6643 frame.header.ver
6644 } else {
6645 PROTOCOL_VERSION
6646 }
6647}
6648
6649fn control_flags() -> Flags {
6650 Flags::new(false, Priority::Passive, false)
6651}
6652
6653fn send_goodbye_target_best_effort(
6658 counters: &DaemonCounters,
6659 target: &GoodbyeTarget,
6660 context: &'static str,
6661) {
6662 let Ok(frame) = Frame::build_with_version(
6663 target.negotiated_ver,
6664 FrameType::Goodbye,
6665 control_flags(),
6666 target.channel,
6667 target.epoch,
6668 0,
6669 Vec::new(),
6670 ) else {
6671 return;
6672 };
6673 crate::forwarding::send_module_route_goodbye(
6674 counters,
6675 &target.sink,
6676 frame,
6677 target.module_id.as_deref(),
6678 context,
6679 );
6680}
6681
6682pub(crate) fn send_route_control_pushes(
6683 forwarding: &ForwardingTable,
6684 routes: Vec<EndpointRoute>,
6685 push: ClientControlPush,
6686) {
6687 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
6688 for route in routes {
6689 let target = route.goodbye_target;
6690 if let Some((existing, channels)) = targets
6691 .iter_mut()
6692 .find(|(existing, _)| existing.connection_id == target.connection_id)
6693 {
6694 debug_assert_eq!(
6695 existing.negotiated_ver, target.negotiated_ver,
6696 "one connection cannot negotiate multiple frame versions"
6697 );
6698 if !channels.contains(&target.channel) {
6699 channels.push(target.channel);
6700 }
6701 continue;
6702 }
6703 let channel = target.channel;
6704 targets.push((target, vec![channel]));
6705 }
6706 for (target, mut channels) in targets {
6707 channels.sort_unstable();
6708 let mut push = push.clone();
6709 match &mut push {
6710 ClientControlPush::RouteClosing {
6711 channels: covered, ..
6712 }
6713 | ClientControlPush::RouteClosed {
6714 channels: covered, ..
6715 } => *covered = channels,
6716 }
6717 let body = match serde_json::to_vec(&push) {
6718 Ok(body) => body,
6719 Err(err) => {
6720 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
6721 continue;
6722 }
6723 };
6724 let frame = match Frame::build_with_version(
6725 target.negotiated_ver,
6726 FrameType::Push,
6727 control_flags(),
6728 0,
6729 0,
6730 0,
6731 body.clone(),
6732 ) {
6733 Ok(frame) => frame,
6734 Err(err) => {
6735 warn!(
6736 route_channel = target.channel,
6737 error = %err,
6738 "failed to build route lifecycle control PUSH frame"
6739 );
6740 continue;
6741 }
6742 };
6743 if let Err(err) = target.sink.try_send(frame) {
6744 if target.close_on_delivery_failure() {
6745 warn!(
6746 target_connection_id = target.connection_id.get(),
6747 route_channel = target.channel,
6748 error = %err,
6749 "route lifecycle control PUSH was not delivered to client; closing target connection"
6750 );
6751 let _ = forwarding.escalate_client_delivery_failure(
6752 target.connection_id,
6753 target.channel,
6754 target.epoch,
6755 CloseReason::new(
6756 "route_lifecycle_push_delivery_failed",
6757 format!(
6758 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
6759 target.channel
6760 ),
6761 ),
6762 crate::forwarding::UndeliveredFrame {
6763 module_id: target.module_id.as_deref(),
6764 sink: &target.sink,
6765 },
6766 );
6767 }
6768 }
6769 }
6770}
6771
6772#[cfg(test)]
6773mod tests {
6774 #[cfg(unix)]
6775 #[tokio::test]
6776 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
6777 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
6778 let path = dir.join("subc.jsonc");
6779 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
6780 "program":"/bin/sleep","args":["60"],"protocol":"none",
6781 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
6782 }}}).to_string()).unwrap();
6783 let mut configured = crate::daemon_config::load(&path)
6784 .unwrap()
6785 .unwrap()
6786 .modules
6787 .pop()
6788 .unwrap();
6789 let registry = std::sync::Arc::new(crate::Registry::default());
6790 let handle = crate::SupervisorHandle::new();
6791 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
6792 .with_handle(handle.clone());
6793 let module = supervisor
6794 .supervise_configured_with_health(
6795 configured.module_spec(),
6796 true,
6797 configured.health.clone(),
6798 None,
6799 configured.restart,
6800 )
6801 .unwrap();
6802 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
6803 let before = module.status().unwrap().pid;
6804 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
6805 configured.health.cadence = std::time::Duration::from_secs(3600);
6806 let health_only = handler
6807 .reconcile_supervised_modules(&supervisor, vec![configured.clone()], false)
6808 .await
6809 .unwrap();
6810 assert!(
6811 health_only.changed_pending_reload.is_empty(),
6812 "health policy is already applied live"
6813 );
6814 assert_eq!(module.status().unwrap().pid, before);
6815 assert_eq!(
6816 module.configuration().unwrap().1.http,
6817 configured.health.http
6818 );
6819 configured.args = vec!["61".into()];
6820 let launch = handler
6821 .reconcile_supervised_modules(&supervisor, vec![configured], false)
6822 .await
6823 .unwrap();
6824 assert_eq!(launch.changed_pending_reload, ["stock"]);
6825 assert_eq!(
6826 module.status().unwrap().pid,
6827 before,
6828 "a launch edit is stored until reload"
6829 );
6830 module.drain().await.unwrap();
6831 }
6832 use cortexkit_test_support::ScratchDir;
6833 use std::{
6834 collections::BTreeMap,
6835 fmt,
6836 path::PathBuf,
6837 sync::{Arc, Mutex},
6838 time::Duration,
6839 };
6840
6841 use serde_json::{json, Value};
6842 use subc_protocol::{
6843 manifest::{
6844 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
6845 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
6846 },
6847 session::HealthStatus,
6848 FrameType,
6849 };
6850
6851 use super::*;
6852 use crate::{
6853 forwarding::{DataRoute, DataRouteState},
6854 registry::ChannelState,
6855 router::FrameSink,
6856 stderr_tail::DEFAULT_MAX_LINE_BYTES,
6857 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
6858 RouteCtx, Router,
6859 };
6860 use tokio::{
6861 sync::mpsc,
6862 time::{sleep, Instant},
6863 };
6864 use tracing::{
6865 field::{Field, Visit},
6866 Event, Subscriber,
6867 };
6868 use tracing_subscriber::{layer::Context, prelude::*, Layer};
6869
6870 fn fake_aft_stub_path() -> PathBuf {
6889 let mut path = std::env::current_exe().expect("current_exe available in tests");
6890 path.pop(); path.pop(); path.push(if cfg!(windows) {
6893 "fake-aft-stub.exe"
6894 } else {
6895 "fake-aft-stub"
6896 });
6897 assert!(
6898 path.exists(),
6899 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
6900 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
6901 path.display()
6902 );
6903 path
6904 }
6905
6906 fn client_retries(code: &str) -> bool {
6911 subc_protocol::error_codes::is_retryable_route_open(code)
6912 }
6913
6914 #[test]
6925 fn retryability_of_forwarding_codes_matches_the_failure() {
6926 let transient = [
6929 ForwardingError::NoModuleConnection,
6930 ForwardingError::ModuleReloading {
6931 module_id: "m".into(),
6932 },
6933 ForwardingError::StaleModuleEndpoint,
6934 ForwardingError::UnknownReservation {
6935 client_channel: 1,
6936 module_channel: 1,
6937 },
6938 ForwardingError::ConnectionClosing {
6939 connection_id: ConnectionId::new(1),
6940 },
6941 ForwardingError::ClientEgressClosed {
6942 connection_id: ConnectionId::new(1),
6943 },
6944 ForwardingError::ModuleEgressUnavailable {
6945 connection_id: ConnectionId::new(1),
6946 },
6947 ];
6948 for err in transient {
6949 let code = forwarding_error_code(&err);
6950 assert!(
6951 client_retries(code),
6952 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
6953 );
6954 }
6955
6956 let permanent = [
6961 ForwardingError::ConnectionRoleConflict {
6962 connection_id: ConnectionId::new(1),
6963 },
6964 ForwardingError::ClientRouteChannelExhausted {
6965 connection_id: ConnectionId::new(1),
6966 },
6967 ForwardingError::ModuleRouteChannelExhausted {
6968 endpoint: ModuleEndpointId {
6969 connection_id: ConnectionId::new(1),
6970 generation: 1,
6971 },
6972 },
6973 ForwardingError::RelayCorrelationExhausted,
6974 ForwardingError::RouteOpenBuild("x".into()),
6975 ForwardingError::Poisoned,
6976 ];
6977 for err in permanent {
6978 let code = forwarding_error_code(&err);
6979 assert!(
6980 !client_retries(code),
6981 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
6982 );
6983 }
6984 }
6985
6986 #[tokio::test]
7014 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
7015 let handler = ControlHandler::default();
7016 let frame =
7017 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
7018
7019 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
7023 assert_eq!(
7024 stamped,
7025 Principal::Direct,
7026 "a caller that proved nothing must not be stamped as a supervised module"
7027 );
7028
7029 let forged = handler
7034 .route_open_principal(
7035 &frame,
7036 Some(ConsumerIdentity {
7037 module_id: "aft".to_string(),
7038 launch_nonce: "not-a-real-nonce".to_string(),
7039 }),
7040 )
7041 .unwrap();
7042 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
7043 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
7044 }
7045
7046 #[test]
7063 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
7064 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
7065 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
7066 let ClientControlRequest::RouteOpen {
7067 consumer_identity, ..
7068 } = parsed
7069 else {
7070 panic!("route.open body must parse as RouteOpen");
7071 };
7072 assert_eq!(
7073 consumer_identity,
7074 Some(ConsumerIdentity {
7075 module_id: "aft".to_string(),
7076 launch_nonce: "n".to_string(),
7077 }),
7078 "the wire field name must reach the value route_open_principal reads"
7079 );
7080 }
7081
7082 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
7083 ModuleManifest::builder(module_id, "0.1.0")
7084 .protocol_ver(protocol_ver)
7085 .provides(vec![ProviderRole::ToolProvider {
7086 tools: vec![Tool {
7087 name: "read".to_string(),
7088 description: None,
7089 execution_mode: ExecutionMode::Pure,
7090 schema: json!({"type": "object"}),
7091 }],
7092 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
7093 concurrency: Concurrency::ModuleManaged,
7094 emits_push: true,
7095 sub_supervises: true,
7096 }])
7097 .build()
7098 }
7099
7100 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
7101 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
7102 }
7103
7104 fn hello_frame_with_control_ops(
7105 module_id: &str,
7106 protocol_ver: u8,
7107 corr: u64,
7108 control_ops: Option<Vec<String>>,
7109 ) -> Frame {
7110 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
7111 }
7112
7113 fn hello_frame_with_nonce(
7114 module_id: &str,
7115 protocol_ver: u8,
7116 corr: u64,
7117 launch_nonce: Option<&str>,
7118 ) -> Frame {
7119 hello_frame_full(
7120 module_id,
7121 protocol_ver,
7122 corr,
7123 None,
7124 launch_nonce.map(ToOwned::to_owned),
7125 )
7126 }
7127
7128 fn hello_frame_full(
7129 module_id: &str,
7130 protocol_ver: u8,
7131 corr: u64,
7132 control_ops: Option<Vec<String>>,
7133 launch_nonce: Option<String>,
7134 ) -> Frame {
7135 let body = serde_json::to_vec(&ModuleHelloBody {
7136 manifest: manifest(module_id, protocol_ver),
7137 protocol_ver,
7138 control_ops,
7139 launch_nonce,
7140 })
7141 .unwrap();
7142 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7143 }
7144
7145 fn non_routable_hello_frame_with_control_ops(
7146 module_id: &str,
7147 corr: u64,
7148 control_ops: Option<Vec<String>>,
7149 ) -> Frame {
7150 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
7151 manifest.provides.clear();
7152 let body = serde_json::to_vec(&ModuleHelloBody {
7153 manifest,
7154 protocol_ver: PROTOCOL_VERSION,
7155 control_ops,
7156 launch_nonce: None,
7157 })
7158 .unwrap();
7159 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7160 }
7161
7162 fn capability_grammar_hello_frame(
7163 capabilities: Value,
7164 runtime_computed: Option<Value>,
7165 corr: u64,
7166 ) -> Frame {
7167 let mut body = serde_json::to_value(ModuleHelloBody {
7168 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
7169 protocol_ver: PROTOCOL_VERSION,
7170 control_ops: None,
7171 launch_nonce: None,
7172 })
7173 .expect("HELLO body serializes");
7174 body["manifest"]["capabilities"] = capabilities;
7175 if let Some(runtime_computed) = runtime_computed {
7176 body["runtime_computed"] = runtime_computed;
7177 }
7178 Frame::build(
7179 FrameType::Hello,
7180 control_flags(),
7181 0,
7182 0,
7183 corr,
7184 serde_json::to_vec(&body).expect("HELLO body reserializes"),
7185 )
7186 .expect("HELLO frame builds")
7187 }
7188
7189 fn channel_request(channel: u16, corr: u64) -> Frame {
7190 Frame::build(
7191 FrameType::Request,
7192 Flags::new(true, Priority::Interactive, false),
7193 channel,
7194 0,
7195 corr,
7196 b"opaque".to_vec(),
7197 )
7198 .unwrap()
7199 }
7200
7201 fn route_ctx(
7202 connection_id: ConnectionId,
7203 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
7204 let (tx, rx) = mpsc::channel(8);
7205 (
7206 RouteCtx {
7207 connection_id,
7208 egress: FrameSink::new(tx),
7209 },
7210 rx,
7211 )
7212 }
7213
7214 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
7215 serde_json::from_slice(&frame.body).unwrap()
7216 }
7217
7218 async fn hello_via_sink(
7223 handler: &ControlHandler,
7224 ctx: &RouteCtx,
7225 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7226 hello: Frame,
7227 ) -> Frame {
7228 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
7229 assert!(
7230 replies.is_empty(),
7231 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
7232 );
7233 let ack = rx
7234 .try_recv()
7235 .expect("HELLO_ACK is queued on the module sink")
7236 .frame;
7237 assert_eq!(ack.header.ty, FrameType::HelloAck);
7238 ack
7239 }
7240
7241 fn parse_error(frame: &Frame) -> Value {
7242 serde_json::from_slice(&frame.body).unwrap()
7243 }
7244
7245 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
7246 serde_json::from_slice(&frame.body).unwrap()
7247 }
7248
7249 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
7250 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
7251 route_channel,
7252 route_epoch: 0,
7253 kind,
7254 })
7255 .unwrap();
7256 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7257 }
7258
7259 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
7260 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
7261 module_id: module_id.to_string(),
7262 })
7263 .unwrap();
7264 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7265 }
7266
7267 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
7268 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
7269 }
7270
7271 fn route_open_frame_with_consumer_capabilities(
7272 corr: u64,
7273 module_id: &str,
7274 project_root: ScratchDir,
7275 consumer_capabilities: Option<Vec<String>>,
7276 ) -> Frame {
7277 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7278 target: RouteTarget::ToolProvider {
7279 module_id: module_id.to_string(),
7280 },
7281 identity: BindIdentity::new(
7282 project_root.path().to_path_buf(),
7283 "unit".to_string(),
7284 "session".to_string(),
7285 ),
7286 consumer_identity: None,
7287 consumer_capabilities,
7288 role_versions: None,
7289 admission_facts: None,
7290 scope: None,
7291 })
7292 .unwrap();
7293 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7294 }
7295
7296 fn route_open_frame_with_role_versions(
7297 corr: u64,
7298 module_id: &str,
7299 project_root: ScratchDir,
7300 role_versions: Option<BTreeMap<String, String>>,
7301 ) -> Frame {
7302 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7303 target: RouteTarget::ToolProvider {
7304 module_id: module_id.to_string(),
7305 },
7306 identity: BindIdentity::new(
7307 project_root.path().to_path_buf(),
7308 "unit".to_string(),
7309 format!("session-{corr}"),
7310 ),
7311 consumer_identity: None,
7312 consumer_capabilities: None,
7313 role_versions,
7314 admission_facts: None,
7315 scope: None,
7316 })
7317 .unwrap();
7318 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7319 }
7320
7321 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
7322 entries
7323 .iter()
7324 .map(|(role, version)| (role.to_string(), version.to_string()))
7325 .collect()
7326 }
7327
7328 fn route_open_frame_with_admission_facts(
7329 corr: u64,
7330 module_id: &str,
7331 project_root: ScratchDir,
7332 consumer_identity: Option<subc_control::ConsumerIdentity>,
7333 facts: Option<Value>,
7334 ) -> Frame {
7335 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7336 target: RouteTarget::ToolProvider {
7337 module_id: module_id.to_string(),
7338 },
7339 identity: BindIdentity::new(
7340 project_root.path().to_path_buf(),
7341 "unit".to_string(),
7342 format!("session-{corr}"),
7343 ),
7344 consumer_identity,
7345 consumer_capabilities: None,
7346 role_versions: None,
7347 admission_facts: facts,
7348 scope: None,
7349 })
7350 .unwrap();
7351 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7352 }
7353
7354 #[derive(Clone, Default)]
7355 struct EventCapture {
7356 events: Arc<Mutex<Vec<CapturedEvent>>>,
7357 }
7358
7359 #[derive(Clone, Debug)]
7360 struct CapturedEvent {
7361 target: String,
7362 level: tracing::Level,
7363 fields: BTreeMap<String, String>,
7364 }
7365
7366 impl EventCapture {
7367 fn events(&self) -> Vec<CapturedEvent> {
7368 self.events.lock().unwrap().clone()
7369 }
7370 }
7371
7372 impl<S> Layer<S> for EventCapture
7373 where
7374 S: Subscriber,
7375 {
7376 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
7377 let mut visitor = EventFieldVisitor::default();
7378 event.record(&mut visitor);
7379 self.events.lock().unwrap().push(CapturedEvent {
7380 target: event.metadata().target().to_string(),
7381 level: *event.metadata().level(),
7382 fields: visitor.fields,
7383 });
7384 }
7385 }
7386
7387 #[derive(Default)]
7388 struct EventFieldVisitor {
7389 fields: BTreeMap<String, String>,
7390 }
7391
7392 impl Visit for EventFieldVisitor {
7393 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
7394 self.fields
7395 .insert(field.name().to_string(), format!("{value:?}"));
7396 }
7397 }
7398
7399 fn health_response(corr: u64, status: HealthStatus) -> Frame {
7400 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
7401 status,
7402 detail: Some("warming".to_string()),
7403 metrics: Some(json!({"queue_depth": 3})),
7404 })
7405 .unwrap();
7406 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7407 }
7408
7409 fn route_bind_ack(corr: u64) -> Frame {
7410 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
7411 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7412 }
7413
7414 fn unique_project_root(label: &str) -> ScratchDir {
7415 ScratchDir::new(label)
7416 }
7417
7418 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
7419 match parse_route_poll(frame) {
7420 ClientControlResponse::RoutePoll {
7421 status: None,
7422 live: Some(live),
7423 ..
7424 } => assert_eq!(live, expected_live),
7425 other => panic!("unexpected route.poll response: {other:?}"),
7426 }
7427 }
7428
7429 fn bind_liveness_route(
7430 registry: &Registry,
7431 forwarding: &ForwardingTable,
7432 module_id: &str,
7433 ) -> (RouteCtx, u16, u32) {
7434 let module_connection = ConnectionId::new(101);
7435 let client_connection = ConnectionId::new(202);
7436 let registration = registry
7437 .register_with_control_ops(
7438 manifest(module_id, PROTOCOL_VERSION),
7439 PROTOCOL_VERSION,
7440 module_connection,
7441 module_baseline_control_ops(),
7442 )
7443 .unwrap();
7444 let (module_tx, _module_rx) = mpsc::channel(8);
7445 let endpoint = forwarding
7446 .register_module_connection(
7447 module_connection,
7448 module_id.to_string(),
7449 PROTOCOL_VERSION,
7450 manifest_concurrency(®istration.manifest),
7451 FrameSink::new(module_tx),
7452 )
7453 .unwrap();
7454 let (client_ctx, _client_rx) = route_ctx(client_connection);
7455 let pending = forwarding
7456 .begin_route_bind_relay_for_test(
7457 client_connection,
7458 client_ctx.egress.clone(),
7459 1,
7460 module_id,
7461 )
7462 .unwrap();
7463 assert_eq!(pending.endpoint, endpoint);
7464 let route_channel = pending.client_channel;
7465 let route_epoch = pending.client_epoch;
7466 forwarding
7467 .complete_pending_relay(
7468 module_connection,
7469 pending.corr,
7470 RouteBindRelayOutcome::Accepted,
7471 )
7472 .unwrap();
7473 (client_ctx, route_channel, route_epoch)
7474 }
7475
7476 struct FakeProcessLiveness {
7477 live: Option<bool>,
7478 }
7479
7480 impl ModuleProcessLiveness for FakeProcessLiveness {
7481 fn process_live(&self, _module_id: &str) -> Option<bool> {
7482 self.live
7483 }
7484 }
7485
7486 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7487 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
7488 {
7489 let registry = Arc::new(Registry::default());
7490 let supervisor_handle = SupervisorHandle::new();
7491 let supervisor = Supervisor::new_for_test(
7492 Arc::clone(®istry),
7493 RestartPolicy::new(1, Duration::from_millis(10)),
7494 )
7495 .with_handle(supervisor_handle.clone());
7496 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
7497 let module = supervisor
7498 .spawn(ModuleSpec {
7499 module_id: "stderr-tail-wire".to_string(),
7500 program: fake_aft_stub_path(),
7501 args: Vec::new(),
7502 env: vec![
7503 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
7504 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
7505 ],
7506 reserved: false,
7507 reserved_prefixes: Vec::new(),
7508 protocol: ModuleProtocol::Subc,
7509 overlap: Default::default(),
7510 })
7511 .unwrap();
7512
7513 let deadline = Instant::now() + Duration::from_secs(5);
7514 loop {
7515 let tail = module.stderr_tail(None, None);
7516 if tail
7517 .entries
7518 .iter()
7519 .any(|entry| matches!(entry, TailEntry::ProcessStart))
7520 && tail.entries.iter().any(|entry| {
7521 matches!(
7522 entry,
7523 TailEntry::Line {
7524 truncated: true,
7525 ..
7526 }
7527 )
7528 })
7529 {
7530 break;
7531 }
7532 assert!(
7533 Instant::now() < deadline,
7534 "module did not produce a truncated line and restart boundary: {tail:?}"
7535 );
7536 sleep(Duration::from_millis(10)).await;
7537 }
7538
7539 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
7540 let request = ClientControlRequest::SupervisorStderrTail {
7541 module_id: "stderr-tail-wire".to_string(),
7542 max_lines: None,
7543 max_bytes: None,
7544 };
7545 let frame = Frame::build(
7546 FrameType::Request,
7547 control_flags(),
7548 0,
7549 0,
7550 1,
7551 serde_json::to_vec(&request).unwrap(),
7552 )
7553 .unwrap();
7554 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7555 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7556 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
7557 serde_json::from_slice(&responses[0].body).unwrap()
7558 else {
7559 panic!("expected supervisor.stderr_tail response");
7560 };
7561
7562 assert!(
7563 tail.entries
7564 .iter()
7565 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
7566 "the control response lost the restart boundary"
7567 );
7568 let Some(StderrTailEntry::Line {
7569 text,
7570 truncated,
7571 at_ms,
7572 }) = tail.entries.iter().find(|entry| {
7573 matches!(
7574 entry,
7575 StderrTailEntry::Line {
7576 truncated: true,
7577 ..
7578 }
7579 )
7580 })
7581 else {
7582 panic!("the control response lost the truncated line");
7583 };
7584 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
7585 assert!(*truncated);
7586 assert!(
7587 at_ms.is_some(),
7588 "the control response lost the line's capture time"
7589 );
7590 }
7591
7592 #[tokio::test(flavor = "current_thread")]
7597 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
7598 let dir = ScratchDir::new("terminals-off-worker");
7599 let journal_path = dir.join("terminals.jsonl");
7600 let registry = Arc::new(Registry::default());
7601 let supervisor_handle = SupervisorHandle::new();
7602 let supervisor =
7603 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7604 .with_handle(supervisor_handle.clone())
7605 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
7606 let module = supervisor
7607 .spawn(ModuleSpec {
7608 module_id: "terminal-off-worker".to_string(),
7609 program: fake_aft_stub_path(),
7610 args: Vec::new(),
7611 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7612 reserved: false,
7613 reserved_prefixes: Vec::new(),
7614 protocol: ModuleProtocol::Subc,
7615 overlap: Default::default(),
7616 })
7617 .unwrap();
7618 let deadline = Instant::now() + Duration::from_secs(5);
7619 while module.terminal_history().entries.len() != 2 {
7620 assert!(Instant::now() < deadline, "module did not record two exits");
7621 sleep(Duration::from_millis(10)).await;
7622 }
7623
7624 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
7625 let handler =
7626 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
7627 let frame = Frame::build(
7628 FrameType::Request,
7629 control_flags(),
7630 0,
7631 0,
7632 1,
7633 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
7634 module_id: "terminal-off-worker".to_string(),
7635 })
7636 .unwrap(),
7637 )
7638 .unwrap();
7639 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7640 let spawned_at = std::time::Instant::now();
7641 let read = tokio::spawn({
7642 let handler = Arc::clone(&handler);
7643 async move { handler.handle_control_frame(&ctx, frame).await }
7644 });
7645 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
7648 .await
7649 .unwrap()
7650 .expect("the history read reached its pause");
7651 let elapsed = spawned_at.elapsed();
7652 assert!(
7653 elapsed < Duration::from_secs(2) && !read.is_finished(),
7654 "this task could not run while the history read was paused \
7655 (resumed after {elapsed:?}, read finished: {})",
7656 read.is_finished()
7657 );
7658
7659 drop(release);
7660 let responses = read.await.unwrap().unwrap();
7661 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7662 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
7663 panic!("expected supervisor.terminals response");
7664 };
7665 assert_eq!(terminals.entries.len(), 2);
7666 assert_eq!(terminals.journal_skipped_lines, 0);
7667 assert_eq!(terminals.journal_read_errors, 0);
7668 }
7669
7670 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7671 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
7672 let registry = Arc::new(Registry::default());
7673 let supervisor_handle = SupervisorHandle::new();
7674 let supervisor =
7675 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7676 .with_handle(supervisor_handle.clone());
7677 let module = supervisor
7678 .spawn(ModuleSpec {
7679 module_id: "terminal-golden".to_string(),
7680 program: fake_aft_stub_path(),
7681 args: Vec::new(),
7682 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7683 reserved: false,
7684 reserved_prefixes: Vec::new(),
7685 protocol: ModuleProtocol::Subc,
7686 overlap: Default::default(),
7687 })
7688 .unwrap();
7689
7690 let deadline = Instant::now() + Duration::from_secs(5);
7691 while module.terminal_history().entries.len() != 2 {
7692 assert!(
7693 Instant::now() < deadline,
7694 "module did not retain two terminal exits: {:?}",
7695 module.terminal_history()
7696 );
7697 sleep(Duration::from_millis(10)).await;
7698 }
7699
7700 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
7701 let request = ClientControlRequest::SupervisorTerminals {
7702 module_id: "terminal-golden".to_string(),
7703 };
7704 let frame = Frame::build(
7705 FrameType::Request,
7706 control_flags(),
7707 0,
7708 0,
7709 1,
7710 serde_json::to_vec(&request).unwrap(),
7711 )
7712 .unwrap();
7713 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7714 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7715 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7716 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
7717 panic!("expected supervisor.terminals response");
7718 };
7719 assert_eq!(terminals.entries.len(), 2);
7720 assert_eq!(terminals.dropped, 0);
7721
7722 let mut rendered = serde_json::to_value(response).unwrap();
7723 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
7726 for (index, entry) in rendered["entries"]
7727 .as_array_mut()
7728 .expect("terminal response entries array")
7729 .iter_mut()
7730 .enumerate()
7731 {
7732 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
7733 }
7734
7735 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7736 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
7737 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
7738 if std::env::var_os("UPDATE_GOLDEN").is_some() {
7739 std::fs::write(&golden_path, &serialized).unwrap();
7740 }
7741 let expected: Value =
7742 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
7743 assert_eq!(rendered, expected);
7744 }
7745
7746 #[test]
7747 fn hello_registers_manifest_and_returns_ack() {
7748 let registry = Arc::new(Registry::default());
7749 let handler = ControlHandler::new(Arc::clone(®istry));
7750 let conn = ConnectionId::new(1);
7751
7752 let responses = handler
7753 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
7754 .unwrap();
7755
7756 assert_eq!(responses.len(), 1);
7757 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7758 assert_eq!(responses[0].header.channel, 0);
7759 assert_eq!(responses[0].header.corr, 7);
7760 let ack = parse_ack(&responses[0]);
7761 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
7762 assert!(ack
7763 .subc_capabilities
7764 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
7765 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
7766 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
7767 assert!(ack
7768 .subc_ops
7769 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
7770 assert!(ack
7771 .subc_ops
7772 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
7773
7774 let registration = registry.get_module("aft").unwrap().unwrap();
7775 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
7776 assert_eq!(registration.state, ChannelState::Active);
7777 assert_eq!(registration.connection_id, conn);
7778 assert_eq!(registration.control_ops, module_baseline_control_ops());
7779 }
7780
7781 #[test]
7782 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
7783 let invalid_identifiers = [
7784 ("case_change", "credentials-Provider/v1"),
7785 ("leading_zero", "credentials-provider/v01"),
7786 ("trailing_hyphen", "credentials-provider-/v1"),
7787 ("consecutive_hyphens", "credentials--provider/v1"),
7788 ("uppercase", "Credentials-provider/v1"),
7789 ("missing_v", "credentials-provider/1"),
7790 ("whitespace", "credentials provider/v1"),
7791 ("zero_version", "credentials-provider/v0"),
7792 ("out_of_range_version", "credentials-provider/v4294967296"),
7793 (
7794 "overlength_name",
7795 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
7796 ),
7797 ];
7798 let mut cases = invalid_identifiers
7799 .into_iter()
7800 .map(|(name, identifier)| {
7801 (
7802 format!("identifier_{name}"),
7803 "capabilities.provides[0]".to_string(),
7804 identifier.to_string(),
7805 json!({ "provides": [identifier] }),
7806 None,
7807 )
7808 })
7809 .collect::<Vec<_>>();
7810 cases.extend([
7811 (
7812 "unknown_need".to_string(),
7813 "capabilities.requires[0].need".to_string(),
7814 "deferred".to_string(),
7815 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
7816 None,
7817 ),
7818 (
7819 "duplicate_provides".to_string(),
7820 "capabilities.provides[1]".to_string(),
7821 "credentials-provider/v1".to_string(),
7822 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
7823 None,
7824 ),
7825 (
7826 "duplicate_must_never_reach".to_string(),
7827 "capabilities.must_never_reach[1]".to_string(),
7828 "credentials-provider/v1".to_string(),
7829 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
7830 None,
7831 ),
7832 (
7833 "duplicate_requires_same_need".to_string(),
7834 "capabilities.requires[1]".to_string(),
7835 "credentials-provider/v1".to_string(),
7836 json!({ "requires": [
7837 { "capability": "credentials-provider/v1", "need": "required" },
7838 { "capability": "credentials-provider/v1", "need": "required" }
7839 ] }),
7840 None,
7841 ),
7842 (
7843 "duplicate_requires_conflicting_need".to_string(),
7844 "capabilities.requires[1]".to_string(),
7845 "credentials-provider/v1".to_string(),
7846 json!({ "requires": [
7847 { "capability": "credentials-provider/v1", "need": "required" },
7848 { "capability": "credentials-provider/v1", "need": "optional" }
7849 ] }),
7850 None,
7851 ),
7852 (
7853 "capabilities_root_pointer".to_string(),
7854 "runtime_computed[0]".to_string(),
7855 "/capabilities".to_string(),
7856 json!({}),
7857 Some(json!(["/capabilities"])),
7858 ),
7859 (
7860 "capabilities_descendant_pointer".to_string(),
7861 "runtime_computed[0]".to_string(),
7862 "/capabilities/provides".to_string(),
7863 json!({}),
7864 Some(json!(["/capabilities/provides"])),
7865 ),
7866 (
7867 "malformed_pointer_without_leading_slash".to_string(),
7868 "runtime_computed[0]".to_string(),
7869 "capabilities".to_string(),
7870 json!({}),
7871 Some(json!(["capabilities"])),
7872 ),
7873 (
7874 "malformed_pointer_escape".to_string(),
7875 "runtime_computed[0]".to_string(),
7876 "/roles/~2/tools".to_string(),
7877 json!({}),
7878 Some(json!(["/roles/~2/tools"])),
7879 ),
7880 (
7881 "unknown_capabilities_field".to_string(),
7882 "capabilities.future".to_string(),
7883 "<array>".to_string(),
7884 json!({ "future": [] }),
7885 None,
7886 ),
7887 ]);
7888
7889 for (index, (name, field, value, capabilities, runtime_computed)) in
7890 cases.into_iter().enumerate()
7891 {
7892 let registry = Arc::new(Registry::default());
7893 let handler = ControlHandler::new(Arc::clone(®istry));
7894 let response = handler
7895 .handle_control(
7896 ConnectionId::new((index + 1) as u64),
7897 capability_grammar_hello_frame(
7898 capabilities,
7899 runtime_computed,
7900 index as u64 + 1,
7901 ),
7902 )
7903 .expect("invalid HELLO returns a refusal");
7904
7905 assert_eq!(response.len(), 1, "{name} must emit one refusal");
7906 let error = parse_error(&response[0]);
7907 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
7908 let message = error["message"]
7909 .as_str()
7910 .expect("error message is a string");
7911 assert!(
7912 message.contains(&field),
7913 "{name}: field missing from {message}"
7914 );
7915 assert!(
7916 message.contains(&value),
7917 "{name}: value missing from {message}"
7918 );
7919 assert_eq!(
7920 registry
7921 .active_registration_count()
7922 .expect("registry reads"),
7923 0,
7924 "{name}: refused HELLO must not create a catalog entry"
7925 );
7926 }
7927 }
7928
7929 #[test]
7930 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
7931 let registry = Arc::new(Registry::default());
7932 let handler = ControlHandler::new(Arc::clone(®istry));
7933 let capabilities = json!({
7934 "provides": ["credentials-provider/v1"],
7935 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
7936 "must_never_reach": ["federation-transport/v1"]
7937 });
7938 let response = handler
7939 .handle_control(
7940 ConnectionId::new(99),
7941 capability_grammar_hello_frame(
7942 capabilities.clone(),
7943 Some(json!(["/roles/0/tools"])),
7944 99,
7945 ),
7946 )
7947 .expect("valid HELLO registers");
7948 assert_eq!(response[0].header.ty, FrameType::HelloAck);
7949
7950 let request = Frame::build(
7951 FrameType::Request,
7952 control_flags(),
7953 0,
7954 0,
7955 100,
7956 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7957 .expect("catalog request serializes"),
7958 )
7959 .expect("catalog request frame builds");
7960 let response = handler
7961 .handle_catalog_list(request, None)
7962 .expect("catalog list succeeds");
7963 let ClientControlResponse::CatalogList { modules, .. } =
7964 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
7965 else {
7966 panic!("catalog request must return catalog.list");
7967 };
7968 assert_eq!(modules.len(), 1);
7969 assert_eq!(
7970 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
7971 capabilities
7972 );
7973 }
7974
7975 #[test]
7976 fn catalog_list_mirrors_management_operation_description() {
7977 let registry = Arc::new(Registry::default());
7978 let handler = ControlHandler::new(Arc::clone(®istry));
7979 let description = "List managed records and return their identifiers and metadata.";
7980 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
7981 manifest.provides = vec![ProviderRole::ManagementSurface {
7982 operations: vec![ManagementOperation {
7983 name: "records.list".to_string(),
7984 kind: ManagementOperationKind::Query,
7985 description: Some(description.to_string()),
7986 }],
7987 config_schema: json!({"type": "object"}),
7988 observability: vec![ObservabilitySurface {
7989 name: "records.stats".to_string(),
7990 kind: ObservabilityKind::Snapshot,
7991 }],
7992 identity_scope: vec![IdentityScope::Project],
7993 concurrency: Concurrency::ModuleManaged,
7994 }];
7995 registry
7996 .register_with_control_ops(
7997 manifest,
7998 PROTOCOL_VERSION,
7999 ConnectionId::new(99),
8000 Vec::new(),
8001 )
8002 .expect("described management manifest registers");
8003
8004 let request = Frame::build(
8005 FrameType::Request,
8006 control_flags(),
8007 0,
8008 0,
8009 100,
8010 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8011 .expect("catalog request serializes"),
8012 )
8013 .expect("catalog request frame builds");
8014 let response = handler
8015 .handle_catalog_list(request, None)
8016 .expect("catalog list succeeds");
8017 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8018 assert_eq!(
8019 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
8020 "catalog.list must preserve the declared operation description verbatim"
8021 );
8022 }
8023
8024 #[test]
8025 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
8026 let registry = Arc::new(Registry::default());
8027 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8028 [("vault".to_string(), true), ("squatter".to_string(), true)],
8029 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8030 );
8031 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
8032 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8033 provides: vec!["credentials-provider/v1".to_string()],
8034 requires: Vec::new(),
8035 must_never_reach: Vec::new(),
8036 });
8037 let frame = Frame::build(
8038 FrameType::Hello,
8039 control_flags(),
8040 0,
8041 0,
8042 77,
8043 serde_json::to_vec(&ModuleHelloBody {
8044 manifest: squatter,
8045 protocol_ver: PROTOCOL_VERSION,
8046 control_ops: None,
8047 launch_nonce: None,
8048 })
8049 .expect("HELLO serializes"),
8050 )
8051 .expect("HELLO frame builds");
8052 let response = handler
8053 .handle_control(ConnectionId::new(77), frame)
8054 .expect("reserved claim receives a typed refusal");
8055 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8056 assert_eq!(
8057 registry
8058 .active_registration_count()
8059 .expect("registry reads"),
8060 0,
8061 "a reserved capability refusal must not leave a catalog entry"
8062 );
8063 }
8064
8065 #[test]
8066 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
8067 for settlement in ["timeout", "inconclusive", "drop"] {
8068 let breakers = RouteBindBreakers::default();
8069 let RouteBindAdmission::Admitted {
8070 guard: mut old,
8071 probe: false,
8072 } = breakers.admit("prov")
8073 else {
8074 panic!("ordinary relay admitted")
8075 };
8076 let RouteBindAdmission::Admitted {
8077 guard: mut opener, ..
8078 } = breakers.admit("prov")
8079 else {
8080 panic!("second relay admitted")
8081 };
8082 assert!(
8083 !opener
8084 .record_timeout(1, Duration::ZERO)
8085 .unwrap()
8086 .reopened_after_probe
8087 );
8088 let RouteBindAdmission::Admitted {
8089 guard: mut probe,
8090 probe: true,
8091 } = breakers.admit("prov")
8092 else {
8093 panic!("one cooldown probe admitted")
8094 };
8095 match settlement {
8096 "timeout" => assert!(
8097 !old.record_timeout(1, Duration::ZERO)
8098 .unwrap()
8099 .reopened_after_probe
8100 ),
8101 "inconclusive" => old.record_inconclusive(),
8102 "drop" => drop(old),
8103 _ => unreachable!(),
8104 }
8105 assert!(
8106 matches!(
8107 breakers.admit("prov"),
8108 RouteBindAdmission::Refused {
8109 probe_in_flight: true,
8110 ..
8111 }
8112 ),
8113 "{settlement} of a pre-open relay cannot release the real probe"
8114 );
8115 assert!(
8116 probe
8117 .record_timeout(1, Duration::ZERO)
8118 .unwrap()
8119 .reopened_after_probe
8120 );
8121 assert!(matches!(
8122 breakers.admit("prov"),
8123 RouteBindAdmission::Admitted { probe: true, .. }
8124 ));
8125 }
8126 let breakers = RouteBindBreakers::default();
8127 let admit = || match breakers.admit("prov") {
8128 RouteBindAdmission::Admitted { guard, .. } => guard,
8129 _ => panic!("relay admitted"),
8130 };
8131 admit().record_timeout(1, Duration::ZERO);
8132 let mut old_probe = admit();
8133 breakers.reset_for_new_module_connection("prov");
8134 admit().record_timeout(1, Duration::ZERO);
8135 let _new_probe = admit();
8136 old_probe.record_inconclusive();
8137 assert!(matches!(
8138 breakers.admit("prov"),
8139 RouteBindAdmission::Refused {
8140 probe_in_flight: true,
8141 ..
8142 }
8143 ));
8144 }
8145
8146 #[tokio::test]
8147 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
8148 for candidate in [false, true] {
8149 let registry = Arc::new(Registry::default());
8150 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8151 [("vault".to_string(), true), ("squatter".to_string(), true)],
8152 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8153 );
8154 let conn = ConnectionId::new(77);
8155 let (ctx, mut rx) = route_ctx(conn);
8156 let initial = capability_manifest("squatter", &[], &[]);
8157 if candidate {
8158 registry
8159 .register_candidate_with_control_ops(
8160 initial.clone(),
8161 PROTOCOL_VERSION,
8162 conn,
8163 module_baseline_control_ops(),
8164 )
8165 .unwrap();
8166 handler
8167 .forwarding
8168 .register_candidate_module_connection(
8169 conn,
8170 "squatter".to_string(),
8171 PROTOCOL_VERSION,
8172 manifest_concurrency(&initial),
8173 ctx.egress.clone(),
8174 )
8175 .unwrap();
8176 } else {
8177 hello_via_sink(
8178 &handler,
8179 &ctx,
8180 &mut rx,
8181 hello_frame_with_manifest(initial.clone(), 1),
8182 )
8183 .await;
8184 }
8185 let response = handler
8186 .handle_control_frame(
8187 &ctx,
8188 catalog_update_with_capabilities_frame(
8189 2,
8190 capability_manifest("squatter", &["credentials-provider/v1"], &[])
8191 .capabilities
8192 .unwrap(),
8193 ),
8194 )
8195 .await
8196 .unwrap();
8197 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8198 assert_eq!(
8199 registry
8200 .get_module_by_connection(conn)
8201 .unwrap()
8202 .unwrap()
8203 .manifest,
8204 initial
8205 );
8206 }
8207 }
8208
8209 #[test]
8210 fn server_describe_surfaces_required_capability_verdict_fields() {
8211 let registry = Arc::new(Registry::default());
8212 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8213 [
8214 ("consumer".to_string(), true),
8215 ("provider".to_string(), false),
8216 ],
8217 BTreeMap::new(),
8218 );
8219 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
8220 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8221 provides: Vec::new(),
8222 requires: vec![subc_protocol::manifest::CapabilityRequirement {
8223 capability: "credentials-provider/v1".to_string(),
8224 need: subc_protocol::manifest::CapabilityNeed::Required,
8225 }],
8226 must_never_reach: Vec::new(),
8227 });
8228 let hello = Frame::build(
8229 FrameType::Hello,
8230 control_flags(),
8231 0,
8232 0,
8233 78,
8234 serde_json::to_vec(&ModuleHelloBody {
8235 manifest: consumer,
8236 protocol_ver: PROTOCOL_VERSION,
8237 control_ops: None,
8238 launch_nonce: None,
8239 })
8240 .expect("HELLO serializes"),
8241 )
8242 .expect("HELLO frame builds");
8243 handler
8244 .handle_control(ConnectionId::new(78), hello)
8245 .expect("consumer registers");
8246 let describe = Frame::build(
8247 FrameType::Request,
8248 control_flags(),
8249 0,
8250 0,
8251 79,
8252 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
8253 .expect("request serializes"),
8254 )
8255 .expect("describe frame builds");
8256 let response = handler
8257 .handle_server_describe(describe)
8258 .expect("server.describe succeeds");
8259 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
8260 let requirement = &rendered["capability_requirements"][0];
8261 assert_eq!(requirement["consumer"], "consumer");
8262 assert_eq!(requirement["verdict"], "never_provided");
8263 assert_eq!(requirement["episode_seq"], 1);
8264 assert_eq!(requirement["config_satisfiable"], false);
8265 assert_eq!(requirement["runtime_available"], false);
8266 assert!(requirement["detail"]
8267 .as_str()
8268 .expect("detail string")
8269 .contains("credentials-provider/v1"));
8270 }
8271
8272 #[test]
8273 fn catalog_list_omits_capabilities_for_legacy_manifest() {
8274 let registry = Arc::new(Registry::default());
8275 let handler = ControlHandler::new(Arc::clone(®istry));
8276 let hello = handler
8277 .handle_control(
8278 ConnectionId::new(101),
8279 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
8280 )
8281 .expect("legacy HELLO registers");
8282 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
8283
8284 let request = Frame::build(
8285 FrameType::Request,
8286 control_flags(),
8287 0,
8288 0,
8289 102,
8290 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8291 .expect("catalog request serializes"),
8292 )
8293 .expect("catalog request frame builds");
8294 let response = handler
8295 .handle_catalog_list(request, None)
8296 .expect("catalog list succeeds");
8297 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8298 assert!(
8299 body["modules"][0].get("capabilities").is_none(),
8300 "legacy manifest must retain an absent capabilities field on catalog.list"
8301 );
8302 }
8303
8304 #[test]
8305 fn hello_ack_omits_storage_when_no_storage_config() {
8306 let registry = Arc::new(Registry::default());
8307 let handler = ControlHandler::new(Arc::clone(®istry));
8308 let responses = handler
8309 .handle_control(
8310 ConnectionId::new(1),
8311 hello_frame("aft", PROTOCOL_VERSION, 7),
8312 )
8313 .unwrap();
8314 let ack = parse_ack(&responses[0]);
8315 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
8316 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
8317 }
8318
8319 #[tokio::test]
8320 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
8321 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
8322 let registry = Arc::new(Registry::default());
8323 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
8324 let responses = handler
8325 .handle_control(
8326 ConnectionId::new(1),
8327 hello_frame("aft", PROTOCOL_VERSION, 7),
8328 )
8329 .unwrap();
8330 let ack = parse_ack(&responses[0]);
8331 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
8332
8333 let described = handler
8334 .handle_control_frame(
8335 &route_ctx(ConnectionId::new(2)).0,
8336 Frame::build(
8337 FrameType::Request,
8338 control_flags(),
8339 0,
8340 0,
8341 9,
8342 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8343 )
8344 .unwrap(),
8345 )
8346 .await
8347 .unwrap();
8348 let ClientControlResponse::ServerDescribe { machine_id, .. } =
8349 serde_json::from_slice(&described[0].body).unwrap()
8350 else {
8351 panic!("server.describe answered with another shape");
8352 };
8353 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
8354 }
8355
8356 #[test]
8357 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
8358 let registry = Arc::new(Registry::default());
8361 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
8362 crate::daemon_config::StorageConfig::Sqlite {
8363 data_home: std::path::PathBuf::from("/data"),
8364 },
8365 ));
8366
8367 let responses = handler
8368 .handle_control(
8369 ConnectionId::new(1),
8370 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
8371 )
8372 .unwrap();
8373 let ack = parse_ack(&responses[0]);
8374 assert_eq!(
8375 ack.storage,
8376 Some(serde_json::json!({
8377 "module_id": "alfonso-routing",
8378 "storage_namespace": "default",
8379 "isolation": { "kind": "module" },
8380 "backend": {
8381 "backend": "sqlite",
8382 "path": "/data/cortexkit/alfonso-routing/store.db"
8383 }
8384 })),
8385 "the delivered descriptor is the module's own sqlite store path"
8386 );
8387 }
8388
8389 #[test]
8390 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
8391 let registry = Arc::new(Registry::default());
8392 let handler = ControlHandler::new(Arc::clone(®istry));
8393 let conn = ConnectionId::new(1);
8394 let responses = handler
8395 .handle_control(
8396 conn,
8397 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8398 )
8399 .unwrap();
8400 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8401 let registration = registry.get_module("aft").unwrap().unwrap();
8402 assert_eq!(registration.control_ops, module_baseline_control_ops());
8403
8404 let frame =
8405 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
8406 assert!(handler
8407 .guard_module_control_op(&frame, "aft", "route.bind")
8408 .unwrap()
8409 .is_none());
8410 let error = handler
8411 .guard_module_control_op(&frame, "aft", "test.synthetic")
8412 .unwrap()
8413 .expect("synthetic ungranted op should be rejected");
8414 assert_eq!(error.header.ty, FrameType::Error);
8415 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
8416 }
8417
8418 #[test]
8419 fn hello_control_ops_some_adds_optional_grants() {
8420 let registry = Arc::new(Registry::default());
8421 let handler = ControlHandler::new(Arc::clone(®istry));
8422 handler
8423 .handle_control(
8424 ConnectionId::new(1),
8425 hello_frame_with_control_ops(
8426 "aft",
8427 PROTOCOL_VERSION,
8428 7,
8429 Some(vec![
8430 "future.synthetic".to_string(),
8431 "route.bind".to_string(),
8432 ]),
8433 ),
8434 )
8435 .unwrap();
8436 let registration = registry.get_module("aft").unwrap().unwrap();
8437 assert_eq!(
8438 registration.control_ops,
8439 vec![
8440 "route.bind".to_string(),
8441 "route.status".to_string(),
8442 "future.synthetic".to_string(),
8443 ]
8444 );
8445 let frame =
8446 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
8447 assert!(handler
8448 .guard_module_control_op(&frame, "aft", "future.synthetic")
8449 .unwrap()
8450 .is_none());
8451 }
8452
8453 #[tokio::test]
8454 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
8455 let registry = Arc::new(Registry::default());
8456 let forwarding = Arc::new(ForwardingTable::default());
8457 let handler =
8458 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8459 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
8460 hello_via_sink(
8461 &handler,
8462 &module_ctx,
8463 &mut module_rx,
8464 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8465 )
8466 .await;
8467
8468 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
8469 let responses = handler
8470 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
8471 .await
8472 .unwrap();
8473 assert_eq!(responses.len(), 1);
8474 assert_eq!(responses[0].header.ty, FrameType::Error);
8475 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
8476 assert!(module_rx.try_recv().is_err());
8477 }
8478
8479 #[tokio::test]
8480 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
8481 let registry = Arc::new(Registry::default());
8482 let forwarding = Arc::new(ForwardingTable::default());
8483 let handler =
8484 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8485 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
8486 hello_via_sink(
8487 &handler,
8488 &module_ctx,
8489 &mut module_rx,
8490 hello_frame_with_control_ops(
8491 "aft",
8492 PROTOCOL_VERSION,
8493 7,
8494 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8495 ),
8496 )
8497 .await;
8498
8499 let project_root = unique_project_root("demux");
8500 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
8501 let route_handler = handler.clone();
8502 let route_task = tokio::spawn(async move {
8503 route_handler
8504 .handle_control_frame(
8505 &route_client_ctx,
8506 route_open_frame(100, "aft", project_root),
8507 )
8508 .await
8509 .unwrap()
8510 });
8511 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8512 .await
8513 .unwrap()
8514 .unwrap();
8515 assert!(matches!(
8516 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
8517 ModuleControlRequest::RouteBind { .. }
8518 ));
8519
8520 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
8521 let health_handler = handler.clone();
8522 let health_task = tokio::spawn(async move {
8523 health_handler
8524 .handle_control_frame(
8525 &health_client_ctx,
8526 supervisor_health_probe_frame(101, "aft"),
8527 )
8528 .await
8529 .unwrap()
8530 });
8531 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8532 .await
8533 .unwrap()
8534 .unwrap();
8535 assert_eq!(
8536 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8537 ModuleControlRequest::HealthCheck {}
8538 );
8539
8540 handler
8541 .handle_control_frame(
8542 &module_ctx,
8543 health_response(health_frame.header.corr, HealthStatus::Degraded),
8544 )
8545 .await
8546 .unwrap();
8547 let health_response = health_task.await.unwrap();
8548 assert_eq!(health_response.len(), 1);
8549 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8550 ClientControlResponse::SupervisorHealthProbe {
8551 module_id,
8552 status,
8553 detail,
8554 metrics,
8555 } => {
8556 assert_eq!(module_id, "aft");
8557 assert_eq!(status, HealthStatus::Degraded);
8558 assert_eq!(detail.as_deref(), Some("warming"));
8559 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
8560 }
8561 other => panic!("unexpected health response: {other:?}"),
8562 }
8563
8564 handler
8565 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8566 .await
8567 .unwrap();
8568 let route_response = route_task.await.unwrap();
8569 assert!(route_response.is_empty());
8570 let published = route_client_rx.recv().await.unwrap();
8571 assert!(matches!(
8572 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8573 ClientControlResponse::RouteOpen { .. }
8574 ));
8575 }
8576
8577 async fn relay_route_open(
8582 handler: &ControlHandler,
8583 client_connection: ConnectionId,
8584 client_egress: &FrameSink,
8585 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8586 corr: u64,
8587 module_id: &str,
8588 project_root_label: &str,
8589 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
8590 let ctx = RouteCtx {
8591 connection_id: client_connection,
8592 egress: client_egress.clone(),
8593 };
8594 let handler = handler.clone();
8595 let project_root = unique_project_root(project_root_label);
8596 let module_id = module_id.to_string();
8597 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
8598 let task = tokio::spawn(async move {
8599 let _guard = tracing::dispatcher::set_default(&dispatch);
8600 handler
8601 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
8602 .await
8603 .unwrap()
8604 });
8605 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
8606 .await
8607 .expect("module receives the relayed route.bind")
8608 .expect("module egress is open");
8609 (task, bind.frame)
8610 }
8611
8612 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
8613 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
8614 ModuleControlRequest::RouteBind {
8615 route_channel,
8616 epoch,
8617 ..
8618 } => (route_channel, epoch),
8619 other => panic!("expected a route.bind request, got {other:?}"),
8620 }
8621 }
8622
8623 fn published_route(frame: &Frame) -> (u16, u32) {
8624 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
8625 ClientControlResponse::RouteOpen {
8626 route_channel,
8627 route_epoch,
8628 } => (route_channel, route_epoch),
8629 other => panic!("expected a route.open response, got {other:?}"),
8630 }
8631 }
8632
8633 #[tokio::test]
8653 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
8654 let registry = Arc::new(Registry::default());
8655 let forwarding = Arc::new(ForwardingTable::default());
8656 let handler =
8657 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8658
8659 let module_connection = ConnectionId::new(30);
8660 let (module_ctx, mut module_rx) = route_ctx(module_connection);
8661 hello_via_sink(
8662 &handler,
8663 &module_ctx,
8664 &mut module_rx,
8665 hello_frame("aft", PROTOCOL_VERSION, 7),
8666 )
8667 .await;
8668
8669 let dying_client = ConnectionId::new(31);
8670 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
8671
8672 let (first_task, first_bind) = relay_route_open(
8675 &handler,
8676 dying_client,
8677 &dying_ctx.egress,
8678 &mut module_rx,
8679 100,
8680 "aft",
8681 "closing-first",
8682 )
8683 .await;
8684 handler
8685 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
8686 .await
8687 .unwrap();
8688 assert!(first_task.await.unwrap().is_empty());
8689 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
8690
8691 let (second_task, second_bind) = relay_route_open(
8693 &handler,
8694 dying_client,
8695 &dying_ctx.egress,
8696 &mut module_rx,
8697 101,
8698 "aft",
8699 "closing-second",
8700 )
8701 .await;
8702 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
8703
8704 assert!(forwarding
8707 .escalate_client_delivery_failure(
8708 dying_client,
8709 first_channel,
8710 first_epoch,
8711 CloseReason::new(
8712 "module_to_client_delivery_failed",
8713 "client egress refused a module frame",
8714 ),
8715 crate::forwarding::UndeliveredFrame {
8716 module_id: None,
8717 sink: &dying_ctx.egress,
8718 },
8719 )
8720 .unwrap());
8721 assert!(!dying_ctx.egress.is_closed());
8722
8723 let ack = handler
8725 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
8726 .await;
8727 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
8728 if module_loop_error.is_some() {
8729 handler.cleanup_connection(module_connection).unwrap();
8733 }
8734 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8738 .await
8739 .ok()
8740 .flatten();
8741
8742 assert!(
8744 registry
8745 .get_module_by_connection(module_connection)
8746 .unwrap()
8747 .is_some(),
8748 "one client's closing connection ended the shared module connection: \
8749 {module_loop_error:?}"
8750 );
8751 let cotenant = ConnectionId::new(32);
8753 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
8754 let (cotenant_task, cotenant_bind) = relay_route_open(
8755 &handler,
8756 cotenant,
8757 &cotenant_ctx.egress,
8758 &mut module_rx,
8759 102,
8760 "aft",
8761 "closing-cotenant",
8762 )
8763 .await;
8764 handler
8765 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
8766 .await
8767 .unwrap();
8768 assert!(cotenant_task.await.unwrap().is_empty());
8769 let (cotenant_channel, cotenant_epoch) =
8770 published_route(&cotenant_rx.recv().await.unwrap());
8771 assert!(matches!(
8772 forwarding
8773 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
8774 .unwrap(),
8775 DataRoute::Client(DataRouteState::Bound(_))
8776 ));
8777
8778 let goodbye = post_ack_module_frame
8781 .expect("module receives a GOODBYE for the abandoned route channel");
8782 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8783 assert_eq!(goodbye.header.channel, abandoned_channel);
8784 assert_eq!(goodbye.header.epoch, abandoned_epoch);
8785
8786 assert!(dying_rx.try_recv().is_err());
8790 let second_response = second_task.await.unwrap();
8791 assert_eq!(second_response.len(), 1);
8792 assert_eq!(
8793 parse_error(&second_response[0])["code"],
8794 "target_unavailable"
8795 );
8796 }
8797
8798 async fn route_open_held_after_bind_send() -> (
8802 ControlHandler,
8803 Arc<tokio::sync::Barrier>,
8804 RouteCtx,
8805 mpsc::Receiver<crate::router::OutboundFrame>,
8806 ) {
8807 let hold = Arc::new(tokio::sync::Barrier::new(2));
8808 let handler = ControlHandler::with_forwarding(
8809 Arc::new(Registry::default()),
8810 Arc::new(ForwardingTable::default()),
8811 )
8812 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
8813 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
8814 hello_via_sink(
8815 &handler,
8816 &module_ctx,
8817 &mut module_rx,
8818 hello_frame("aft", PROTOCOL_VERSION, 7),
8819 )
8820 .await;
8821 (handler, hold, module_ctx, module_rx)
8822 }
8823
8824 #[tokio::test]
8831 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
8832 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
8833 let client = ConnectionId::new(41);
8834 let (client_ctx, mut client_rx) = route_ctx(client);
8835
8836 let (open_task, bind) = relay_route_open(
8837 &handler,
8838 client,
8839 &client_ctx.egress,
8840 &mut module_rx,
8841 200,
8842 "aft",
8843 "acked-before-mark",
8844 )
8845 .await;
8846 let (module_channel, module_epoch) = route_bind_channel(&bind);
8847 hold.wait().await;
8849
8850 assert!(handler
8852 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
8853 .await
8854 .unwrap()
8855 .is_empty());
8856 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
8857
8858 hold.wait().await;
8859 assert!(
8860 open_task.await.unwrap().is_empty(),
8861 "route.open answers accepted"
8862 );
8863
8864 match module_rx.try_recv() {
8867 Err(mpsc::error::TryRecvError::Empty) => {}
8868 Ok(outbound) => panic!(
8869 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
8870 outbound.frame.header.ty,
8871 outbound.frame.header.channel,
8872 outbound.frame.header.epoch
8873 ),
8874 Err(err) => panic!("module egress closed: {err}"),
8875 }
8876
8877 let router = Router::with_control_handler(Arc::new(handler.clone()));
8880 let request = Frame::build(
8881 FrameType::Request,
8882 Flags::new(true, Priority::Interactive, false),
8883 client_channel,
8884 client_epoch,
8885 201,
8886 b"first request".to_vec(),
8887 )
8888 .unwrap();
8889 router
8890 .route_for_connection(&client_ctx, request)
8891 .await
8892 .unwrap();
8893 let delivered = module_rx.try_recv().unwrap().frame;
8894 assert_eq!(delivered.header.ty, FrameType::Request);
8895 assert_eq!(
8896 (delivered.header.channel, delivered.header.epoch),
8897 (module_channel, module_epoch)
8898 );
8899 assert_eq!(delivered.body, b"first request");
8900 }
8901
8902 #[tokio::test]
8908 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
8909 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
8910 let client = ConnectionId::new(42);
8911 let (client_ctx, _client_rx) = route_ctx(client);
8912
8913 let (open_task, bind) = relay_route_open(
8914 &handler,
8915 client,
8916 &client_ctx.egress,
8917 &mut module_rx,
8918 300,
8919 "aft",
8920 "released-before-mark",
8921 )
8922 .await;
8923 let (module_channel, module_epoch) = route_bind_channel(&bind);
8924 hold.wait().await;
8925
8926 handler.cleanup_connection(client).unwrap();
8927 assert!(
8928 module_rx.try_recv().is_err(),
8929 "client teardown must not GOODBYE a bind route.open is still queueing"
8930 );
8931
8932 hold.wait().await;
8933 let response = open_task.await.unwrap();
8934 assert_eq!(response.len(), 1);
8935 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8936
8937 let goodbye = module_rx
8938 .try_recv()
8939 .expect("module is told to drop the binding the daemon abandoned")
8940 .frame;
8941 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8942 assert_eq!(
8943 (goodbye.header.channel, goodbye.header.epoch),
8944 (module_channel, module_epoch)
8945 );
8946 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
8947 }
8948
8949 #[tokio::test]
8954 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
8955 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
8956 let client = ConnectionId::new(43);
8957 let (client_ctx, client_rx) = route_ctx(client);
8958
8959 let (open_task, bind) = relay_route_open(
8960 &handler,
8961 client,
8962 &client_ctx.egress,
8963 &mut module_rx,
8964 400,
8965 "aft",
8966 "gone-client-before-mark",
8967 )
8968 .await;
8969 let (module_channel, module_epoch) = route_bind_channel(&bind);
8970 hold.wait().await;
8971
8972 drop(client_rx);
8973 assert!(client_ctx.egress.is_closed());
8974 handler
8975 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
8976 .await
8977 .unwrap();
8978
8979 hold.wait().await;
8980 let response = open_task.await.unwrap();
8981 assert_eq!(response.len(), 1);
8982 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8983
8984 let goodbye = module_rx
8985 .try_recv()
8986 .expect("module is told to drop the binding it acked")
8987 .frame;
8988 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8989 assert_eq!(
8990 (goodbye.header.channel, goodbye.header.epoch),
8991 (module_channel, module_epoch)
8992 );
8993 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
8994 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
8995 }
8996
8997 #[test]
9004 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
9005 let handler = ControlHandler::default();
9006 let module_connection = ConnectionId::new(30);
9007 let client_connection = ConnectionId::new(31);
9008
9009 handler
9010 .refuse_to_end_module_connection_for_a_client(
9011 module_connection,
9012 77,
9013 ForwardingError::ConnectionClosing {
9014 connection_id: client_connection,
9015 },
9016 )
9017 .expect("a closing client must never end the module connection");
9018
9019 assert!(matches!(
9020 handler.refuse_to_end_module_connection_for_a_client(
9021 module_connection,
9022 78,
9023 ForwardingError::ConnectionClosing {
9024 connection_id: module_connection,
9025 },
9026 ),
9027 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
9028 connection_id
9029 })) if connection_id == module_connection
9030 ));
9031 assert!(matches!(
9032 handler.refuse_to_end_module_connection_for_a_client(
9033 module_connection,
9034 79,
9035 ForwardingError::Poisoned,
9036 ),
9037 Err(RouterError::Forwarding(ForwardingError::Poisoned))
9038 ));
9039 assert!(matches!(
9040 handler.refuse_to_end_module_connection_for_a_client(
9041 module_connection,
9042 80,
9043 ForwardingError::StaleModuleEndpoint,
9044 ),
9045 Err(RouterError::Forwarding(
9046 ForwardingError::StaleModuleEndpoint
9047 ))
9048 ));
9049 }
9050
9051 #[tokio::test]
9064 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
9065 let registry = Arc::new(Registry::default());
9066 let forwarding = Arc::new(ForwardingTable::default());
9067 let supervisor = SupervisorHandle::new();
9068 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9069 let handler =
9070 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9071 .with_supervisor(supervisor);
9072
9073 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9074 hello_via_sink(
9075 &handler,
9076 &target_ctx,
9077 &mut target_rx,
9078 hello_frame("target", PROTOCOL_VERSION, 1),
9079 )
9080 .await;
9081
9082 let wrong_nonce = handler
9086 .handle_control_frame(
9087 &route_ctx(ConnectionId::new(91)).0,
9088 route_open_frame_with_admission_facts(
9089 20,
9090 "target",
9091 unique_project_root("admission-facts"),
9092 Some(subc_control::ConsumerIdentity {
9093 module_id: "fed".to_string(),
9094 launch_nonce: "not-the-real-nonce".to_string(),
9095 }),
9096 None,
9097 ),
9098 )
9099 .await
9100 .unwrap();
9101 assert_eq!(
9102 parse_error(&wrong_nonce[0])["code"],
9103 "bad_consumer_identity",
9104 "a mismatched launch nonce must be refused, not stamped Reserved"
9105 );
9106
9107 let never_spawned = handler
9111 .handle_control_frame(
9112 &route_ctx(ConnectionId::new(92)).0,
9113 route_open_frame_with_admission_facts(
9114 21,
9115 "target",
9116 unique_project_root("admission-facts"),
9117 Some(subc_control::ConsumerIdentity {
9118 module_id: "never-spawned".to_string(),
9119 launch_nonce: "any-nonce".to_string(),
9120 }),
9121 None,
9122 ),
9123 )
9124 .await
9125 .unwrap();
9126 assert_eq!(
9127 parse_error(&never_spawned[0])["code"],
9128 "bad_consumer_identity",
9129 "an unspawned module_id must be refused rather than accepted for lack of a record"
9130 );
9131 }
9132
9133 #[tokio::test]
9156 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
9157 let registry = Arc::new(Registry::default());
9158 let forwarding = Arc::new(ForwardingTable::default());
9159 let supervisor = SupervisorHandle::new();
9160 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9161 let handler =
9162 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9163 .with_supervisor(supervisor);
9164
9165 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
9166 hello_via_sink(
9167 &handler,
9168 &target_ctx,
9169 &mut target_rx,
9170 hello_frame("target", PROTOCOL_VERSION, 1),
9171 )
9172 .await;
9173
9174 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
9175 let route_handler = handler.clone();
9176 let route_task = tokio::spawn(async move {
9177 route_handler
9178 .handle_control_frame(
9179 &client_ctx,
9180 route_open_frame_with_admission_facts(
9181 30,
9182 "target",
9183 unique_project_root("admission-facts"),
9184 Some(subc_control::ConsumerIdentity {
9185 module_id: "fed".to_string(),
9186 launch_nonce: "fed-nonce".to_string(),
9187 }),
9188 None,
9189 ),
9190 )
9191 .await
9192 .unwrap()
9193 });
9194
9195 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9202 .await
9203 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
9204 .expect("module control channel closed before route.bind");
9205 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9206 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
9207 panic!("expected route.bind")
9208 };
9209 assert_eq!(
9210 principal,
9211 Some(Principal::Reserved {
9212 module_id: "fed".to_string()
9213 }),
9214 "a correctly attested consumer must be stamped Reserved for its own id"
9215 );
9216
9217 handler
9218 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9219 .await
9220 .unwrap();
9221 assert!(route_task.await.unwrap().is_empty());
9222 assert!(
9223 matches!(
9224 serde_json::from_slice::<ClientControlResponse>(
9225 &client_rx.recv().await.unwrap().body
9226 )
9227 .unwrap(),
9228 ClientControlResponse::RouteOpen { .. }
9229 ),
9230 "the route must actually open, not merely avoid an error"
9231 );
9232 }
9233
9234 #[tokio::test(start_paused = true)]
9235 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
9236 let registry = Arc::new(Registry::default());
9237 let forwarding = Arc::new(ForwardingTable::default());
9238 let supervisor = SupervisorHandle::new();
9239 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9240 let handler =
9241 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9242 .with_supervisor(supervisor);
9243
9244 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
9245 hello_via_sink(
9246 &handler,
9247 &target_ctx,
9248 &mut target_rx,
9249 hello_frame("target", PROTOCOL_VERSION, 1),
9250 )
9251 .await;
9252
9253 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
9254 let direct_handler = handler.clone();
9255 let direct_open = tokio::spawn(async move {
9256 direct_handler
9257 .handle_control_frame(
9258 &direct_ctx,
9259 route_open_frame(2, "target", unique_project_root("route-census-direct")),
9260 )
9261 .await
9262 .unwrap()
9263 });
9264 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9265 .await
9266 .expect("no direct route.bind within 5s")
9267 .expect("target control channel closed before direct route.bind");
9268 handler
9269 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
9270 .await
9271 .unwrap();
9272 assert!(direct_open.await.unwrap().is_empty());
9273 let _ = direct_rx.recv().await.unwrap();
9274
9275 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
9276 let reserved_handler = handler.clone();
9277 let reserved_open = tokio::spawn(async move {
9278 reserved_handler
9279 .handle_control_frame(
9280 &reserved_ctx,
9281 route_open_frame_with_admission_facts(
9282 3,
9283 "target",
9284 unique_project_root("admission-facts"),
9285 Some(ConsumerIdentity {
9286 module_id: "fed".to_string(),
9287 launch_nonce: "fed-nonce".to_string(),
9288 }),
9289 None,
9290 ),
9291 )
9292 .await
9293 .unwrap()
9294 });
9295 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9296 .await
9297 .expect("no reserved route.bind within 5s")
9298 .expect("target control channel closed before reserved route.bind");
9299 handler
9300 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
9301 .await
9302 .unwrap();
9303 assert!(reserved_open.await.unwrap().is_empty());
9304 let _ = reserved_rx.recv().await.unwrap();
9305
9306 forwarding
9307 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
9308 .unwrap();
9309 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
9310 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
9311 module_id: Some("target".to_string()),
9312 })
9313 .unwrap();
9314 let census_frame =
9315 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
9316 let response = handler
9317 .handle_control_frame(&census_ctx, census_frame)
9318 .await
9319 .unwrap()
9320 .pop()
9321 .unwrap();
9322 let actual: Value = serde_json::from_slice(&response.body).unwrap();
9323 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
9324 assert!(matches!(
9325 decoded,
9326 ClientControlResponse::SupervisorRoutes { .. }
9327 ));
9328 let routes = actual["modules"][0]["routes"].as_array().unwrap();
9329 assert_eq!(routes.len(), 2);
9330 assert!(routes.iter().all(|route| route["draining"] == true));
9331 assert!(
9334 routes.iter().all(|route| route["drain_reason"] == "reload"),
9335 "draining routes must name the drain's reason: {routes:?}"
9336 );
9337 assert!(routes.iter().any(|route| {
9338 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
9339 }));
9340 assert!(routes.iter().any(|route| {
9341 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
9342 }));
9343
9344 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9345 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
9346 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9347 std::fs::write(
9348 &golden_path,
9349 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
9350 )
9351 .unwrap();
9352 }
9353 let expected: Value =
9354 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
9355 assert_eq!(actual, expected);
9356 }
9357
9358 async fn query_live_roots(
9359 handler: &ControlHandler,
9360 module_ctx: &RouteCtx,
9361 ) -> ModuleControlResponseToModule {
9362 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
9363 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
9364 let response = handler
9365 .handle_control_frame(module_ctx, frame)
9366 .await
9367 .unwrap()
9368 .pop()
9369 .unwrap();
9370 serde_json::from_slice(&response.body).unwrap()
9371 }
9372
9373 #[tokio::test(start_paused = true)]
9374 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
9375 let registry = Arc::new(Registry::default());
9376 let forwarding = Arc::new(ForwardingTable::default());
9377 let handler = ControlHandler::with_forwarding(registry, forwarding);
9378 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
9379 hello_via_sink(
9380 &handler,
9381 &target_ctx,
9382 &mut target_rx,
9383 hello_frame("target", PROTOCOL_VERSION, 1),
9384 )
9385 .await;
9386 let root = unique_project_root("live-roots-known");
9387 let path = ProjectRootId::from_path_allowing_missing(root.path())
9388 .unwrap()
9389 .as_path()
9390 .to_path_buf();
9391 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
9392 let open_handler = handler.clone();
9393 let opened = tokio::spawn(async move {
9394 open_handler
9395 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
9396 .await
9397 .unwrap()
9398 });
9399 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9400 .await
9401 .unwrap()
9402 .unwrap();
9403 handler
9404 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
9405 .await
9406 .unwrap();
9407 assert!(opened.await.unwrap().is_empty());
9408 let _ = client_rx.recv().await.unwrap();
9409
9410 let root = unique_project_root("live-roots-pending");
9411 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
9412 .unwrap()
9413 .as_path()
9414 .to_path_buf();
9415 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
9416 let open_handler = handler.clone();
9417 let pending = tokio::spawn(async move {
9418 open_handler
9419 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
9420 .await
9421 .unwrap()
9422 });
9423 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9424 .await
9425 .unwrap()
9426 .unwrap();
9427 let actual = query_live_roots(&handler, &target_ctx).await;
9428 let ModuleControlResponseToModule::LiveRoots {
9429 roots,
9430 unknown_root_bindings,
9431 total_bindings,
9432 } = actual
9433 else {
9434 panic!("expected live roots")
9435 };
9436 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
9437 assert_eq!(unknown_root_bindings, 0);
9438 assert_eq!(
9439 roots.len(),
9440 2,
9441 "root-known arm must retain each canonical root"
9442 );
9443 assert_eq!(
9444 total_bindings,
9445 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9446 );
9447 let counts = roots
9448 .iter()
9449 .map(|root| (root.project_root.clone(), root.bound, root.pending))
9450 .collect::<Vec<_>>();
9451 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
9452 expected.sort_by(|a, b| a.0.cmp(&b.0));
9453 assert_eq!(
9454 counts, expected,
9455 "roots must sort by path and count pending separately"
9456 );
9457 handler
9458 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
9459 .await
9460 .unwrap();
9461 assert!(pending.await.unwrap().is_empty());
9462 }
9463
9464 #[tokio::test(start_paused = true)]
9465 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
9466 let forwarding = Arc::new(ForwardingTable::default());
9467 let handler =
9468 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
9469 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
9470 hello_via_sink(
9471 &handler,
9472 &target_ctx,
9473 &mut target_rx,
9474 hello_frame("target", PROTOCOL_VERSION, 1),
9475 )
9476 .await;
9477 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
9478 let pending = forwarding
9479 .begin_route_bind_relay_for_test(
9480 client_ctx.connection_id,
9481 client_ctx.egress.clone(),
9482 2,
9483 "target",
9484 )
9485 .unwrap();
9486 forwarding
9487 .complete_pending_relay(
9488 target_ctx.connection_id,
9489 pending.corr,
9490 RouteBindRelayOutcome::Accepted,
9491 )
9492 .unwrap();
9493 let actual = query_live_roots(&handler, &target_ctx).await;
9494 let ModuleControlResponseToModule::LiveRoots {
9495 roots,
9496 unknown_root_bindings,
9497 total_bindings,
9498 } = actual
9499 else {
9500 panic!("expected live roots")
9501 };
9502 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
9503 assert_eq!(
9504 unknown_root_bindings, 1,
9505 "unknown-root arm must not read as no bindings"
9506 );
9507 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
9508 assert_eq!(
9509 total_bindings,
9510 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9511 );
9512 }
9513
9514 #[tokio::test(start_paused = true)]
9522 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
9523 let forwarding = Arc::new(ForwardingTable::default());
9524 let handler =
9525 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
9526 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
9527 let replies = handler
9528 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
9529 .await
9530 .unwrap();
9531 let queued_by_hello = module_rx.len();
9532
9533 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
9534 let open_handler = handler.clone();
9535 let open = tokio::spawn(async move {
9536 open_handler
9537 .handle_control_frame(
9538 &client_ctx,
9539 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
9540 )
9541 .await
9542 .unwrap()
9543 });
9544 let mut spins = 0;
9546 while module_rx.len() == queued_by_hello {
9547 spins += 1;
9548 assert!(spins < 10_000, "route.open never queued a route.bind");
9549 tokio::task::yield_now().await;
9550 }
9551
9552 for reply in replies {
9554 module_ctx.egress.send(reply).await.unwrap();
9555 }
9556
9557 let first = module_rx.recv().await.unwrap().frame;
9558 assert_eq!(
9559 first.header.ty,
9560 FrameType::HelloAck,
9561 "the first frame a registering module reads must be its HELLO_ACK"
9562 );
9563 assert_eq!(first.header.corr, 7);
9564 let second = module_rx.recv().await.unwrap().frame;
9565 assert_eq!(second.header.ty, FrameType::Request);
9566 assert!(
9567 matches!(
9568 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
9569 ModuleControlRequest::RouteBind { .. }
9570 ),
9571 "the route.bind follows the ack"
9572 );
9573 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
9574
9575 handler
9576 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
9577 .await
9578 .unwrap();
9579 assert!(open.await.unwrap().is_empty());
9580 let _ = client_rx.recv().await.unwrap();
9581 }
9582
9583 #[tokio::test(start_paused = true)]
9584 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
9585 let handler = ControlHandler::with_forwarding(
9586 Arc::new(Registry::default()),
9587 Arc::new(ForwardingTable::default()),
9588 );
9589 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
9590 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
9591 hello_via_sink(
9592 &handler,
9593 &first_ctx,
9594 &mut first_rx,
9595 hello_frame("first", PROTOCOL_VERSION, 1),
9596 )
9597 .await;
9598 hello_via_sink(
9599 &handler,
9600 &second_ctx,
9601 &mut second_rx,
9602 hello_frame("second", PROTOCOL_VERSION, 2),
9603 )
9604 .await;
9605 let root = unique_project_root("second-only");
9606 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
9607 let cloned = handler.clone();
9608 let open = tokio::spawn(async move {
9609 cloned
9610 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
9611 .await
9612 .unwrap()
9613 });
9614 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
9615 .await
9616 .unwrap()
9617 .unwrap();
9618 let first = query_live_roots(&handler, &first_ctx).await;
9619 let second = query_live_roots(&handler, &second_ctx).await;
9620 assert!(
9621 matches!(
9622 first,
9623 ModuleControlResponseToModule::LiveRoots {
9624 total_bindings: 0,
9625 ..
9626 }
9627 ),
9628 "cross-module scope must not expose another module's roots"
9629 );
9630 assert!(
9631 matches!(
9632 second,
9633 ModuleControlResponseToModule::LiveRoots {
9634 total_bindings: 1,
9635 ..
9636 }
9637 ),
9638 "second module must see its pending route"
9639 );
9640 handler
9641 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
9642 .await
9643 .unwrap();
9644 assert!(open.await.unwrap().is_empty());
9645 }
9646
9647 #[tokio::test(start_paused = true)]
9648 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
9649 let handler = ControlHandler::with_forwarding(
9650 Arc::new(Registry::default()),
9651 Arc::new(ForwardingTable::default()),
9652 );
9653 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
9654 hello_via_sink(
9655 &handler,
9656 &target_ctx,
9657 &mut target_rx,
9658 hello_frame("target", PROTOCOL_VERSION, 1),
9659 )
9660 .await;
9661 let actual = query_live_roots(&handler, &target_ctx).await;
9662 let ModuleControlResponseToModule::LiveRoots {
9663 roots,
9664 unknown_root_bindings,
9665 total_bindings,
9666 } = actual
9667 else {
9668 panic!("expected live roots")
9669 };
9670 assert!(roots.is_empty());
9671 assert_eq!(unknown_root_bindings, 0);
9672 assert_eq!(total_bindings, 0);
9673 assert_eq!(
9674 total_bindings,
9675 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9676 );
9677 }
9678
9679 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
9686 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9687 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
9688 let text = std::fs::read_to_string(&path)
9689 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
9690 let vectors: Vec<(String, Value)> = text
9691 .lines()
9692 .filter(|line| !line.trim().is_empty())
9693 .map(|line| {
9694 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
9695 let id = entry["corpus_id"]
9696 .as_str()
9697 .expect("every vector carries a corpus_id")
9698 .to_string();
9699 (id, entry["package"].clone())
9700 })
9701 .collect();
9702 assert_eq!(
9706 vectors.len(),
9707 3,
9708 "vendored fed corpus changed size; re-sync from subc-federation"
9709 );
9710
9711 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
9726 let richest = vectors
9727 .iter()
9728 .filter_map(|(_, package)| package.as_object())
9729 .map(|object| {
9730 object
9731 .keys()
9732 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
9733 .count()
9734 })
9735 .max()
9736 .unwrap_or(0);
9737 assert!(
9738 richest >= 2,
9739 "vendored corpus no longer carries a package with unmodelled fields, \
9740 so the relay test can no longer distinguish a verbatim relay from a lossy one"
9741 );
9742
9743 vectors
9744 }
9745
9746 #[tokio::test]
9754 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
9755 for (corpus_id, package) in fed_admission_facts_vectors() {
9756 let registry = Arc::new(Registry::default());
9757 let forwarding = Arc::new(ForwardingTable::default());
9758 let supervisor = SupervisorHandle::new();
9759 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9760 let handler =
9761 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9762 .with_supervisor(supervisor)
9763 .with_admission_facts_config(
9764 Some("fed".to_string()),
9765 Some(vec!["target".to_string()]),
9766 );
9767
9768 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9769 hello_via_sink(
9770 &handler,
9771 &target_ctx,
9772 &mut target_rx,
9773 hello_frame("target", PROTOCOL_VERSION, 1),
9774 )
9775 .await;
9776
9777 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
9778 let route_handler = handler.clone();
9779 let expected = package.clone();
9780 let route_task = tokio::spawn(async move {
9781 route_handler
9782 .handle_control_frame(
9783 &client_ctx,
9784 route_open_frame_with_admission_facts(
9785 20,
9786 "target",
9787 unique_project_root("admission-facts"),
9788 Some(subc_control::ConsumerIdentity {
9789 module_id: "fed".to_string(),
9790 launch_nonce: "fed-nonce".to_string(),
9791 }),
9792 Some(package),
9793 ),
9794 )
9795 .await
9796 .unwrap()
9797 });
9798
9799 let bind_frame = target_rx.recv().await.unwrap();
9800 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9801 let ModuleControlRequest::RouteBind {
9802 admission_facts, ..
9803 } = bind
9804 else {
9805 panic!("{corpus_id}: expected route.bind")
9806 };
9807 assert_eq!(
9808 admission_facts,
9809 Some(expected),
9810 "{corpus_id}: relay must not add, drop or reshape any field"
9811 );
9812
9813 handler
9814 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9815 .await
9816 .unwrap();
9817 route_task.await.unwrap();
9818 }
9819 }
9820
9821 #[tokio::test]
9822 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
9823 let registry = Arc::new(Registry::default());
9824 let forwarding = Arc::new(ForwardingTable::default());
9825 let supervisor = SupervisorHandle::new();
9826 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9827 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
9828 let handler =
9829 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9830 .with_supervisor(supervisor)
9831 .with_admission_facts_config(
9832 Some("fed".to_string()),
9833 Some(vec!["target".to_string()]),
9834 );
9835
9836 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
9837 hello_via_sink(
9838 &handler,
9839 &target_ctx,
9840 &mut target_rx,
9841 hello_frame("target", PROTOCOL_VERSION, 1),
9842 )
9843 .await;
9844 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
9845 hello_via_sink(
9846 &handler,
9847 &other_ctx,
9848 &mut other_rx,
9849 hello_frame("other", PROTOCOL_VERSION, 2),
9850 )
9851 .await;
9852
9853 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
9854 let expected_facts = facts.clone();
9855 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
9856 let route_handler = handler.clone();
9857 let route_task = tokio::spawn(async move {
9858 route_handler
9859 .handle_control_frame(
9860 &client_ctx,
9861 route_open_frame_with_admission_facts(
9862 10,
9863 "target",
9864 unique_project_root("admission-facts"),
9865 Some(subc_control::ConsumerIdentity {
9866 module_id: "fed".to_string(),
9867 launch_nonce: "fed-nonce".to_string(),
9868 }),
9869 Some(facts.clone()),
9870 ),
9871 )
9872 .await
9873 .unwrap()
9874 });
9875 let bind_frame = target_rx.recv().await.unwrap();
9876 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9877 let ModuleControlRequest::RouteBind {
9878 admission_facts, ..
9879 } = bind
9880 else {
9881 panic!("expected route.bind")
9882 };
9883 assert_eq!(admission_facts, Some(expected_facts));
9884 handler
9885 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9886 .await
9887 .unwrap();
9888 assert!(route_task.await.unwrap().is_empty());
9889 assert!(matches!(
9890 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
9891 .unwrap(),
9892 ClientControlResponse::RouteOpen { .. }
9893 ));
9894
9895 let direct = handler
9896 .handle_control_frame(
9897 &route_ctx(ConnectionId::new(73)).0,
9898 route_open_frame_with_admission_facts(
9899 11,
9900 "target",
9901 unique_project_root("admission-facts"),
9902 None,
9903 Some(json!({"x": 1})),
9904 ),
9905 )
9906 .await
9907 .unwrap();
9908 assert_eq!(
9909 parse_error(&direct[0])["code"],
9910 "admission_facts_not_permitted"
9911 );
9912
9913 let different_reserved = handler
9914 .handle_control_frame(
9915 &route_ctx(ConnectionId::new(77)).0,
9916 route_open_frame_with_admission_facts(
9917 15,
9918 "target",
9919 unique_project_root("admission-facts"),
9920 Some(subc_control::ConsumerIdentity {
9921 module_id: "other".to_string(),
9922 launch_nonce: "other-nonce".to_string(),
9923 }),
9924 Some(json!({"x": 1})),
9925 ),
9926 )
9927 .await
9928 .unwrap();
9929 assert_eq!(
9930 parse_error(&different_reserved[0])["code"],
9931 "admission_facts_not_permitted"
9932 );
9933
9934 let other_target = handler
9935 .handle_control_frame(
9936 &route_ctx(ConnectionId::new(74)).0,
9937 route_open_frame_with_admission_facts(
9938 12,
9939 "other",
9940 unique_project_root("admission-facts"),
9941 Some(subc_control::ConsumerIdentity {
9942 module_id: "fed".to_string(),
9943 launch_nonce: "fed-nonce".to_string(),
9944 }),
9945 Some(json!({"x": 1})),
9946 ),
9947 )
9948 .await
9949 .unwrap();
9950 assert_eq!(
9951 parse_error(&other_target[0])["code"],
9952 "admission_facts_target_not_allowed"
9953 );
9954
9955 let nonexistent = handler
9956 .handle_control_frame(
9957 &route_ctx(ConnectionId::new(75)).0,
9958 route_open_frame_with_admission_facts(
9959 13,
9960 "missing",
9961 unique_project_root("admission-facts"),
9962 None,
9963 Some(json!({"x": 1})),
9964 ),
9965 )
9966 .await
9967 .unwrap();
9968 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
9969
9970 let described = handler
9971 .handle_control_frame(
9972 &route_ctx(ConnectionId::new(76)).0,
9973 Frame::build(
9974 FrameType::Request,
9975 control_flags(),
9976 0,
9977 0,
9978 14,
9979 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9980 )
9981 .unwrap(),
9982 )
9983 .await
9984 .unwrap();
9985 let ClientControlResponse::ServerDescribe { capabilities, .. } =
9986 serde_json::from_slice(&described[0].body).unwrap()
9987 else {
9988 panic!("expected server.describe response")
9989 };
9990 assert!(capabilities
9991 .iter()
9992 .any(|cap| cap == "admission_facts_relay_v1"));
9993 }
9994
9995 #[tokio::test]
9996 async fn admission_facts_without_configured_carrier_are_rejected() {
9997 let registry = Arc::new(Registry::default());
9998 let forwarding = Arc::new(ForwardingTable::default());
9999 let handler = ControlHandler::with_forwarding(registry, forwarding);
10000 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
10001 hello_via_sink(
10002 &handler,
10003 &target_ctx,
10004 &mut target_rx,
10005 hello_frame("target", PROTOCOL_VERSION, 1),
10006 )
10007 .await;
10008
10009 let responses = handler
10010 .handle_control_frame(
10011 &route_ctx(ConnectionId::new(79)).0,
10012 route_open_frame_with_admission_facts(
10013 16,
10014 "target",
10015 unique_project_root("admission-facts"),
10016 None,
10017 Some(json!({"x": 1})),
10018 ),
10019 )
10020 .await
10021 .unwrap();
10022 assert_eq!(
10023 parse_error(&responses[0])["code"],
10024 "admission_facts_not_permitted"
10025 );
10026 }
10027
10028 #[tokio::test]
10029 async fn route_open_relays_consumer_capabilities_verbatim() {
10030 let registry = Arc::new(Registry::default());
10031 let forwarding = Arc::new(ForwardingTable::default());
10032 let handler =
10033 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10034 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
10035 hello_via_sink(
10036 &handler,
10037 &module_ctx,
10038 &mut module_rx,
10039 hello_frame("aft", PROTOCOL_VERSION, 7),
10040 )
10041 .await;
10042
10043 let expected = vec!["elicitation".to_string(), "roots".to_string()];
10044 let expected_for_request = expected.clone();
10045 let project_root = unique_project_root("consumer-capabilities-present");
10046 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
10047 let route_handler = handler.clone();
10048 let route_task = tokio::spawn(async move {
10049 route_handler
10050 .handle_control_frame(
10051 &client_ctx,
10052 route_open_frame_with_consumer_capabilities(
10053 401,
10054 "aft",
10055 project_root,
10056 Some(expected_for_request),
10057 ),
10058 )
10059 .await
10060 .unwrap()
10061 });
10062 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10063 .await
10064 .unwrap()
10065 .unwrap();
10066 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10067 let ModuleControlRequest::RouteBind {
10068 consumer_capabilities,
10069 ..
10070 } = bind
10071 else {
10072 panic!("expected route.bind request, got {bind:?}");
10073 };
10074 assert_eq!(consumer_capabilities, Some(expected.clone()));
10075
10076 handler
10077 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10078 .await
10079 .unwrap();
10080 let route_response = route_task.await.unwrap();
10081 assert!(route_response.is_empty());
10082 let published = client_rx.recv().await.unwrap();
10083 assert!(matches!(
10084 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10085 ClientControlResponse::RouteOpen { .. }
10086 ));
10087 }
10088
10089 #[tokio::test]
10090 async fn route_open_without_consumer_capabilities_relays_none() {
10091 let registry = Arc::new(Registry::default());
10092 let forwarding = Arc::new(ForwardingTable::default());
10093 let handler =
10094 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10095 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
10096 hello_via_sink(
10097 &handler,
10098 &module_ctx,
10099 &mut module_rx,
10100 hello_frame("aft", PROTOCOL_VERSION, 7),
10101 )
10102 .await;
10103
10104 let project_root = unique_project_root("consumer-capabilities-absent");
10105 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
10106 let route_handler = handler.clone();
10107 let route_task = tokio::spawn(async move {
10108 route_handler
10109 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
10110 .await
10111 .unwrap()
10112 });
10113 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10114 .await
10115 .unwrap()
10116 .unwrap();
10117 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10118 let ModuleControlRequest::RouteBind {
10119 consumer_capabilities,
10120 ..
10121 } = bind
10122 else {
10123 panic!("expected route.bind request, got {bind:?}");
10124 };
10125 assert_eq!(consumer_capabilities, None);
10126
10127 handler
10128 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10129 .await
10130 .unwrap();
10131 let route_response = route_task.await.unwrap();
10132 assert!(route_response.is_empty());
10133 let published = client_rx.recv().await.unwrap();
10134 assert!(matches!(
10135 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10136 ClientControlResponse::RouteOpen { .. }
10137 ));
10138 }
10139
10140 async fn bind_role_versions_for(
10144 sent: Option<BTreeMap<String, String>>,
10145 connection: u64,
10146 ) -> Option<BTreeMap<String, String>> {
10147 let registry = Arc::new(Registry::default());
10148 let forwarding = Arc::new(ForwardingTable::default());
10149 let handler =
10150 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10151 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
10152 hello_via_sink(
10153 &handler,
10154 &module_ctx,
10155 &mut module_rx,
10156 hello_frame("aft", PROTOCOL_VERSION, 7),
10157 )
10158 .await;
10159 let project_root = unique_project_root("role-versions");
10160 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
10161 let route_handler = handler.clone();
10162 let route_task = tokio::spawn(async move {
10163 route_handler
10164 .handle_control_frame(
10165 &client_ctx,
10166 route_open_frame_with_role_versions(403, "aft", project_root, sent),
10167 )
10168 .await
10169 .unwrap()
10170 });
10171 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10172 .await
10173 .expect("a well-formed route.open reaches the module as a bind")
10174 .unwrap();
10175 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10176 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
10177 panic!("expected route.bind request, got {bind:?}");
10178 };
10179 handler
10180 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10181 .await
10182 .unwrap();
10183 assert!(route_task.await.unwrap().is_empty());
10184 let published = client_rx.recv().await.unwrap();
10185 assert!(matches!(
10186 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10187 ClientControlResponse::RouteOpen { .. }
10188 ));
10189 role_versions
10190 }
10191
10192 #[tokio::test]
10193 async fn route_open_relays_role_versions_verbatim() {
10194 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
10195 assert_eq!(
10196 bind_role_versions_for(Some(sent.clone()), 141).await,
10197 Some(sent)
10198 );
10199 }
10200
10201 #[tokio::test]
10204 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
10205 assert_eq!(bind_role_versions_for(None, 143).await, None);
10206 assert_eq!(
10207 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
10208 None
10209 );
10210 }
10211
10212 #[tokio::test]
10213 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
10214 let registry = Arc::new(Registry::default());
10215 let forwarding = Arc::new(ForwardingTable::default());
10216 let handler =
10217 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10218 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
10219 hello_via_sink(
10220 &handler,
10221 &module_ctx,
10222 &mut module_rx,
10223 hello_frame("aft", PROTOCOL_VERSION, 7),
10224 )
10225 .await;
10226
10227 let nine: BTreeMap<String, String> = (0..9)
10228 .map(|index| (format!("role-{index}"), "v1".to_string()))
10229 .collect();
10230 for (label, malformed) in [
10231 (
10232 "invalid role name",
10233 role_versions(&[("Tool_Provider", "v1")]),
10234 ),
10235 ("invalid version", role_versions(&[("tool-provider", "v0")])),
10236 ("nine entries", nine),
10237 ] {
10238 let responses = tokio::time::timeout(
10241 Duration::from_secs(1),
10242 handler.handle_control_frame(
10243 &route_ctx(ConnectionId::new(148)).0,
10244 route_open_frame_with_role_versions(
10245 404,
10246 "aft",
10247 unique_project_root("role-versions-malformed"),
10248 Some(malformed),
10249 ),
10250 ),
10251 )
10252 .await
10253 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
10254 .unwrap();
10255 assert_eq!(responses.len(), 1, "{label}");
10256 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
10257 let error = parse_error(&responses[0]);
10258 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
10259 assert_eq!(
10260 error["detail"]["field"], "role_versions",
10261 "{label}: {error}"
10262 );
10263 assert!(
10264 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
10265 "{label}: a malformed declaration is terminal"
10266 );
10267 assert!(
10268 module_rx.try_recv().is_err(),
10269 "{label}: the module must never see a bind"
10270 );
10271 }
10272 }
10273
10274 #[tokio::test]
10278 async fn route_role_versions_capability_is_advertised() {
10279 let handler = ControlHandler::new(Arc::new(Registry::default()));
10280 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
10281 let ack = hello_via_sink(
10282 &handler,
10283 &ctx,
10284 &mut rx,
10285 hello_frame("m", PROTOCOL_VERSION, 1),
10286 )
10287 .await;
10288 let ack = parse_ack(&ack);
10289 assert!(
10290 ack.subc_capabilities
10291 .iter()
10292 .any(|c| c == "route-role-versions/v1"),
10293 "{:?}",
10294 ack.subc_capabilities
10295 );
10296
10297 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
10298 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
10299 let reply = handler
10300 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
10301 .await
10302 .unwrap()
10303 .pop()
10304 .unwrap();
10305 let ClientControlResponse::ServerDescribe { capabilities, .. } =
10306 serde_json::from_slice(&reply.body).unwrap()
10307 else {
10308 panic!("not a server.describe reply");
10309 };
10310 assert!(
10311 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
10312 "{capabilities:?}"
10313 );
10314 }
10315
10316 #[tokio::test]
10320 async fn catalog_events_capability_is_advertised() {
10321 let handler = ControlHandler::new(Arc::new(Registry::default()));
10322 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
10323 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
10324 let reply = handler
10325 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
10326 .await
10327 .unwrap()
10328 .pop()
10329 .unwrap();
10330 let ClientControlResponse::ServerDescribe { capabilities, .. } =
10331 serde_json::from_slice(&reply.body).unwrap()
10332 else {
10333 panic!("not a server.describe reply");
10334 };
10335 assert!(
10336 capabilities
10337 .iter()
10338 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
10339 "{capabilities:?}"
10340 );
10341 }
10342
10343 #[tokio::test]
10344 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
10345 let registry = Arc::new(Registry::default());
10346 let forwarding = Arc::new(ForwardingTable::default());
10347 let handler =
10348 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10349 .with_health_probe_timeout(Duration::from_secs(5));
10350 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
10351 hello_via_sink(
10352 &handler,
10353 &module_ctx,
10354 &mut module_rx,
10355 non_routable_hello_frame_with_control_ops(
10356 "mcp",
10357 300,
10358 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10359 ),
10360 )
10361 .await;
10362 assert!(registry
10363 .get_module("mcp")
10364 .unwrap()
10365 .unwrap()
10366 .manifest
10367 .provides
10368 .is_empty());
10369
10370 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
10371 let route_response = handler
10372 .handle_control_frame(
10373 &route_client_ctx,
10374 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
10375 )
10376 .await
10377 .unwrap();
10378 assert_eq!(route_response[0].header.ty, FrameType::Error);
10379 assert_eq!(
10380 parse_error(&route_response[0])["code"],
10381 "target_unavailable"
10382 );
10383 assert!(parse_error(&route_response[0])["message"]
10384 .as_str()
10385 .unwrap()
10386 .contains("does not provide the requested target"));
10387 assert!(module_rx.try_recv().is_err());
10388
10389 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
10390 let health_handler = handler.clone();
10391 let health_task = tokio::spawn(async move {
10392 health_handler
10393 .handle_control_frame(
10394 &health_client_ctx,
10395 supervisor_health_probe_frame(302, "mcp"),
10396 )
10397 .await
10398 .unwrap()
10399 });
10400 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10401 .await
10402 .unwrap()
10403 .unwrap();
10404 assert_eq!(
10405 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10406 ModuleControlRequest::HealthCheck {}
10407 );
10408 handler
10409 .handle_control_frame(
10410 &module_ctx,
10411 health_response(health_frame.header.corr, HealthStatus::Ok),
10412 )
10413 .await
10414 .unwrap();
10415 let health_response = health_task.await.unwrap();
10416 assert_eq!(health_response[0].header.ty, FrameType::Response);
10417 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10418 ClientControlResponse::SupervisorHealthProbe {
10419 module_id, status, ..
10420 } => {
10421 assert_eq!(module_id, "mcp");
10422 assert_eq!(status, HealthStatus::Ok);
10423 }
10424 other => panic!("unexpected health response: {other:?}"),
10425 }
10426
10427 forwarding
10432 .cleanup_connection(module_ctx.connection_id)
10433 .unwrap();
10434 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
10435 let cleanup_response = tokio::time::timeout(
10436 Duration::from_millis(200),
10437 handler.handle_control_frame(
10438 &cleanup_probe_ctx,
10439 supervisor_health_probe_frame(303, "mcp"),
10440 ),
10441 )
10442 .await
10443 .expect("probe should fail immediately when the control lane is gone")
10444 .unwrap();
10445 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
10446 assert_eq!(
10447 parse_error(&cleanup_response[0])["code"],
10448 "target_unavailable"
10449 );
10450 assert!(parse_error(&cleanup_response[0])["message"]
10451 .as_str()
10452 .unwrap()
10453 .contains("no module connection"));
10454
10455 handler
10456 .cleanup_connection(module_ctx.connection_id)
10457 .unwrap();
10458 }
10459
10460 #[tokio::test]
10461 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
10462 let registry = Arc::new(Registry::default());
10463 let supervisor_handle = SupervisorHandle::new();
10464 let supervisor =
10465 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
10466 .with_handle(supervisor_handle.clone())
10467 .with_connection_file_path(
10468 std::env::temp_dir()
10469 .join(format!("subc-route-open-warming-{}", std::process::id())),
10470 );
10471 let module = supervisor
10472 .supervise_configured(
10473 ModuleSpec {
10474 module_id: "warming".to_string(),
10475 program: fake_aft_stub_path(),
10476 args: Vec::new(),
10477 env: Vec::new(),
10478 reserved: false,
10479 reserved_prefixes: Vec::new(),
10480 protocol: ModuleProtocol::Subc,
10481 overlap: Default::default(),
10482 },
10483 true,
10484 )
10485 .unwrap();
10486 assert_eq!(module.state().unwrap(), ModuleState::Running);
10487
10488 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10489 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
10490 let response = handler
10491 .handle_control_frame(
10492 &ctx,
10493 route_open_frame(304, "warming", unique_project_root("warming")),
10494 )
10495 .await
10496 .unwrap();
10497 module.stop().await.unwrap();
10498
10499 assert_eq!(response[0].header.ty, FrameType::Error);
10500 let error = parse_error(&response[0]);
10501 assert_eq!(error["code"], "module_warming");
10502 assert!(error["message"]
10503 .as_str()
10504 .unwrap()
10505 .contains("state=running, enabled=true, live=false"));
10506 }
10507
10508 #[test]
10509 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
10510 let handler = ControlHandler::new(Arc::new(Registry::default()));
10511 let capture = EventCapture::default();
10512 let _subscriber =
10513 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10514 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
10515 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
10516 let pending = (0..limit).collect::<Vec<_>>();
10517 let response = handler
10518 .route_open_capacity_refusal(
10519 &ctx,
10520 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
10521 "busy",
10522 pending.len(),
10523 limit,
10524 )
10525 .unwrap();
10526 assert_eq!(parse_error(&response)["code"], "target_unavailable");
10527 let event = capture
10528 .events()
10529 .into_iter()
10530 .find(|event| {
10531 event.target == "control"
10532 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
10533 })
10534 .expect("connection admission refusal event");
10535 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
10536 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
10537 }
10538
10539 #[test]
10540 fn route_open_target_cap_logs_admission_reason_and_capacity() {
10541 let handler = ControlHandler::new(Arc::new(Registry::default()));
10542 let capture = EventCapture::default();
10543 let _subscriber =
10544 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10545 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
10546 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
10547 let guards = (0..limit)
10548 .map(|_| {
10549 handler
10550 .route_bind_concurrency
10551 .try_admit("busy", limit)
10552 .unwrap()
10553 })
10554 .collect::<Vec<_>>();
10555 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
10556 Err(in_flight) => in_flight,
10557 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
10558 };
10559 let response = handler
10560 .route_open_target_capacity_refusal(
10561 &ctx,
10562 &route_open_frame(397, "busy", unique_project_root("target-cap")),
10563 "busy",
10564 in_flight,
10565 )
10566 .unwrap();
10567 assert_eq!(parse_error(&response)["code"], "target_unavailable");
10568 let event = capture
10569 .events()
10570 .into_iter()
10571 .find(|event| {
10572 event.target == "control"
10573 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
10574 })
10575 .expect("target admission refusal event");
10576 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
10577 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
10578 drop(guards);
10579 }
10580
10581 #[tokio::test(flavor = "current_thread")]
10582 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
10583 let handler = ControlHandler::new(Arc::new(Registry::default()));
10584 let capture = EventCapture::default();
10585 let _subscriber =
10586 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10587 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
10588 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
10589
10590 let _ = handler
10591 .handle_client_control_request(
10592 &ctx,
10593 frame,
10594 ClientControlRequest::SupervisorSetEnabled {
10595 module_id: "broca".to_string(),
10596 enabled: false,
10597 },
10598 )
10599 .await
10600 .unwrap();
10601
10602 let events = capture
10603 .events()
10604 .into_iter()
10605 .filter(|event| {
10606 event.target == "control"
10607 && event.fields.get("message").map(String::as_str)
10608 == Some("supervisor request received")
10609 })
10610 .collect::<Vec<_>>();
10611 assert_eq!(events.len(), 1, "one supervisor request log line");
10612 let event = &events[0];
10613 assert_eq!(event.level, tracing::Level::INFO);
10614 assert_eq!(
10615 event.fields.get("op"),
10616 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
10617 );
10618 assert_eq!(
10619 event.fields.get("module_id"),
10620 Some(&"\"broca\"".to_string())
10621 );
10622 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
10623 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
10624 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
10625 }
10626
10627 #[tokio::test(flavor = "current_thread")]
10628 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
10629 let registry = Arc::new(Registry::default());
10630 let module_connection = ConnectionId::new(303);
10631 registry
10632 .register_with_control_ops(
10633 manifest("aft", PROTOCOL_VERSION),
10634 PROTOCOL_VERSION,
10635 module_connection,
10636 module_baseline_control_ops(),
10637 )
10638 .unwrap();
10639 let handler = ControlHandler::new(registry);
10640 let capture = EventCapture::default();
10641 let _subscriber =
10642 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10643 let (ctx, _rx) = route_ctx(module_connection);
10644 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
10645
10646 let _ = handler
10647 .handle_client_control_request(
10648 &ctx,
10649 frame,
10650 ClientControlRequest::SupervisorSetEnabled {
10651 module_id: "broca".to_string(),
10652 enabled: false,
10653 },
10654 )
10655 .await
10656 .unwrap();
10657
10658 let callers = capture
10659 .events()
10660 .into_iter()
10661 .filter(|event| {
10662 event.fields.get("message").map(String::as_str)
10663 == Some("supervisor request received")
10664 })
10665 .map(|event| event.fields.get("caller").cloned())
10666 .collect::<Vec<_>>();
10667 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
10668 }
10669
10670 #[tokio::test(flavor = "current_thread")]
10671 async fn supervisor_list_does_not_log_request_received() {
10672 let handler = ControlHandler::new(Arc::new(Registry::default()));
10673 let capture = EventCapture::default();
10674 let _subscriber =
10675 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10676 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
10677 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
10678
10679 handler
10680 .handle_client_control_request(&ctx, frame, ClientControlRequest::SupervisorList {})
10681 .await
10682 .unwrap();
10683
10684 assert!(capture.events().into_iter().all(|event| {
10685 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
10686 }));
10687 }
10688
10689 #[tokio::test(flavor = "current_thread")]
10690 async fn supervisor_rescan_preview_does_not_log_request_received() {
10691 let handler = ControlHandler::new(Arc::new(Registry::default()));
10692 let capture = EventCapture::default();
10693 let _subscriber =
10694 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10695 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
10696 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
10697
10698 handler
10699 .handle_client_control_request(
10700 &ctx,
10701 frame,
10702 ClientControlRequest::SupervisorRescan { preview: true },
10703 )
10704 .await
10705 .unwrap();
10706
10707 assert!(capture.events().into_iter().all(|event| {
10708 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
10709 }));
10710 }
10711
10712 #[tokio::test]
10717 async fn route_open_refusal_names_the_check_that_refused() {
10718 let handler = ControlHandler::new(Arc::new(Registry::default()));
10719 let capture = EventCapture::default();
10720 let _subscriber =
10721 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10722 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
10723 let response = handler
10724 .handle_control_frame(
10725 &ctx,
10726 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
10727 )
10728 .await
10729 .unwrap();
10730
10731 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10732 let event = capture
10733 .events()
10734 .into_iter()
10735 .find(|event| {
10736 event.target == "control"
10737 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
10738 })
10739 .expect("route.open refusal event");
10740 assert_eq!(
10741 event.fields.get("reason"),
10742 Some(&"\"not_registered\"".to_string())
10743 );
10744 }
10745
10746 #[tokio::test]
10747 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
10748 let registry = Arc::new(Registry::default());
10749 let supervisor_handle = SupervisorHandle::new();
10750 let supervisor =
10751 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
10752 .with_handle(supervisor_handle.clone())
10753 .with_connection_file_path(std::env::temp_dir().join(format!(
10754 "subc-route-open-refusal-info-{}",
10755 std::process::id()
10756 )));
10757 let module = supervisor
10758 .supervise_configured(
10759 ModuleSpec {
10760 module_id: "warming".to_string(),
10761 program: fake_aft_stub_path(),
10762 args: Vec::new(),
10763 env: Vec::new(),
10764 reserved: false,
10765 reserved_prefixes: Vec::new(),
10766 protocol: ModuleProtocol::Subc,
10767 overlap: Default::default(),
10768 },
10769 true,
10770 )
10771 .unwrap();
10772 assert_eq!(module.state().unwrap(), ModuleState::Running);
10773
10774 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10775 assert!(handler
10776 .counters()
10777 .snapshot()
10778 .get("route_open_refused_by_code")
10779 .is_none());
10780 let capture = EventCapture::default();
10781 let _subscriber =
10782 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10783 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
10784 let response = handler
10785 .handle_control_frame(
10786 &ctx,
10787 route_open_frame(394, "warming", unique_project_root("refusal-info")),
10788 )
10789 .await
10790 .unwrap();
10791 module.stop().await.unwrap();
10792
10793 assert_eq!(parse_error(&response[0])["code"], "module_warming");
10794 let event = capture
10795 .events()
10796 .into_iter()
10797 .find(|event| {
10798 event.target == "control"
10799 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
10800 })
10801 .expect("route.open refusal event");
10802 assert_eq!(
10803 event.fields.get("module_id"),
10804 Some(&"\"warming\"".to_string())
10805 );
10806 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
10807 assert_eq!(
10808 event.fields.get("reason"),
10809 Some(&"\"supervised_not_registered\"".to_string())
10810 );
10811 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
10812 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
10813 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
10814 assert_eq!(
10815 handler.counters().snapshot()["route_open_refused_by_code"],
10816 json!({ "module_warming": 1 })
10817 );
10818 }
10819
10820 const OUTAGE_START: &str = "route.open refusing module: not serving";
10821 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
10822
10823 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
10824 capture
10825 .events()
10826 .into_iter()
10827 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
10828 .collect()
10829 }
10830
10831 fn supervise_stub(
10832 registry: &Arc<Registry>,
10833 module_id: &str,
10834 enabled: bool,
10835 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
10836 let supervisor_handle = SupervisorHandle::new();
10837 let supervisor =
10838 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
10839 .with_handle(supervisor_handle.clone())
10840 .with_connection_file_path(std::env::temp_dir().join(format!(
10841 "subc-route-outage-{module_id}-{}",
10842 std::process::id()
10843 )));
10844 let module = supervisor
10845 .supervise_configured(
10846 ModuleSpec {
10847 module_id: module_id.to_string(),
10848 program: fake_aft_stub_path(),
10849 args: Vec::new(),
10850 env: Vec::new(),
10851 reserved: false,
10852 reserved_prefixes: Vec::new(),
10853 protocol: ModuleProtocol::Subc,
10854 overlap: Default::default(),
10855 },
10856 enabled,
10857 )
10858 .unwrap();
10859 (supervisor_handle, module)
10860 }
10861
10862 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
10863 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
10864 module_id: module_id.to_string(),
10865 drain_timeout_ms: Some(50),
10866 })
10867 .unwrap();
10868 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
10869 }
10870
10871 #[test]
10875 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
10876 let registry = Arc::new(Registry::default());
10877 let forwarding = Arc::new(ForwardingTable::default());
10878 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10879 let second = ControlHandler::with_forwarding(registry, forwarding);
10880 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
10881 }
10882
10883 #[tokio::test(flavor = "current_thread")]
10887 async fn route_open_unknown_module_refusals_add_no_outage_state() {
10888 let handler = ControlHandler::new(Arc::new(Registry::default()));
10889 let capture = EventCapture::default();
10890 let _subscriber =
10891 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10892 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
10893 for corr in 0..8 {
10894 let response = handler
10895 .handle_control_frame(
10896 &ctx,
10897 route_open_frame(
10898 380 + corr,
10899 &format!("nobody-{corr}"),
10900 unique_project_root("outage-unknown"),
10901 ),
10902 )
10903 .await
10904 .unwrap();
10905 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10906 }
10907
10908 assert_eq!(handler.route_outages.tracked_module_count(), 0);
10909 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
10910 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
10911 }
10912
10913 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10918 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
10919 let registry = Arc::new(Registry::default());
10920 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
10921 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10922 let capture = EventCapture::default();
10923 let _subscriber =
10924 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10925 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
10926 handler.route_outages.record_accepted("outage-restart");
10929
10930 let response = handler
10931 .handle_control_frame(
10932 &ctx,
10933 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
10934 )
10935 .await
10936 .unwrap();
10937 assert_eq!(response[0].header.ty, FrameType::Error);
10938 let starts = outage_lines(&capture, OUTAGE_START);
10939 assert_eq!(starts.len(), 1, "{starts:?}");
10940 assert_eq!(starts[0].level, tracing::Level::WARN);
10941 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
10942 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
10943 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
10944 handler.route_outages.record_accepted("outage-restart");
10945 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
10946
10947 let restart = handler
10948 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
10949 .await
10950 .unwrap();
10951 assert_eq!(
10952 restart[0].header.ty,
10953 FrameType::Response,
10954 "{:?}",
10955 parse_error(&restart[0])
10956 );
10957 handler
10958 .handle_control_frame(
10959 &ctx,
10960 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
10961 )
10962 .await
10963 .unwrap();
10964 module.stop().await.unwrap();
10965
10966 let starts = outage_lines(&capture, OUTAGE_START);
10967 assert_eq!(starts.len(), 2, "{starts:?}");
10968 assert_eq!(starts[1].level, tracing::Level::INFO);
10969 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
10970 }
10971
10972 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10976 async fn failed_operator_restart_leaves_no_operator_mark() {
10977 let registry = Arc::new(Registry::default());
10978 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
10979 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10980 let capture = EventCapture::default();
10981 let _subscriber =
10982 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10983 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
10984 handler.route_outages.record_accepted("outage-disabled");
10985
10986 let restart = handler
10987 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
10988 .await
10989 .unwrap();
10990 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
10991 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
10992
10993 handler
10994 .handle_control_frame(
10995 &ctx,
10996 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
10997 )
10998 .await
10999 .unwrap();
11000 let starts = outage_lines(&capture, OUTAGE_START);
11001 assert_eq!(starts.len(), 1, "{starts:?}");
11002 assert_eq!(starts[0].level, tracing::Level::WARN);
11003 }
11004
11005 #[tokio::test(flavor = "current_thread")]
11006 async fn route_open_unknown_module_escapes_target_module_id() {
11007 let handler = ControlHandler::new(Arc::new(Registry::default()));
11008 let capture = EventCapture::default();
11009 let _subscriber =
11010 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11011 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
11012 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
11013 let response = handler
11014 .handle_control_frame(
11015 &ctx,
11016 route_open_frame(
11017 395,
11018 hostile_module_id,
11019 unique_project_root("hostile-target-module-id"),
11020 ),
11021 )
11022 .await
11023 .unwrap();
11024
11025 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11026 let event = capture
11027 .events()
11028 .into_iter()
11029 .find(|event| {
11030 event.target == "control"
11031 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
11032 })
11033 .expect("route.open unknown-module refusal event");
11034 let logged = event.fields.get("module_id").expect("module_id field");
11035 assert!(!logged.bytes().any(|byte| byte < 0x20));
11036 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
11037 }
11038
11039 #[tokio::test(flavor = "current_thread")]
11040 async fn route_open_module_rejection_uses_daemon_counter_key() {
11041 let registry = Arc::new(Registry::default());
11042 let forwarding = Arc::new(ForwardingTable::default());
11043 let handler =
11044 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11045 let module_connection = ConnectionId::new(95);
11046 let (module_ctx, mut module_rx) = route_ctx(module_connection);
11047 hello_via_sink(
11048 &handler,
11049 &module_ctx,
11050 &mut module_rx,
11051 hello_frame("aft", PROTOCOL_VERSION, 395),
11052 )
11053 .await;
11054
11055 let client_connection = ConnectionId::new(96);
11056 let (client_ctx, _client_rx) = route_ctx(client_connection);
11057 let capture = EventCapture::default();
11058 let _subscriber =
11059 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11060 let (route_task, bind) = relay_route_open(
11061 &handler,
11062 client_connection,
11063 &client_ctx.egress,
11064 &mut module_rx,
11065 396,
11066 "aft",
11067 "hostile-module-code",
11068 )
11069 .await;
11070 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
11071 let rejection = Frame::build(
11072 FrameType::Error,
11073 control_flags(),
11074 0,
11075 0,
11076 bind.header.corr,
11077 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
11078 )
11079 .unwrap();
11080 handler
11081 .handle_control_frame(&module_ctx, rejection)
11082 .await
11083 .unwrap();
11084
11085 let response = route_task.await.unwrap();
11086 assert_eq!(parse_error(&response[0])["code"], hostile_code);
11087 let counters = handler.counters().snapshot();
11088 assert_eq!(
11089 counters["route_open_refused_by_code"],
11090 json!({ "module_rejected": 1 })
11091 );
11092 assert!(counters["route_open_refused_by_code"]
11093 .get(hostile_code)
11094 .is_none());
11095
11096 let event = capture
11097 .events()
11098 .into_iter()
11099 .find(|event| {
11100 event.target == "control"
11101 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
11102 })
11103 .expect("route.open module-rejection refusal event");
11104 let logged = event.fields.get("module_code").expect("module_code field");
11105 assert!(!logged.bytes().any(|byte| byte < 0x20));
11106 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
11107 }
11108
11109 #[tokio::test]
11110 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
11111 let registry = Arc::new(Registry::default());
11112 let supervisor_handle = SupervisorHandle::new();
11113 let missing_program = std::env::temp_dir().join(format!(
11114 "subc-route-open-missing-program-{}",
11115 std::process::id()
11116 ));
11117 let supervisor =
11118 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11119 .with_handle(supervisor_handle.clone());
11120 let module = supervisor
11121 .supervise_configured(
11122 ModuleSpec {
11123 module_id: "failed".to_string(),
11124 program: missing_program,
11125 args: Vec::new(),
11126 env: Vec::new(),
11127 reserved: false,
11128 reserved_prefixes: Vec::new(),
11129 protocol: ModuleProtocol::Subc,
11130 overlap: Default::default(),
11131 },
11132 true,
11133 )
11134 .unwrap();
11135 assert_eq!(module.state().unwrap(), ModuleState::Failed);
11136
11137 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11138 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
11139 let response = handler
11140 .handle_control_frame(
11141 &ctx,
11142 route_open_frame(305, "failed", unique_project_root("failed")),
11143 )
11144 .await
11145 .unwrap();
11146
11147 assert_eq!(response[0].header.ty, FrameType::Error);
11148 let error = parse_error(&response[0]);
11149 assert_eq!(error["code"], "target_unavailable");
11150 assert!(error["message"]
11151 .as_str()
11152 .unwrap()
11153 .contains("state=failed, enabled=true, live=false"));
11154 }
11155
11156 #[tokio::test]
11157 async fn route_open_role_mismatch_remains_target_unavailable() {
11158 let registry = Arc::new(Registry::default());
11159 let handler = ControlHandler::new(Arc::clone(®istry));
11160 handler
11161 .handle_control(
11162 ConnectionId::new(41),
11163 non_routable_hello_frame_with_control_ops("health-only", 306, None),
11164 )
11165 .unwrap();
11166
11167 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
11168 let response = handler
11169 .handle_control_frame(
11170 &ctx,
11171 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
11172 )
11173 .await
11174 .unwrap();
11175
11176 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11177 assert!(parse_error(&response[0])["message"]
11178 .as_str()
11179 .unwrap()
11180 .contains("does not provide the requested target"));
11181 }
11182
11183 #[tokio::test]
11184 async fn route_open_inactive_registration_remains_target_unavailable() {
11185 let registry = Arc::new(Registry::default());
11186 let handler = ControlHandler::new(Arc::clone(®istry));
11187 handler
11188 .handle_control(
11189 ConnectionId::new(43),
11190 hello_frame("inactive", PROTOCOL_VERSION, 308),
11191 )
11192 .unwrap();
11193 assert!(registry
11194 .set_module_state_for_test("inactive", ChannelState::Closed)
11195 .unwrap());
11196
11197 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
11198 let response = handler
11199 .handle_control_frame(
11200 &ctx,
11201 route_open_frame(309, "inactive", unique_project_root("inactive")),
11202 )
11203 .await
11204 .unwrap();
11205
11206 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11207 assert!(parse_error(&response[0])["message"]
11208 .as_str()
11209 .unwrap()
11210 .contains("is not active"));
11211 }
11212
11213 #[tokio::test]
11214 async fn late_health_reply_is_recorded_through_the_module_response_path() {
11215 let registry = Arc::new(Registry::default());
11216 let forwarding = Arc::new(ForwardingTable::default());
11217 let supervisor_handle = SupervisorHandle::new();
11218 let supervisor =
11219 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
11220 .with_forwarding(Arc::clone(&forwarding))
11221 .with_handle(supervisor_handle.clone());
11222 let module = supervisor
11223 .supervise_configured(
11224 crate::ModuleSpec {
11225 module_id: "late-health-response".to_string(),
11226 program: PathBuf::from("disabled-module"),
11227 args: Vec::new(),
11228 env: Vec::new(),
11229 reserved: false,
11230 reserved_prefixes: Vec::new(),
11231 protocol: ModuleProtocol::Subc,
11232 overlap: Default::default(),
11233 },
11234 false,
11235 )
11236 .unwrap();
11237 let handler =
11238 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11239 .with_supervisor(supervisor_handle);
11240 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
11241 handler
11242 .handle_control_frame(
11243 &module_ctx,
11244 hello_frame_with_control_ops(
11245 "late-health-response",
11246 PROTOCOL_VERSION,
11247 7,
11248 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11249 ),
11250 )
11251 .await
11252 .unwrap();
11253 let probe_started_at = Instant::now() - Duration::from_millis(80);
11254 let pending = forwarding
11255 .begin_health_probe_rpc_for(
11256 "late-health-response",
11257 MODULE_CONTROL_OP_HEALTH_CHECK,
11258 probe_started_at,
11259 Instant::now() - Duration::from_millis(1),
11260 )
11261 .unwrap();
11262 assert!(forwarding
11263 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
11264 .unwrap());
11265
11266 let responses = handler
11267 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
11268 .await
11269 .unwrap();
11270
11271 assert!(responses.is_empty());
11272 let health = module.status().unwrap().health;
11273 assert_eq!(health.late_answer_count, 1);
11274 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
11275 }
11276
11277 #[tokio::test]
11278 async fn health_probe_timeout_and_module_death_are_typed() {
11279 let registry = Arc::new(Registry::default());
11280 let forwarding = Arc::new(ForwardingTable::default());
11281 let handler =
11282 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11283 .with_health_probe_timeout(Duration::from_millis(50));
11284 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11285 hello_via_sink(
11286 &handler,
11287 &module_ctx,
11288 &mut module_rx,
11289 hello_frame_with_control_ops(
11290 "aft",
11291 PROTOCOL_VERSION,
11292 7,
11293 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11294 ),
11295 )
11296 .await;
11297
11298 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11299 let responses = handler
11300 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
11301 .await
11302 .unwrap();
11303 assert_eq!(responses[0].header.ty, FrameType::Error);
11304 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
11305 let _ = module_rx.try_recv();
11306
11307 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
11308 let health_handler = handler.clone();
11309 let death_task = tokio::spawn(async move {
11310 health_handler
11311 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
11312 .await
11313 .unwrap()
11314 });
11315 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11316 .await
11317 .unwrap()
11318 .unwrap();
11319 handler
11320 .cleanup_connection(module_ctx.connection_id)
11321 .unwrap();
11322 let responses = death_task.await.unwrap();
11323 assert_eq!(responses[0].header.ty, FrameType::Error);
11324 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
11325 }
11326
11327 #[test]
11328 fn hello_requires_exact_protocol_version() {
11329 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
11330 let registry = Arc::new(Registry::default());
11331 let handler = ControlHandler::new(Arc::clone(®istry));
11332 let responses = handler
11333 .handle_control(
11334 ConnectionId::new(connection),
11335 hello_frame("aft", offered, 9),
11336 )
11337 .unwrap();
11338
11339 assert_eq!(responses.len(), 1);
11340 assert_eq!(responses[0].header.ty, FrameType::Error);
11341 let error = parse_error(&responses[0]);
11342 assert_eq!(error["code"], "version_unsupported");
11343 assert!(registry.get_module("aft").unwrap().is_none());
11344 assert_eq!(registry.active_registration_count().unwrap(), 0);
11345 }
11346 }
11347
11348 #[test]
11349 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
11350 let registry = Arc::new(Registry::default());
11351 let forwarding = Arc::new(ForwardingTable::default());
11352 let handler =
11353 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11354 let module_connection = ConnectionId::new(301);
11355 let registration = registry
11356 .register_with_control_ops(
11357 manifest("aft-push", PROTOCOL_VERSION),
11358 PROTOCOL_VERSION,
11359 module_connection,
11360 module_baseline_control_ops(),
11361 )
11362 .unwrap();
11363 let (module_tx, _module_rx) = mpsc::channel(8);
11364 let endpoint = forwarding
11365 .register_module_connection(
11366 module_connection,
11367 "aft-push".to_string(),
11368 PROTOCOL_VERSION,
11369 manifest_concurrency(®istration.manifest),
11370 FrameSink::new(module_tx),
11371 )
11372 .unwrap();
11373
11374 let unknown = Frame::build(
11376 FrameType::Push,
11377 control_flags(),
11378 0,
11379 0,
11380 5,
11381 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
11382 )
11383 .unwrap();
11384 let out = handler.handle_status_update(endpoint, unknown).unwrap();
11385 assert!(
11386 out.is_empty(),
11387 "unknown push op must be ignored, got {out:?}"
11388 );
11389
11390 let malformed = Frame::build(
11392 FrameType::Push,
11393 control_flags(),
11394 0,
11395 0,
11396 6,
11397 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
11398 )
11399 .unwrap();
11400 let out = handler.handle_status_update(endpoint, malformed).unwrap();
11401 assert_eq!(out.len(), 1);
11402 assert_eq!(out[0].header.ty, FrameType::Error);
11403 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
11404 }
11405
11406 #[test]
11407 fn hello_rejected_when_connection_already_owns_client_routes() {
11408 let registry = Arc::new(Registry::default());
11409 let forwarding = Arc::new(ForwardingTable::default());
11410 let handler =
11411 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11412 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
11414 let client_connection = ConnectionId::new(202);
11415
11416 let responses = handler
11419 .handle_control(
11420 client_connection,
11421 hello_frame("aft-second", PROTOCOL_VERSION, 9),
11422 )
11423 .unwrap();
11424 assert_eq!(responses[0].header.ty, FrameType::Error);
11425 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
11426 assert!(registry.get_module("aft-second").unwrap().is_none());
11427 }
11428
11429 #[tokio::test]
11430 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
11431 let registry = Arc::new(Registry::default());
11432 let forwarding = Arc::new(ForwardingTable::default());
11433 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
11434 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
11435 hello_via_sink(
11436 &handler,
11437 &module_ctx,
11438 &mut module_rx,
11439 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
11440 )
11441 .await;
11442 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
11443 let pending = forwarding
11444 .begin_route_bind_relay_for_test(
11445 client_ctx.connection_id,
11446 client_ctx.egress.clone(),
11447 2,
11448 "alpha",
11449 )
11450 .unwrap();
11451 forwarding
11452 .complete_pending_relay(
11453 module_ctx.connection_id,
11454 pending.corr,
11455 RouteBindRelayOutcome::Accepted,
11456 )
11457 .unwrap();
11458 client_rx.try_recv().unwrap();
11459 for module_id in ["beta", "alpha"] {
11460 let replies = handler
11461 .handle_control_frame(
11462 &module_ctx,
11463 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
11464 )
11465 .await
11466 .unwrap();
11467 assert_eq!(replies.len(), 1, "second HELLO must be refused");
11468 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
11469 }
11470 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
11471 assert!(registry.get_module("beta").unwrap().is_none());
11472 assert!(matches!(
11473 forwarding
11474 .lookup_data_route(
11475 client_ctx.connection_id,
11476 pending.client_channel,
11477 pending.client_epoch,
11478 )
11479 .unwrap(),
11480 DataRoute::Client(DataRouteState::Bound(_))
11481 ));
11482 assert!(handler
11483 .hello_launch_nonces
11484 .lock()
11485 .unwrap()
11486 .presented(module_ctx.connection_id, Some("alpha-nonce")));
11487 assert!(module_rx.try_recv().is_err());
11488 }
11489
11490 #[test]
11491 fn reserved_module_hello_requires_matching_launch_nonce() {
11492 let registry = Arc::new(Registry::default());
11493 let supervisor = SupervisorHandle::new();
11494 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
11497 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11498
11499 let no_nonce = handler
11501 .handle_control(
11502 ConnectionId::new(1),
11503 hello_frame("vault", PROTOCOL_VERSION, 1),
11504 )
11505 .unwrap();
11506 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
11507 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
11508 assert!(registry.get_module("vault").unwrap().is_none());
11509
11510 let wrong = handler
11512 .handle_control(
11513 ConnectionId::new(2),
11514 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
11515 )
11516 .unwrap();
11517 assert_eq!(wrong[0].header.ty, FrameType::Error);
11518 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
11519 assert!(registry.get_module("vault").unwrap().is_none());
11520
11521 let ok = handler
11523 .handle_control(
11524 ConnectionId::new(3),
11525 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
11526 )
11527 .unwrap();
11528 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
11529 assert!(registry.get_module("vault").unwrap().is_some());
11530 }
11531
11532 #[test]
11533 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
11534 let registry = Arc::new(Registry::default());
11535 let supervisor = SupervisorHandle::new();
11536 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
11537 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
11538 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11539
11540 let squat = handler
11541 .handle_control(
11542 ConnectionId::new(1),
11543 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
11544 )
11545 .unwrap();
11546 assert_eq!(squat[0].header.ty, FrameType::Error);
11547 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
11548 assert!(parse_error(&squat[0])["message"]
11549 .as_str()
11550 .unwrap()
11551 .contains("fed:"));
11552
11553 let accepted_peer = handler
11554 .handle_control(
11555 ConnectionId::new(2),
11556 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
11557 )
11558 .unwrap();
11559 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
11560
11561 let accepted_short = handler
11562 .handle_control(
11563 ConnectionId::new(3),
11564 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
11565 )
11566 .unwrap();
11567 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
11568
11569 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
11570 let response = handler
11571 .handle_control(
11572 ConnectionId::new(conn),
11573 hello_frame(module_id, PROTOCOL_VERSION, conn),
11574 )
11575 .unwrap();
11576 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
11577 }
11578 }
11579
11580 #[test]
11581 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
11582 let registry = Arc::new(Registry::default());
11583 let supervisor = SupervisorHandle::new();
11584 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
11585 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
11586 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
11587 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11588
11589 let owner_nonce = handler
11590 .handle_control(
11591 ConnectionId::new(1),
11592 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
11593 )
11594 .unwrap();
11595 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
11596 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
11597 assert!(registry.get_module("fed:special").unwrap().is_none());
11598
11599 let exact_nonce = handler
11600 .handle_control(
11601 ConnectionId::new(2),
11602 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
11603 )
11604 .unwrap();
11605 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
11606 assert!(registry.get_module("fed:special").unwrap().is_some());
11607 }
11608
11609 #[test]
11610 fn non_reserved_module_ignores_launch_nonce() {
11611 let registry = Arc::new(Registry::default());
11612 let handler = ControlHandler::new(Arc::clone(®istry));
11615 let no_nonce = handler
11616 .handle_control(
11617 ConnectionId::new(1),
11618 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
11619 )
11620 .unwrap();
11621 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
11622 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
11623
11624 let echoed_nonce = handler
11625 .handle_control(
11626 ConnectionId::new(2),
11627 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
11628 )
11629 .unwrap();
11630 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
11631 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
11632 }
11633
11634 #[test]
11635 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
11636 let handler = ControlHandler::default();
11637 let conn = ConnectionId::new(1);
11638 let malformed = Frame::build(
11639 FrameType::Hello,
11640 control_flags(),
11641 0,
11642 0,
11643 3,
11644 b"{not json".to_vec(),
11645 )
11646 .unwrap();
11647
11648 let error = handler.handle_control(conn, malformed).unwrap();
11649 assert_eq!(error[0].header.ty, FrameType::Error);
11650 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
11651
11652 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
11653 let pong = handler.handle_control(conn, ping).unwrap();
11654 assert_eq!(pong[0].header.ty, FrameType::Pong);
11655 assert_eq!(pong[0].header.corr, 4);
11656 }
11657
11658 #[test]
11659 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
11660 let registry = Arc::new(Registry::default());
11661 let handler = ControlHandler::new(Arc::clone(®istry));
11662
11663 handler
11664 .handle_control(
11665 ConnectionId::new(1),
11666 hello_frame("aft", PROTOCOL_VERSION, 1),
11667 )
11668 .unwrap();
11669 let duplicate = handler
11670 .handle_control(
11671 ConnectionId::new(2),
11672 hello_frame("aft", PROTOCOL_VERSION, 2),
11673 )
11674 .unwrap();
11675
11676 assert_eq!(duplicate[0].header.ty, FrameType::Error);
11677 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11678 let registration = registry.get_module("aft").unwrap().unwrap();
11679 assert_eq!(registration.connection_id, ConnectionId::new(1));
11680 }
11681
11682 #[test]
11683 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
11684 let registry = Arc::new(Registry::default());
11685 let forwarding = Arc::new(ForwardingTable::default());
11686 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
11687 let handler =
11688 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11689 .with_process_liveness(process_liveness);
11690 let (ctx, route_channel, route_epoch) =
11691 bind_liveness_route(®istry, &forwarding, "aft-dead");
11692 let responses = handler
11693 .handle_route_poll(
11694 &ctx,
11695 route_poll_frame(41, PollKind::Liveness, route_channel),
11696 route_channel,
11697 route_epoch,
11698 PollKind::Liveness,
11699 )
11700 .unwrap();
11701
11702 assert_eq!(responses.len(), 1);
11703 assert_eq!(responses[0].header.ty, FrameType::Response);
11704 assert_route_poll_liveness(&responses[0], false);
11705 }
11706
11707 #[test]
11708 fn liveness_poll_without_process_source_uses_bound_route() {
11709 let registry = Arc::new(Registry::default());
11710 let forwarding = Arc::new(ForwardingTable::default());
11711 let handler =
11712 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11713 let (ctx, route_channel, route_epoch) =
11714 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
11715 let responses = handler
11716 .handle_route_poll(
11717 &ctx,
11718 route_poll_frame(42, PollKind::Liveness, route_channel),
11719 route_channel,
11720 route_epoch,
11721 PollKind::Liveness,
11722 )
11723 .unwrap();
11724
11725 assert_route_poll_liveness(&responses[0], true);
11726 }
11727
11728 #[test]
11729 fn liveness_poll_untracked_process_source_uses_bound_route() {
11730 let registry = Arc::new(Registry::default());
11731 let forwarding = Arc::new(ForwardingTable::default());
11732 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
11733 let handler =
11734 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11735 .with_process_liveness(process_liveness);
11736 let (ctx, route_channel, route_epoch) =
11737 bind_liveness_route(®istry, &forwarding, "aft-untracked");
11738 let responses = handler
11739 .handle_route_poll(
11740 &ctx,
11741 route_poll_frame(43, PollKind::Liveness, route_channel),
11742 route_channel,
11743 route_epoch,
11744 PollKind::Liveness,
11745 )
11746 .unwrap();
11747
11748 assert_route_poll_liveness(&responses[0], true);
11749 }
11750
11751 #[tokio::test]
11752 async fn unknown_op_returns_unknown_control_op() {
11753 let handler = ControlHandler::default();
11754 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
11755 let request = Frame::build(
11756 FrameType::Request,
11757 control_flags(),
11758 0,
11759 0,
11760 55,
11761 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
11762 )
11763 .unwrap();
11764
11765 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11766
11767 assert_eq!(response.len(), 1);
11768 assert_eq!(response[0].header.ty, FrameType::Error);
11769 assert_eq!(response[0].header.corr, 55);
11770 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
11771 }
11772
11773 #[tokio::test]
11774 async fn supervisor_provenance_rejects_unknown_exact_module() {
11775 let handler = ControlHandler::default();
11776 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
11777 let request = Frame::build(
11778 FrameType::Request,
11779 control_flags(),
11780 0,
11781 0,
11782 57,
11783 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
11784 )
11785 .unwrap();
11786
11787 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11788
11789 assert_eq!(response.len(), 1);
11790 assert_eq!(response[0].header.ty, FrameType::Error);
11791 assert_eq!(response[0].header.corr, 57);
11792 let error = parse_error(&response[0]);
11793 assert_eq!(error["code"], "unknown_module");
11794 assert_eq!(error["message"], "module_id 'missing' is not supervised");
11795 }
11796
11797 #[test]
11798 fn provenance_probe_override_keeps_handler_tests_deterministic() {
11799 let expected = subc_control::RunningImageAgreement::Unavailable {
11800 reason: subc_control::RunningImageUnavailableReason::HashFailed,
11801 };
11802 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
11803 assert_eq!(handler.provenance_probe_override, Some(expected));
11804 }
11805
11806 #[test]
11807 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
11808 let verdict = reload_verdict(
11809 std::path::Path::new("/bin/new"),
11810 Some(std::path::Path::new("/bin/old")),
11811 subc_control::RunningImageAgreement::Unavailable {
11812 reason: subc_control::RunningImageUnavailableReason::HashFailed,
11813 },
11814 );
11815 assert!(matches!(
11816 verdict.path,
11817 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
11818 if configured == std::path::Path::new("/bin/new")
11819 && spawned_from == std::path::Path::new("/bin/old")
11820 ));
11821 }
11822
11823 #[cfg(windows)]
11824 #[test]
11825 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
11826 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
11827 let configured = dir.join("module.exe");
11828 std::fs::write(&configured, b"executable path fixture").unwrap();
11829 let spawned = std::fs::canonicalize(&configured).unwrap();
11830 assert_ne!(
11831 configured, spawned,
11832 "fixture must exercise different path spellings"
11833 );
11834 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
11835 let lower_case = configured.to_string_lossy().to_lowercase();
11836 for spelling in [
11837 &configured,
11838 Path::new(&forward_slashes),
11839 Path::new(&lower_case),
11840 ] {
11841 let verdict = reload_verdict(
11842 spelling,
11843 Some(&spawned),
11844 subc_control::RunningImageAgreement::Unavailable {
11845 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
11846 },
11847 );
11848 assert_eq!(
11849 verdict.path,
11850 ReloadPathAgreement::Match,
11851 "configured {spelling:?}, spawned {spawned:?}"
11852 );
11853 }
11854 }
11855
11856 #[cfg(windows)]
11857 #[test]
11858 fn windows_reload_path_keeps_different_existing_program_pending() {
11859 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
11860 let old = dir.join("old.exe");
11861 let new = dir.join("new.exe");
11862 std::fs::write(&old, b"same executable content").unwrap();
11863 std::fs::copy(&old, &new).unwrap();
11864 let spawned = std::fs::canonicalize(&old).unwrap();
11865 let image = subc_control::RunningImageAgreement::Unavailable {
11866 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
11867 };
11868 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
11869 assert!(
11870 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
11871 if configured == new && spawned_from == spawned)
11872 );
11873 let missing = dir.join("missing.exe");
11874 assert!(matches!(
11875 reload_verdict(&missing, Some(&spawned), image).path,
11876 ReloadPathAgreement::Mismatch { .. }
11877 ));
11878 }
11879
11880 #[test]
11881 fn reload_verdict_detects_replaced_image_at_same_path() {
11882 let image = subc_control::RunningImageAgreement::Mismatch {
11883 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
11884 digest: "old".into(),
11885 },
11886 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
11887 digest: "new".into(),
11888 },
11889 };
11890 let verdict = reload_verdict(
11891 std::path::Path::new("/bin/same"),
11892 Some(std::path::Path::new("/bin/same")),
11893 image.clone(),
11894 );
11895 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11896 assert_eq!(verdict.image, image);
11897 }
11898
11899 #[test]
11900 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
11901 let image = subc_control::RunningImageAgreement::Unavailable {
11902 reason: subc_control::RunningImageUnavailableReason::NotRunning,
11903 };
11904 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
11905 assert_eq!(
11906 verdict.path,
11907 subc_control::ReloadPathAgreement::Unavailable {
11908 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
11909 }
11910 );
11911 assert_eq!(verdict.image, image);
11912
11913 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
11914 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
11915 };
11916 let verdict = reload_verdict(
11917 std::path::Path::new("/bin/same"),
11918 Some(std::path::Path::new("/bin/same")),
11919 unconfirmed.clone(),
11920 );
11921 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11922 assert_eq!(verdict.image, unconfirmed);
11923 }
11924
11925 #[test]
11926 fn reload_verdict_preserves_each_image_unavailability_reason() {
11927 use subc_control::RunningImageUnavailableReason as Reason;
11928
11929 for reason in [
11930 Reason::NotRunning,
11931 Reason::UnsupportedPlatform,
11932 Reason::RunningExecutableUnreadable,
11933 Reason::SpawnedPathUnreadable,
11934 Reason::HashFailed,
11935 Reason::ProcessIdentityUnconfirmed,
11936 Reason::Unknown("future_probe_reason".to_string()),
11937 ] {
11938 let image = subc_control::RunningImageAgreement::Unavailable {
11939 reason: reason.clone(),
11940 };
11941 let verdict = reload_verdict(
11942 std::path::Path::new("/bin/same"),
11943 Some(std::path::Path::new("/bin/same")),
11944 image.clone(),
11945 );
11946 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11947 assert_eq!(verdict.image, image, "{reason:?}");
11948 }
11949 }
11950
11951 #[tokio::test]
11952 async fn malformed_control_bodies_return_invalid_control_body() {
11953 let handler = ControlHandler::default();
11954 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
11955
11956 for (corr, body) in [
11957 (56, br#"{"route_channel":1}"#.as_slice()),
11958 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
11959 (
11960 58,
11961 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
11962 ),
11963 ] {
11964 let request = Frame::build(
11965 FrameType::Request,
11966 control_flags(),
11967 0,
11968 0,
11969 corr,
11970 body.to_vec(),
11971 )
11972 .unwrap();
11973 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11974
11975 assert_eq!(response.len(), 1);
11976 assert_eq!(response[0].header.ty, FrameType::Error);
11977 assert_eq!(response[0].header.corr, corr);
11978 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
11979 }
11980 }
11981
11982 #[tokio::test]
11983 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
11984 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
11985 let registry = Arc::new(Registry::default());
11986 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
11987 let router = Router::with_control_handler(Arc::clone(&control));
11988 let connection = router.begin_connection();
11989 let (ctx, mut rx) = route_ctx(connection.id());
11990
11991 router
11992 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
11993 .await
11994 .unwrap();
11995 let response = rx.recv().await.unwrap();
11996 let ack = parse_ack(&response);
11997 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11998 let channel = 1;
11999
12000 let goodbye =
12001 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
12002 router.route_for_connection(&ctx, goodbye).await.unwrap();
12003 assert!(rx.try_recv().is_err());
12004 assert!(registry.get_module("aft").unwrap().is_none());
12005
12006 router
12007 .route_for_connection(&ctx, channel_request(channel, 13))
12008 .await
12009 .unwrap();
12010 let error_frame = rx.recv().await.unwrap();
12011 assert_eq!(error_frame.header.ty, FrameType::Error);
12012 assert_eq!(error_frame.header.channel, channel);
12013 let captured = crate::router::test_log::captured_logs(&logs);
12014 assert_eq!(
12015 captured
12016 .lines()
12017 .filter(|line| {
12018 line.contains("module_id=aft")
12019 && line.contains("reason=explicit_goodbye")
12020 && line.contains("module registration ended")
12021 })
12022 .count(),
12023 1,
12024 "unexpected GOODBYE registry log: {captured}"
12025 );
12026 }
12027
12028 #[tokio::test]
12029 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
12030 let registry = Arc::new(Registry::default());
12031 let handler = ControlHandler::new(registry).with_capability_config(
12032 [("prov".to_string(), true), ("cons".to_string(), true)],
12033 BTreeMap::new(),
12034 );
12035 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
12036 register_capability_manifest(
12037 &handler,
12038 &provider_ctx,
12039 &mut provider_rx,
12040 capability_manifest("prov", &["thing/v1"], &[]),
12041 1,
12042 )
12043 .await;
12044 let mut consumer = capability_manifest("cons", &[], &[]);
12045 consumer.capabilities.as_mut().unwrap().requires.push(
12046 subc_protocol::manifest::CapabilityRequirement {
12047 capability: "thing/v1".to_string(),
12048 need: subc_protocol::manifest::CapabilityNeed::Required,
12049 },
12050 );
12051 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
12052 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
12053 assert_eq!(
12054 handler.capability_evaluator.verdict("cons", "thing/v1"),
12055 Some(CapabilityVerdict::Provided)
12056 );
12057 let (mut client_rx, _) = open_route_for_capability_test(
12058 &handler,
12059 &provider_ctx,
12060 &mut provider_rx,
12061 703,
12062 3,
12063 "prov",
12064 None,
12065 )
12066 .await;
12067 let goodbye =
12068 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
12069 handler
12070 .handle_control_frame(&provider_ctx, goodbye)
12071 .await
12072 .unwrap();
12073 assert_eq!(
12074 handler.capability_evaluator.verdict("cons", "thing/v1"),
12075 Some(CapabilityVerdict::NeverProvided)
12076 );
12077 let closed = client_rx
12078 .try_recv()
12079 .expect("GOODBYE pushes route.closed before route GOODBYE");
12080 assert!(
12081 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
12082 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
12083 );
12084 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
12085 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
12086 }
12087
12088 #[tokio::test]
12089 async fn dropping_router_connection_releases_registration() {
12090 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12091 let registry = Arc::new(Registry::default());
12092 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
12093 let router = Router::with_control_handler(Arc::clone(&control));
12094 let connection = router.begin_connection();
12095 let connection_id = connection.id();
12096 let (ctx, mut rx) = route_ctx(connection_id);
12097
12098 router
12099 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
12100 .await
12101 .unwrap();
12102 let response = rx.recv().await.unwrap();
12103 let ack = parse_ack(&response);
12104 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
12105 assert!(registry.get_module("aft").unwrap().is_some());
12106
12107 drop(connection);
12108
12109 assert!(registry.get_module("aft").unwrap().is_none());
12110 assert_eq!(registry.active_registration_count().unwrap(), 0);
12111
12112 control
12113 .cleanup_connection(ConnectionId::new(u64::MAX))
12114 .unwrap();
12115 let captured = crate::router::test_log::captured_logs(&logs);
12116 println!("captured registry lifecycle logs:\n{captured}");
12117 let events: Vec<_> = captured
12118 .lines()
12119 .filter(|line| {
12120 line.contains("module registered module_id=aft ")
12121 || line.contains("module registration ended")
12122 })
12123 .collect();
12124 assert_eq!(
12125 events.len(),
12126 2,
12127 "unexpected registry lifecycle logs: {captured}"
12128 );
12129 assert!(events[0].contains("module registered module_id=aft "));
12130 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
12131 assert!(events[1].contains(&format!(
12132 "module_id=aft connection_id={}",
12133 connection_id.get()
12134 )));
12135 assert!(events[1].contains("reason=connection_closed"));
12136 assert!(events[1].contains("module registration ended"));
12137 assert_eq!(
12138 captured
12139 .lines()
12140 .filter(|line| line.contains("module registered module_id=aft "))
12141 .count(),
12142 1,
12143 "legacy registration admission line must appear once: {captured}"
12144 );
12145 }
12146
12147 #[tokio::test]
12148 async fn hello_registration_keeps_legacy_module_registered_line_once() {
12149 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12150 let registry = Arc::new(Registry::default());
12151 let control = ControlHandler::new(Arc::clone(®istry));
12152 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
12153 hello_via_sink(
12154 &control,
12155 &ctx,
12156 &mut rx,
12157 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
12158 )
12159 .await;
12160
12161 let captured = crate::router::test_log::captured_logs(&logs);
12162 let admissions: Vec<_> = captured
12163 .lines()
12164 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
12165 .collect();
12166 assert_eq!(
12167 admissions.len(),
12168 1,
12169 "legacy admission line must remain exactly once: {captured}"
12170 );
12171 assert!(admissions[0].contains("routable_provider=true"));
12172 assert!(admissions[0].contains("connection_id=777"));
12173 }
12174
12175 fn capability_manifest(
12176 module_id: &str,
12177 provides: &[&str],
12178 must_never_reach: &[&str],
12179 ) -> ModuleManifest {
12180 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
12181 manifest.capabilities = Some(CapabilityDeclarations {
12182 provides: provides
12183 .iter()
12184 .map(|capability| (*capability).to_string())
12185 .collect(),
12186 requires: Vec::new(),
12187 must_never_reach: must_never_reach
12188 .iter()
12189 .map(|capability| (*capability).to_string())
12190 .collect(),
12191 });
12192 manifest
12193 }
12194
12195 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
12196 Frame::build(
12197 FrameType::Hello,
12198 control_flags(),
12199 0,
12200 0,
12201 corr,
12202 serde_json::to_vec(&ModuleHelloBody {
12203 protocol_ver: manifest.protocol_ver,
12204 manifest,
12205 control_ops: None,
12206 launch_nonce: None,
12207 })
12208 .expect("capability test HELLO serializes"),
12209 )
12210 .expect("capability test HELLO frame builds")
12211 }
12212
12213 fn catalog_update_with_capabilities_frame(
12214 corr: u64,
12215 capabilities: CapabilityDeclarations,
12216 ) -> Frame {
12217 Frame::build(
12218 FrameType::Request,
12219 control_flags(),
12220 0,
12221 0,
12222 corr,
12223 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
12224 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
12225 capabilities: Some(capabilities),
12226 ready: None,
12227 })
12228 .expect("capability catalog.update serializes"),
12229 )
12230 .expect("capability catalog.update frame builds")
12231 }
12232
12233 async fn register_capability_manifest(
12234 handler: &ControlHandler,
12235 ctx: &RouteCtx,
12236 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
12237 manifest: ModuleManifest,
12238 corr: u64,
12239 ) {
12240 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
12241 }
12242
12243 async fn open_route_for_capability_test(
12244 handler: &ControlHandler,
12245 target_ctx: &RouteCtx,
12246 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
12247 client_connection_id: u64,
12248 corr: u64,
12249 target_module_id: &str,
12250 consumer_identity: Option<ConsumerIdentity>,
12251 ) -> (
12252 mpsc::Receiver<crate::router::OutboundFrame>,
12253 ModuleControlRequest,
12254 ) {
12255 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
12256 let route_handler = handler.clone();
12257 let target_module_id = target_module_id.to_string();
12258 let route_task = tokio::spawn(async move {
12259 route_handler
12260 .handle_control_frame(
12261 &client_ctx,
12262 route_open_frame_with_admission_facts(
12263 corr,
12264 &target_module_id,
12265 unique_project_root("admission-facts"),
12266 consumer_identity,
12267 None,
12268 ),
12269 )
12270 .await
12271 .expect("capability test route.open succeeds")
12272 });
12273 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
12274 .await
12275 .expect("capability test route.open must reach route.bind")
12276 .expect("target control receiver stays open");
12277 let bind_request: ModuleControlRequest =
12278 serde_json::from_slice(&bind.body).expect("route.bind decodes");
12279 handler
12280 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
12281 .await
12282 .expect("capability test route.bind ACK succeeds");
12283 assert!(route_task.await.expect("route.open task joins").is_empty());
12284 let opened = client_rx
12285 .recv()
12286 .await
12287 .expect("successful route.open publishes a response");
12288 assert!(matches!(
12289 serde_json::from_slice::<ClientControlResponse>(&opened.body),
12290 Ok(ClientControlResponse::RouteOpen { .. })
12291 ));
12292 (client_rx, bind_request)
12293 }
12294
12295 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
12296 assert_eq!(frame.header.ty, FrameType::Push);
12297 assert_eq!(frame.header.channel, 0);
12298 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
12299 .expect("route.closed control push decodes");
12300 let ClientControlPush::RouteClosed { channels, .. } = &push else {
12301 panic!("expected route.closed");
12302 };
12303 assert_eq!(channels.len(), 1, "exactly one violating route closed");
12304 let channels = channels.clone();
12305 assert_eq!(
12306 push,
12307 ClientControlPush::RouteClosed {
12308 module_id: target_module_id.to_string(),
12309 channels,
12310 reason: RouteCloseReason::CapabilityDenied,
12311 drained: false,
12312 abandoned: 0,
12313 excluded_subscriptions: 0,
12314 terminal: Some(false),
12315 }
12316 );
12317 }
12318
12319 #[tokio::test]
12320 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
12321 let registry = Arc::new(Registry::default());
12322 let forwarding = Arc::new(ForwardingTable::default());
12323 let supervisor = SupervisorHandle::new();
12324 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12325 let handler =
12326 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12327 .with_supervisor(supervisor);
12328 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
12329 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
12330 register_capability_manifest(
12331 &handler,
12332 &target_ctx,
12333 &mut target_rx,
12334 capability_manifest("target", &["credentials-provider/v1"], &[]),
12335 1,
12336 )
12337 .await;
12338 register_capability_manifest(
12339 &handler,
12340 &opener_ctx,
12341 &mut opener_rx,
12342 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12343 2,
12344 )
12345 .await;
12346
12347 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
12348 let replies = handler
12349 .handle_control_frame(
12350 &client_ctx,
12351 route_open_frame_with_admission_facts(
12352 3,
12353 "target",
12354 unique_project_root("admission-facts"),
12355 Some(ConsumerIdentity {
12356 module_id: "opener".to_string(),
12357 launch_nonce: "opener-nonce".to_string(),
12358 }),
12359 None,
12360 ),
12361 )
12362 .await
12363 .expect("denied route.open returns a typed frame");
12364 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
12365 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12366 assert!(
12367 target_rx.try_recv().is_err(),
12368 "forbidden route.open must not relay route.bind"
12369 );
12370 }
12371
12372 #[tokio::test]
12373 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
12374 let registry = Arc::new(Registry::default());
12375 let forwarding = Arc::new(ForwardingTable::default());
12376 let supervisor = SupervisorHandle::new();
12377 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12378 let handler =
12379 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12380 .with_supervisor(supervisor);
12381 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
12382 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
12383 register_capability_manifest(
12384 &handler,
12385 &target_ctx,
12386 &mut target_rx,
12387 capability_manifest("target", &["credentials-provider/v1"], &[]),
12388 1,
12389 )
12390 .await;
12391 register_capability_manifest(
12392 &handler,
12393 &old_opener_ctx,
12394 &mut old_opener_rx,
12395 capability_manifest("opener", &[], &[]),
12396 2,
12397 )
12398 .await;
12399 let (mut client_rx, _) = open_route_for_capability_test(
12400 &handler,
12401 &target_ctx,
12402 &mut target_rx,
12403 712,
12404 3,
12405 "target",
12406 Some(ConsumerIdentity {
12407 module_id: "opener".to_string(),
12408 launch_nonce: "opener-nonce".to_string(),
12409 }),
12410 )
12411 .await;
12412 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12413
12414 handler
12415 .cleanup_connection(old_opener_ctx.connection_id)
12416 .expect("old opener registration cleans up");
12417 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
12418 register_capability_manifest(
12419 &handler,
12420 &new_opener_ctx,
12421 &mut new_opener_rx,
12422 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12423 4,
12424 )
12425 .await;
12426
12427 assert_capability_denied_push(
12428 client_rx
12429 .try_recv()
12430 .expect("HELLO deny addition must emit route.closed")
12431 .frame,
12432 "target",
12433 );
12434 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12435 assert!(matches!(
12436 target_rx.try_recv(),
12437 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
12438 ));
12439 }
12440
12441 #[tokio::test]
12442 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
12443 let registry = Arc::new(Registry::default());
12444 let forwarding = Arc::new(ForwardingTable::default());
12445 let supervisor = SupervisorHandle::new();
12446 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12447 let handler =
12448 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12449 .with_supervisor(supervisor);
12450 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
12451 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
12452 register_capability_manifest(
12453 &handler,
12454 &target_ctx,
12455 &mut target_rx,
12456 capability_manifest("target", &[], &[]),
12457 1,
12458 )
12459 .await;
12460 register_capability_manifest(
12461 &handler,
12462 &opener_ctx,
12463 &mut opener_rx,
12464 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12465 2,
12466 )
12467 .await;
12468 let (mut client_rx, _) = open_route_for_capability_test(
12469 &handler,
12470 &target_ctx,
12471 &mut target_rx,
12472 722,
12473 3,
12474 "target",
12475 Some(ConsumerIdentity {
12476 module_id: "opener".to_string(),
12477 launch_nonce: "opener-nonce".to_string(),
12478 }),
12479 )
12480 .await;
12481 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12482
12483 let replies = handler
12484 .handle_control_frame(
12485 &target_ctx,
12486 catalog_update_with_capabilities_frame(
12487 4,
12488 CapabilityDeclarations {
12489 provides: vec!["credentials-provider/v1".to_string()],
12490 requires: Vec::new(),
12491 must_never_reach: Vec::new(),
12492 },
12493 ),
12494 )
12495 .await
12496 .expect("claim catalog.update succeeds");
12497 assert!(matches!(
12498 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
12499 Ok(ModuleControlResponseToModule::CatalogUpdate {})
12500 ));
12501 assert_capability_denied_push(
12502 client_rx
12503 .try_recv()
12504 .expect("claim addition must emit route.closed")
12505 .frame,
12506 "target",
12507 );
12508 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12509 assert!(matches!(
12510 target_rx.try_recv(),
12511 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
12512 ));
12513 }
12514
12515 #[tokio::test]
12516 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
12517 let registry = Arc::new(Registry::default());
12518 let forwarding = Arc::new(ForwardingTable::default());
12519 let supervisor = SupervisorHandle::new();
12520 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12521 let handler =
12522 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12523 .with_supervisor(supervisor);
12524 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
12525 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
12526 register_capability_manifest(
12527 &handler,
12528 &target_ctx,
12529 &mut target_rx,
12530 capability_manifest("target", &["credentials-provider/v1"], &[]),
12531 1,
12532 )
12533 .await;
12534 register_capability_manifest(
12535 &handler,
12536 &opener_ctx,
12537 &mut opener_rx,
12538 capability_manifest("opener", &[], &[]),
12539 2,
12540 )
12541 .await;
12542 let (mut client_rx, _) = open_route_for_capability_test(
12543 &handler,
12544 &target_ctx,
12545 &mut target_rx,
12546 732,
12547 3,
12548 "target",
12549 Some(ConsumerIdentity {
12550 module_id: "opener".to_string(),
12551 launch_nonce: "opener-nonce".to_string(),
12552 }),
12553 )
12554 .await;
12555 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12556
12557 handler
12558 .handle_control_frame(
12559 &target_ctx,
12560 catalog_update_with_capabilities_frame(
12561 4,
12562 CapabilityDeclarations {
12563 provides: Vec::new(),
12564 requires: Vec::new(),
12565 must_never_reach: Vec::new(),
12566 },
12567 ),
12568 )
12569 .await
12570 .expect("claim removal catalog.update succeeds");
12571 assert_eq!(
12572 forwarding.active_binding_count().unwrap(),
12573 1,
12574 "removing an attested target claim must leave the route census unchanged"
12575 );
12576 assert!(
12577 client_rx.try_recv().is_err(),
12578 "claim removal must not emit route.closed capability_denied"
12579 );
12580 assert!(
12581 target_rx.try_recv().is_err(),
12582 "claim removal must not send the target a route GOODBYE"
12583 );
12584 }
12585
12586 #[tokio::test]
12589 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
12590 let registry = Arc::new(Registry::default());
12591 let forwarding = Arc::new(ForwardingTable::default());
12592 let supervisor = SupervisorHandle::new();
12593 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12594 let handler =
12595 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12596 .with_supervisor(supervisor);
12597 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
12598 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
12599 register_capability_manifest(
12600 &handler,
12601 &target_ctx,
12602 &mut target_rx,
12603 capability_manifest("target", &["credentials-provider/v1"], &[]),
12604 1,
12605 )
12606 .await;
12607 register_capability_manifest(
12608 &handler,
12609 &opener_ctx,
12610 &mut opener_rx,
12611 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12612 2,
12613 )
12614 .await;
12615
12616 let (_client_rx, bind) = open_route_for_capability_test(
12617 &handler,
12618 &target_ctx,
12619 &mut target_rx,
12620 742,
12621 3,
12622 "target",
12623 None,
12624 )
12625 .await;
12626 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
12627 panic!("direct scope-honesty route must bind");
12628 };
12629 assert_eq!(principal, Some(Principal::Direct));
12630 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12631 }
12632
12633 #[tokio::test]
12636 async fn must_never_reach_self_route_is_capability_forbidden() {
12637 let registry = Arc::new(Registry::default());
12638 let forwarding = Arc::new(ForwardingTable::default());
12639 let supervisor = SupervisorHandle::new();
12640 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
12641 let handler =
12642 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12643 .with_supervisor(supervisor);
12644 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
12645 register_capability_manifest(
12646 &handler,
12647 &self_ctx,
12648 &mut self_rx,
12649 capability_manifest(
12650 "self-provider",
12651 &["credentials-provider/v1"],
12652 &["credentials-provider/v1"],
12653 ),
12654 1,
12655 )
12656 .await;
12657
12658 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
12659 let replies = handler
12660 .handle_control_frame(
12661 &client_ctx,
12662 route_open_frame_with_admission_facts(
12663 2,
12664 "self-provider",
12665 unique_project_root("admission-facts"),
12666 Some(ConsumerIdentity {
12667 module_id: "self-provider".to_string(),
12668 launch_nonce: "self-nonce".to_string(),
12669 }),
12670 None,
12671 ),
12672 )
12673 .await
12674 .expect("self-route refusal returns a typed frame");
12675 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
12676 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12677 assert!(
12678 self_rx.try_recv().is_err(),
12679 "self denial must not relay route.bind"
12680 );
12681 }
12682
12683 #[test]
12684 fn unsupported_channel_zero_frame_returns_error() {
12685 let handler = ControlHandler::default();
12686 let request = Frame::build(
12687 FrameType::Request,
12688 control_flags(),
12689 0,
12690 0,
12691 21,
12692 b"opaque".to_vec(),
12693 )
12694 .unwrap();
12695
12696 let response = handler
12697 .handle_control(ConnectionId::new(1), request)
12698 .unwrap();
12699
12700 assert_eq!(response[0].header.ty, FrameType::Error);
12701 assert_eq!(
12702 parse_error(&response[0])["code"],
12703 "unsupported_control_frame"
12704 );
12705 }
12706
12707 mod swap {
12712 use super::*;
12713
12714 const INCUMBENT: ConnectionId = ConnectionId::new(30);
12715 const CANDIDATE: ConnectionId = ConnectionId::new(40);
12716
12717 struct Swap {
12718 registry: Arc<Registry>,
12719 forwarding: Arc<ForwardingTable>,
12720 handler: ControlHandler,
12721 incumbent_ctx: RouteCtx,
12722 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12723 candidate_ctx: RouteCtx,
12724 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12725 }
12726
12727 async fn swap_with_incumbent() -> Swap {
12728 let registry = Arc::new(Registry::default());
12729 let forwarding = Arc::new(ForwardingTable::default());
12730 let handler =
12731 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12732 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
12733 hello_via_sink(
12734 &handler,
12735 &incumbent_ctx,
12736 &mut incumbent_rx,
12737 hello_frame("aft", PROTOCOL_VERSION, 7),
12738 )
12739 .await;
12740 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
12741 Swap {
12742 registry,
12743 forwarding,
12744 handler,
12745 incumbent_ctx,
12746 incumbent_rx,
12747 candidate_ctx,
12748 candidate_rx,
12749 }
12750 }
12751
12752 fn register_candidate(swap: &Swap, ready: Option<bool>) {
12753 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
12754 candidate_manifest.ready = ready;
12755 let registration = swap
12756 .registry
12757 .register_candidate_with_control_ops(
12758 candidate_manifest,
12759 PROTOCOL_VERSION,
12760 CANDIDATE,
12761 module_baseline_control_ops(),
12762 )
12763 .unwrap();
12764 swap.forwarding
12765 .register_candidate_module_connection(
12766 CANDIDATE,
12767 "aft".to_string(),
12768 PROTOCOL_VERSION,
12769 manifest_concurrency(®istration.manifest),
12770 swap.candidate_ctx.egress.clone(),
12771 )
12772 .unwrap();
12773 }
12774
12775 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
12776 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
12777 swap.registry.promote_candidate("aft").unwrap().unwrap();
12778 cutover.incumbent.unwrap()
12779 }
12780
12781 fn keyed_total(counters: &Value, key: &str) -> u64 {
12782 counters[key]
12783 .as_object()
12784 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
12785 .unwrap_or(0)
12786 }
12787
12788 #[tokio::test]
12789 async fn replacement_logs_old_end_and_new_admission_once() {
12790 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12791 let mut swap = swap_with_incumbent().await;
12792 swap.handler
12793 .supervisor
12794 .open_swap("aft", "candidate-nonce".to_string());
12795 hello_via_sink(
12796 &swap.handler,
12797 &swap.candidate_ctx,
12798 &mut swap.candidate_rx,
12799 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
12800 )
12801 .await;
12802 cutover(&swap);
12803 swap.handler.cleanup_connection(INCUMBENT).unwrap();
12804
12805 let captured = crate::router::test_log::captured_logs(&logs);
12806 println!("captured replacement registry logs:\n{captured}");
12807 let new_admissions: Vec<_> = captured
12808 .lines()
12809 .filter(|line| {
12810 line.contains("connection_id=40")
12811 && line.contains("swap candidate registered; not routable until cutover")
12812 })
12813 .collect();
12814 assert_eq!(
12815 new_admissions.len(),
12816 1,
12817 "unexpected admission logs: {captured}"
12818 );
12819 assert!(new_admissions[0].contains("module_id=aft"));
12820 assert!(new_admissions[0].contains("connection_id=40"));
12821 assert!(
12822 new_admissions[0].contains("swap candidate registered; not routable until cutover")
12823 );
12824 assert!(new_admissions[0].contains("ready=true"));
12825
12826 let old_admissions: Vec<_> = captured
12827 .lines()
12828 .filter(|line| {
12829 line.contains("module registered module_id=aft ")
12830 && line.contains("connection_id=30")
12831 })
12832 .collect();
12833 assert_eq!(
12834 old_admissions.len(),
12835 1,
12836 "unexpected incumbent admission: {captured}"
12837 );
12838
12839 let old_ends: Vec<_> = captured
12840 .lines()
12841 .filter(|line| {
12842 line.contains("connection_id=30") && line.contains("module registration ended")
12843 })
12844 .collect();
12845 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
12846 assert!(old_ends[0].contains("module_id=aft"));
12847 assert!(old_ends[0].contains("connection_id=30"));
12848 assert!(old_ends[0].contains("reason=replaced"));
12849 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
12850 assert_eq!(
12851 captured
12852 .lines()
12853 .filter(|line| line.contains("module registration promoted"))
12854 .count(),
12855 1,
12856 "unexpected promotion logs: {captured}"
12857 );
12858 }
12859
12860 #[tokio::test]
12866 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
12867 let mut swap = swap_with_incumbent().await;
12868 let handler = swap.handler.clone();
12869
12870 let cotenant = ConnectionId::new(31);
12872 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
12873 let (cotenant_task, cotenant_bind) = relay_route_open(
12874 &handler,
12875 cotenant,
12876 &cotenant_ctx.egress,
12877 &mut swap.incumbent_rx,
12878 100,
12879 "aft",
12880 "swap-cotenant",
12881 )
12882 .await;
12883 handler
12884 .handle_control_frame(
12885 &swap.incumbent_ctx,
12886 route_bind_ack(cotenant_bind.header.corr),
12887 )
12888 .await
12889 .unwrap();
12890 assert!(cotenant_task.await.unwrap().is_empty());
12891 let (cotenant_channel, cotenant_epoch) =
12892 published_route(&cotenant_rx.recv().await.unwrap());
12893
12894 let caller = ConnectionId::new(32);
12896 let (caller_ctx, mut caller_rx) = route_ctx(caller);
12897 let (caller_task, caller_bind) = relay_route_open(
12898 &handler,
12899 caller,
12900 &caller_ctx.egress,
12901 &mut swap.incumbent_rx,
12902 101,
12903 "aft",
12904 "swap-caller",
12905 )
12906 .await;
12907 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
12908
12909 register_candidate(&swap, None);
12910 cutover(&swap);
12911
12912 let ack = handler
12914 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
12915 .await;
12916 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
12917 if module_loop_error.is_some() {
12918 handler.cleanup_connection(INCUMBENT).unwrap();
12921 }
12922
12923 assert!(
12925 cotenant_rx.try_recv().is_err(),
12926 "the co-tenant route on the incumbent was torn down by one late ack: \
12927 {module_loop_error:?}"
12928 );
12929 assert!(matches!(
12930 swap.forwarding
12931 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
12932 .unwrap(),
12933 DataRoute::Client(DataRouteState::Bound(_))
12934 ));
12935 assert_eq!(module_loop_error, None);
12936 assert!(swap
12937 .registry
12938 .get_module_by_connection(INCUMBENT)
12939 .unwrap()
12940 .is_some());
12941
12942 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
12944 .await
12945 .expect("the incumbent is told to drop the abandoned binding")
12946 .unwrap()
12947 .frame;
12948 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12949 assert_eq!(goodbye.header.channel, abandoned_channel);
12950 assert_eq!(goodbye.header.epoch, abandoned_epoch);
12951 assert!(swap.incumbent_rx.try_recv().is_err());
12952
12953 let response = caller_task.await.unwrap();
12955 assert_eq!(response.len(), 1);
12956 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
12957 assert!(caller_rx.try_recv().is_err());
12958
12959 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
12963 let counters = handler.counters().snapshot();
12964 assert_eq!(
12965 keyed_total(&counters, "route_open_accepted_by_principal"),
12966 1
12967 );
12968 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
12969 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
12970 }
12971
12972 #[tokio::test]
12976 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
12977 let mut swap = swap_with_incumbent().await;
12978 register_candidate(&swap, None);
12979 let incumbent = cutover(&swap);
12980 swap.forwarding
12981 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
12982 .unwrap()
12983 .expect("the incumbent is still registered");
12984
12985 let client = ConnectionId::new(33);
12986 let (client_ctx, mut client_rx) = route_ctx(client);
12987 let route_handler = swap.handler.clone();
12988 let open_ctx = RouteCtx {
12989 connection_id: client,
12990 egress: client_ctx.egress.clone(),
12991 };
12992 let mut route_task = tokio::spawn(async move {
12993 route_handler
12994 .handle_control_frame(
12995 &open_ctx,
12996 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
12997 )
12998 .await
12999 .unwrap()
13000 });
13001 let bind = tokio::select! {
13002 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
13003 response = &mut route_task => {
13004 let response = response.unwrap();
13005 panic!(
13006 "post-cutover route.open was refused instead of relayed to the candidate: {}",
13007 parse_error(&response[0])["code"]
13008 );
13009 }
13010 };
13011 swap.handler
13012 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
13013 .await
13014 .unwrap();
13015 assert!(route_task.await.unwrap().is_empty());
13016 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
13017 match swap
13018 .forwarding
13019 .lookup_data_route(client, channel, epoch)
13020 .unwrap()
13021 {
13022 DataRoute::Client(DataRouteState::Bound(route)) => {
13023 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
13024 }
13025 other => panic!("expected a bound route on the candidate, got {other:?}"),
13026 }
13027 assert!(swap.incumbent_rx.try_recv().is_err());
13028 }
13029
13030 #[tokio::test]
13035 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
13036 let swap = swap_with_incumbent().await;
13037 register_candidate(&swap, Some(false));
13038 let update = Frame::build(
13039 FrameType::Request,
13040 control_flags(),
13041 0,
13042 0,
13043 55,
13044 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
13045 provides: manifest("aft", PROTOCOL_VERSION).provides,
13046 capabilities: None,
13047 ready: Some(true),
13048 })
13049 .unwrap(),
13050 )
13051 .unwrap();
13052
13053 let replies = swap
13054 .handler
13055 .handle_control_frame(&swap.candidate_ctx, update)
13056 .await
13057 .unwrap();
13058
13059 assert_eq!(replies.len(), 1);
13060 assert_eq!(
13061 replies[0].header.ty,
13062 FrameType::Response,
13063 "candidate catalog.update was refused: {:?}",
13064 serde_json::from_slice::<Value>(&replies[0].body).ok()
13065 );
13066 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
13067 assert_eq!(
13068 swap.registry
13069 .get_module("aft")
13070 .unwrap()
13071 .unwrap()
13072 .connection_id,
13073 INCUMBENT
13074 );
13075 }
13076 }
13077
13078 mod swap_admission {
13082 use super::*;
13083
13084 const INCUMBENT_NONCE: &str = "incumbent-nonce";
13085 const CANDIDATE_NONCE: &str = "candidate-nonce";
13086
13087 fn handler_with_incumbent(
13088 module_id: &str,
13089 reserved: bool,
13090 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
13091 let registry = Arc::new(Registry::default());
13092 let supervisor = SupervisorHandle::new();
13093 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
13094 if reserved {
13095 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
13096 }
13097 let handler =
13098 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
13099 let incumbent = handler
13100 .handle_control(
13101 ConnectionId::new(1),
13102 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
13103 )
13104 .unwrap();
13105 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
13106 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
13107 (registry, supervisor, handler)
13108 }
13109
13110 #[test]
13116 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
13117 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
13118
13119 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
13120 let replies = handler
13121 .handle_control(
13122 ConnectionId::new(connection),
13123 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
13124 )
13125 .unwrap();
13126 assert_eq!(replies[0].header.ty, FrameType::Error);
13127 assert_eq!(
13128 parse_error(&replies[0])["code"],
13129 "swap_token_invalid",
13130 "nonce {nonce:?}"
13131 );
13132 }
13133 assert!(registry.get_candidate("aft").unwrap().is_none());
13134 assert_eq!(
13135 registry.get_module("aft").unwrap().unwrap().connection_id,
13136 ConnectionId::new(1)
13137 );
13138
13139 let admitted = handler
13142 .handle_control(
13143 ConnectionId::new(4),
13144 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
13145 )
13146 .unwrap();
13147 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
13148 assert_eq!(
13149 registry
13150 .get_candidate("aft")
13151 .unwrap()
13152 .unwrap()
13153 .connection_id,
13154 ConnectionId::new(4)
13155 );
13156 assert_eq!(
13157 registry.get_module("aft").unwrap().unwrap().connection_id,
13158 ConnectionId::new(1),
13159 "the candidate must not take the active slot"
13160 );
13161 let replayed = handler
13162 .handle_control(
13163 ConnectionId::new(5),
13164 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
13165 )
13166 .unwrap();
13167 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
13168
13169 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
13173 let squatter = handler
13174 .handle_control(
13175 ConnectionId::new(6),
13176 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
13177 )
13178 .unwrap();
13179 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
13180 assert!(
13181 registry.get_module("aft").unwrap().is_none(),
13182 "a squatter took the active slot of an id being swapped"
13183 );
13184 }
13185
13186 #[test]
13191 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
13192 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
13193
13194 let replies = handler
13195 .handle_control(
13196 ConnectionId::new(2),
13197 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
13198 )
13199 .unwrap();
13200
13201 assert_eq!(
13202 replies[0].header.ty,
13203 FrameType::HelloAck,
13204 "reserved candidate refused: {:?}",
13205 serde_json::from_slice::<Value>(&replies[0].body).ok()
13206 );
13207 assert_eq!(
13208 registry
13209 .get_candidate("vault")
13210 .unwrap()
13211 .unwrap()
13212 .connection_id,
13213 ConnectionId::new(2)
13214 );
13215 }
13216
13217 #[test]
13220 fn without_an_open_swap_the_ordinary_gates_decide() {
13221 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
13222 supervisor.close_swap("vault");
13223
13224 let candidate = handler
13225 .handle_control(
13226 ConnectionId::new(2),
13227 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
13228 )
13229 .unwrap();
13230 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
13231 let duplicate = handler
13232 .handle_control(
13233 ConnectionId::new(3),
13234 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
13235 )
13236 .unwrap();
13237 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
13238 assert!(registry.get_candidate("vault").unwrap().is_none());
13239 }
13240 }
13241
13242 mod scopes {
13246 mod activity {
13247 include!("activity_scope_tests.rs");
13248 }
13249 mod delta {
13250 include!("scope_handler_delta_tests.rs");
13251 }
13252
13253 use subc_protocol::scope::{
13254 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
13255 ScopeStatus,
13256 };
13257
13258 use super::*;
13259
13260 const OWNER: &str = "prefrontal-core";
13261
13262 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
13263 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
13264 }
13265
13266 async fn call(
13267 handler: &ControlHandler,
13268 ctx: &RouteCtx,
13269 request: &ModuleControlRequestFromModule,
13270 ) -> Frame {
13271 let body = serde_json::to_vec(request).unwrap();
13272 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
13273 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
13274 assert_eq!(replies.len(), 1, "{replies:?}");
13275 replies.pop().unwrap()
13276 }
13277
13278 async fn sync(
13279 handler: &ControlHandler,
13280 ctx: &RouteCtx,
13281 generation: u64,
13282 scopes: Vec<ScopeRecord>,
13283 ) -> Result<ModuleControlResponseToModule, String> {
13284 let reply = call(
13285 handler,
13286 ctx,
13287 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
13288 )
13289 .await;
13290 match reply.header.ty {
13291 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
13292 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
13293 }
13294 }
13295
13296 async fn describe(
13297 handler: &ControlHandler,
13298 ctx: &RouteCtx,
13299 owner: &str,
13300 scope_ref: &str,
13301 ) -> ModuleControlResponseToModule {
13302 let reply = call(
13303 handler,
13304 ctx,
13305 &ModuleControlRequestFromModule::ScopeDescribe {
13306 owner: Principal::Reserved {
13307 module_id: owner.to_string(),
13308 },
13309 scope_ref: scope_ref.to_string(),
13310 },
13311 )
13312 .await;
13313 assert_eq!(
13314 reply.header.ty,
13315 FrameType::Response,
13316 "{:?}",
13317 parse_error(&reply)
13318 );
13319 serde_json::from_slice(&reply.body).unwrap()
13320 }
13321
13322 async fn module(
13324 handler: &ControlHandler,
13325 connection: u64,
13326 module_id: &str,
13327 nonce: Option<&str>,
13328 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
13329 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
13330 hello_via_sink(
13331 handler,
13332 &ctx,
13333 &mut rx,
13334 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
13335 )
13336 .await;
13337 (ctx, rx)
13338 }
13339
13340 #[tokio::test]
13343 async fn a_client_connection_cannot_sync_or_describe() {
13344 let handler = ControlHandler::new(Arc::new(Registry::default()));
13345 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
13346 for request in [
13347 ModuleControlRequestFromModule::ScopeSync {
13348 generation: 1,
13349 scopes: vec![head("s", 1)],
13350 },
13351 ModuleControlRequestFromModule::ScopeDescribe {
13352 owner: Principal::Direct,
13353 scope_ref: "s".to_string(),
13354 },
13355 ] {
13356 let reply = call(&handler, &ctx, &request).await;
13357 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
13358 }
13359 assert!(
13360 !handler
13361 .scopes
13362 .read()
13363 .unwrap()
13364 .describe(
13365 &Principal::Reserved {
13366 module_id: OWNER.to_string()
13367 },
13368 "s"
13369 )
13370 .owner_synced
13371 );
13372 }
13373
13374 #[tokio::test]
13377 async fn a_module_without_a_supervised_launch_cannot_sync() {
13378 let handler = ControlHandler::new(Arc::new(Registry::default()));
13379 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
13380 assert_eq!(
13381 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
13382 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13383 );
13384 }
13385
13386 #[tokio::test]
13387 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
13388 let supervisor = SupervisorHandle::new();
13389 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13390 let handler = ControlHandler::new(Arc::new(Registry::default()))
13391 .with_supervisor(supervisor.clone());
13392 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
13393 sync(&handler, &incumbent, 1, vec![head("s", 1)])
13394 .await
13395 .expect("the current launch syncs");
13396
13397 supervisor.open_swap(OWNER, "n2".to_string());
13400 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
13401 assert_eq!(
13402 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
13403 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13404 );
13405 sync(&handler, &incumbent, 2, vec![head("s", 1)])
13406 .await
13407 .expect("the serving owner syncs during the swap");
13408
13409 supervisor.close_swap(OWNER);
13412 assert_eq!(
13413 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
13414 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13415 );
13416 sync(&handler, &incumbent, 3, vec![head("s", 1)])
13417 .await
13418 .expect("the serving owner syncs after the rollback");
13419 handler.cleanup_connection(candidate.connection_id).unwrap();
13420
13421 supervisor.open_swap(OWNER, "n3".to_string());
13426 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
13427 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
13428 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
13429 .await
13430 .expect("the promoted launch takes authority");
13431 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
13432 panic!("unexpected reply {reply:?}");
13433 };
13434 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
13435 assert_eq!(
13436 sync(&handler, &incumbent, 4, Vec::new()).await,
13437 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13438 );
13439 }
13440
13441 #[tokio::test]
13444 async fn closing_the_authority_connection_frees_sync_authority() {
13445 let supervisor = SupervisorHandle::new();
13446 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13447 let handler = ControlHandler::new(Arc::new(Registry::default()))
13448 .with_supervisor(supervisor.clone());
13449 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
13450 sync(&handler, &first, 10, vec![head("s", 1)])
13451 .await
13452 .unwrap();
13453 handler.cleanup_connection(first.connection_id).unwrap();
13454
13455 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
13456 sync(&handler, &second, 1, vec![head("s", 1)])
13457 .await
13458 .expect("the next connection takes the released authority");
13459 }
13460
13461 #[tokio::test]
13462 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
13463 let supervisor = SupervisorHandle::new();
13464 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13465 let handler =
13466 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
13467 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
13468 sync(&handler, &first, 10, vec![head("s", 1)])
13469 .await
13470 .unwrap();
13471 handler
13472 .handle_control_frame(
13473 &first,
13474 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
13475 )
13476 .await
13477 .unwrap();
13478 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
13479 sync(&handler, &second, 1, vec![head("s", 1)])
13480 .await
13481 .expect("GOODBYE releases authority even if the old socket remains open");
13482 }
13483
13484 #[tokio::test]
13485 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
13486 let registry = Arc::new(Registry::default());
13487 let supervisor_handle = SupervisorHandle::new();
13488 let supervisor =
13489 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
13490 .with_handle(supervisor_handle.clone())
13491 .with_daemon_incarnation("incarnation-7".to_string());
13492 supervisor
13495 .supervise_configured(
13496 ModuleSpec {
13497 module_id: OWNER.to_string(),
13498 program: PathBuf::from("/nonexistent/prefrontal-core"),
13499 args: Vec::new(),
13500 env: Vec::new(),
13501 reserved: false,
13502 reserved_prefixes: Vec::new(),
13503 protocol: ModuleProtocol::Subc,
13504 overlap: Default::default(),
13505 },
13506 false,
13507 )
13508 .unwrap();
13509 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
13510 let handler =
13511 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13512 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
13513
13514 let ModuleControlResponseToModule::ScopeDescribe {
13516 status,
13517 daemon_incarnation,
13518 owner_synced,
13519 owner_configured,
13520 scope,
13521 ..
13522 } = describe(&handler, &reader, OWNER, "s").await
13523 else {
13524 panic!("not a describe reply");
13525 };
13526 assert_eq!(status, ScopeStatus::NotLive);
13527 assert_eq!(daemon_incarnation, "incarnation-7");
13528 assert!(!owner_synced);
13529 assert!(owner_configured);
13530 assert!(scope.is_none());
13531
13532 let ModuleControlResponseToModule::ScopeDescribe {
13535 status,
13536 owner_configured,
13537 ..
13538 } = describe(&handler, &reader, "ghost", "s").await
13539 else {
13540 panic!("not a describe reply");
13541 };
13542 assert_eq!(status, ScopeStatus::NotLive);
13543 assert!(!owner_configured);
13544
13545 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
13547 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
13548 let ModuleControlResponseToModule::ScopeDescribe {
13549 status,
13550 scope_epoch,
13551 owner_synced,
13552 scope,
13553 ..
13554 } = describe(&handler, &reader, OWNER, "s").await
13555 else {
13556 panic!("not a describe reply");
13557 };
13558 assert_eq!(status, ScopeStatus::Live);
13559 assert_eq!(scope_epoch, Some(4));
13560 assert!(owner_synced);
13561 let stamp = scope.expect("a live scope carries its stamp");
13562 assert!(
13563 stamp.owner_authorized,
13564 "prefrontal-core is the default authority"
13565 );
13566 assert_eq!(stamp.kind, ScopeKind::Head);
13567 }
13568
13569 #[tokio::test]
13570 async fn scope_authority_owners_decides_owner_authorized() {
13571 let supervisor = SupervisorHandle::new();
13572 supervisor.set_spawn_nonce("broca", "b1".to_string());
13573 let handler = ControlHandler::new(Arc::new(Registry::default()))
13574 .with_supervisor(supervisor)
13575 .with_scope_authority_owners(vec!["broca".to_string()]);
13576 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
13577 let mut gated = head("s", 1);
13578 gated.attributes.agent_id = Some("agent".to_string());
13579 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
13580 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
13581 describe(&handler, &broca, "broca", "s").await
13582 else {
13583 panic!("not a describe reply");
13584 };
13585 assert!(scope.unwrap().owner_authorized);
13586 }
13587
13588 #[tokio::test]
13592 async fn scope_ops_and_the_scopes_capability_are_advertised() {
13593 let handler = ControlHandler::new(Arc::new(Registry::default()));
13594 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
13595 let ack = hello_via_sink(
13596 &handler,
13597 &ctx,
13598 &mut rx,
13599 hello_frame("m", PROTOCOL_VERSION, 1),
13600 )
13601 .await;
13602 let ack = parse_ack(&ack);
13603 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
13604 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
13605 }
13606 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
13607
13608 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
13609 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13610 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13611 let reply = handler
13612 .handle_control_frame(&client, frame)
13613 .await
13614 .unwrap()
13615 .pop()
13616 .unwrap();
13617 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13618 serde_json::from_slice(&reply.body).unwrap()
13619 else {
13620 panic!("not a server.describe reply");
13621 };
13622 assert!(
13623 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
13624 "{capabilities:?}"
13625 );
13626 }
13627
13628 const PLEXUS: &str = "plexus";
13631 const OTHER: &str = "other";
13632 const AFT: &str = "aft";
13633 const BROCA: &str = "broca";
13634 const MAGIC: &str = "magic-context";
13635
13636 fn nonce(module_id: &str) -> String {
13637 format!("nonce-{module_id}")
13638 }
13639
13640 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
13641 let (tx, rx) = mpsc::channel(64);
13642 (
13643 RouteCtx {
13644 connection_id: ConnectionId::new(connection),
13645 egress: FrameSink::new(tx),
13646 },
13647 rx,
13648 )
13649 }
13650
13651 struct Rig {
13655 handler: ControlHandler,
13656 forwarding: Arc<ForwardingTable>,
13657 owner: RouteCtx,
13658 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13659 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
13660 generation: u64,
13661 next_connection: u64,
13662 _supervisor: Supervisor,
13663 }
13664
13665 async fn rig() -> Rig {
13666 rig_with_flow_support(true).await
13667 }
13668
13669 async fn rig_with_flow_support(flow_support: bool) -> Rig {
13670 rig_with_scope_capabilities(flow_support, false).await
13671 }
13672
13673 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
13674 let registry = Arc::new(Registry::default());
13675 let forwarding = Arc::new(ForwardingTable::default());
13676 let supervisor_handle = SupervisorHandle::new();
13677 let supervisor =
13678 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
13679 .with_handle(supervisor_handle.clone());
13680 supervisor
13681 .supervise_configured(
13682 ModuleSpec {
13683 module_id: OWNER.to_string(),
13684 program: PathBuf::from("/nonexistent/prefrontal-core"),
13685 args: Vec::new(),
13686 env: Vec::new(),
13687 reserved: false,
13688 reserved_prefixes: Vec::new(),
13689 protocol: ModuleProtocol::Subc,
13690 overlap: Default::default(),
13691 },
13692 false,
13693 )
13694 .unwrap();
13695 for module_id in [OWNER, AFT, BROCA, MAGIC] {
13696 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
13697 }
13698 let handler =
13699 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13700 .with_supervisor(supervisor_handle);
13701 let (owner, mut owner_rx) = wide_ctx(1);
13702 hello_via_sink(
13703 &handler,
13704 &owner,
13705 &mut owner_rx,
13706 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
13707 )
13708 .await;
13709 let mut modules = BTreeMap::new();
13710 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
13711 let (ctx, mut rx) = wide_ctx(connection);
13712 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
13713 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
13714 body["manifest"]["provenance"] =
13718 serde_json::json!({"wire_crate_version": "0.29.0"});
13719 if flow_support || run_support {
13720 let mut provides = Vec::new();
13721 if flow_support {
13722 provides.push("flow-scopes/v1");
13723 }
13724 if run_support {
13725 provides.push("agent-run-scopes/v1");
13726 }
13727 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
13728 }
13729 let hello = Frame::build(
13730 FrameType::Hello,
13731 control_flags(),
13732 0,
13733 0,
13734 connection,
13735 serde_json::to_vec(&body).unwrap(),
13736 )
13737 .unwrap();
13738 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
13739 modules.insert(module_id.to_string(), (ctx, rx));
13740 }
13741 Rig {
13742 handler,
13743 forwarding,
13744 owner,
13745 _owner_rx: owner_rx,
13746 modules,
13747 generation: 0,
13748 next_connection: 100,
13749 _supervisor: supervisor,
13750 }
13751 }
13752
13753 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
13754 ScopeCarrier::new(Principal::Reserved {
13755 module_id: module_id.to_string(),
13756 })
13757 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
13758 }
13759
13760 fn session(scope_epoch: u64) -> ScopeRecord {
13763 let mut record = head("s", scope_epoch);
13764 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
13765 record.attributes.agent_id = Some("agent-1".to_string());
13766 record.attributes.delegates = true;
13767 record
13768 }
13769
13770 impl Rig {
13771 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
13772 self.generation += 1;
13773 sync(&self.handler, &self.owner, self.generation, scopes)
13774 .await
13775 .expect("the owner's sync is accepted");
13776 }
13777
13778 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
13779 ScopeSelector {
13780 owner: Principal::Reserved {
13781 module_id: OWNER.to_string(),
13782 },
13783 scope_ref: scope_ref.to_string(),
13784 scope_epoch,
13785 }
13786 }
13787
13788 fn open_frame(
13789 &mut self,
13790 opener: Option<&str>,
13791 target: &str,
13792 scope: Option<ScopeSelector>,
13793 ) -> (
13794 RouteCtx,
13795 mpsc::Receiver<crate::router::OutboundFrame>,
13796 Frame,
13797 ) {
13798 self.next_connection += 1;
13799 let (ctx, rx) = wide_ctx(self.next_connection);
13800 let root = unique_project_root("scoped-open");
13801 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
13802 target: RouteTarget::ToolProvider {
13803 module_id: target.to_string(),
13804 },
13805 identity: BindIdentity::new(
13806 root.path().to_path_buf(),
13807 "unit".to_string(),
13808 "session".to_string(),
13809 ),
13810 consumer_identity: opener.map(|module_id| ConsumerIdentity {
13811 module_id: module_id.to_string(),
13812 launch_nonce: nonce(module_id),
13813 }),
13814 consumer_capabilities: None,
13815 role_versions: None,
13816 admission_facts: None,
13817 scope,
13818 })
13819 .unwrap();
13820 let frame = Frame::build(
13821 FrameType::Request,
13822 control_flags(),
13823 0,
13824 0,
13825 self.next_connection,
13826 body,
13827 )
13828 .unwrap();
13829 (ctx, rx, frame)
13830 }
13831
13832 async fn refused(
13834 &mut self,
13835 opener: Option<&str>,
13836 target: &str,
13837 scope: Option<ScopeSelector>,
13838 ) -> String {
13839 self.refusal_body(opener, target, scope).await["code"]
13840 .as_str()
13841 .unwrap()
13842 .to_string()
13843 }
13844
13845 async fn refusal_body(
13846 &mut self,
13847 opener: Option<&str>,
13848 target: &str,
13849 scope: Option<ScopeSelector>,
13850 ) -> Value {
13851 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
13852 let replies = tokio::time::timeout(
13853 Duration::from_secs(2),
13854 self.handler.handle_control_frame(&ctx, frame),
13855 )
13856 .await
13857 .expect("the open must be refused before waiting for a bind ack")
13858 .unwrap();
13859 assert_eq!(replies.len(), 1, "{replies:?}");
13860 assert_eq!(replies[0].header.ty, FrameType::Error);
13861 let (_, module_rx) = self.modules.get_mut(target).unwrap();
13862 assert!(
13863 module_rx.try_recv().is_err(),
13864 "a refused open relays nothing"
13865 );
13866 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
13867 parse_error(&replies[0])
13868 }
13869
13870 async fn relayed(
13872 &mut self,
13873 opener: Option<&str>,
13874 target: &str,
13875 scope: Option<ScopeSelector>,
13876 ) -> Relayed {
13877 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
13878 let handler = self.handler.clone();
13879 let task_ctx = ctx.clone();
13880 let task = tokio::spawn(async move {
13881 handler
13882 .handle_control_frame(&task_ctx, frame)
13883 .await
13884 .unwrap()
13885 });
13886 let (_, module_rx) = self.modules.get_mut(target).unwrap();
13887 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
13888 .await
13889 .expect("the target receives the relayed route.bind")
13890 .unwrap()
13891 .frame;
13892 Relayed {
13893 target: target.to_string(),
13894 client: ctx,
13895 client_rx: rx,
13896 task,
13897 bind,
13898 }
13899 }
13900
13901 async fn ack(&self, relayed: &Relayed) {
13902 let (module, _) = &self.modules[&relayed.target];
13903 self.handler
13904 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
13905 .await
13906 .unwrap();
13907 }
13908
13909 async fn bound(
13911 &mut self,
13912 opener: Option<&str>,
13913 target: &str,
13914 scope: Option<ScopeSelector>,
13915 ) -> Bound {
13916 let relayed = self.relayed(opener, target, scope).await;
13917 self.ack(&relayed).await;
13918 let Relayed {
13919 target,
13920 client,
13921 mut client_rx,
13922 task,
13923 bind,
13924 } = relayed;
13925 assert!(
13926 task.await.unwrap().is_empty(),
13927 "the open is answered by commit"
13928 );
13929 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
13930 Bound {
13931 target,
13932 client,
13933 client_rx,
13934 channel,
13935 epoch,
13936 bind,
13937 }
13938 }
13939
13940 fn live(&self, route: &Bound) -> bool {
13941 matches!(
13942 self.forwarding
13943 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
13944 .unwrap(),
13945 DataRoute::Client(DataRouteState::Bound(_))
13946 )
13947 }
13948 }
13949
13950 struct Relayed {
13951 target: String,
13952 client: RouteCtx,
13953 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13954 task: tokio::task::JoinHandle<Vec<Frame>>,
13955 bind: Frame,
13956 }
13957
13958 struct Bound {
13959 target: String,
13960 client: RouteCtx,
13961 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13962 channel: u16,
13963 epoch: u32,
13964 bind: Frame,
13965 }
13966
13967 impl Bound {
13968 fn closed_reason(&mut self) -> RouteCloseReason {
13971 let mut reason = None;
13972 let mut goodbye = false;
13973 while let Ok(outbound) = self.client_rx.try_recv() {
13974 let frame = outbound.frame;
13975 match frame.header.ty {
13976 FrameType::Goodbye => {
13977 assert_eq!(
13978 (frame.header.channel, frame.header.epoch),
13979 (self.channel, self.epoch)
13980 );
13981 goodbye = true;
13982 }
13983 FrameType::Push => {
13984 let ClientControlPush::RouteClosed {
13985 reason: r,
13986 module_id,
13987 ..
13988 } = serde_json::from_slice(&frame.body).unwrap()
13989 else {
13990 panic!("unexpected push");
13991 };
13992 assert_eq!(module_id, self.target);
13993 reason = Some(r);
13994 }
13995 other => panic!("unexpected frame {other:?}"),
13996 }
13997 }
13998 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
13999 reason.expect("the client is told why the route closed")
14000 }
14001
14002 fn untouched(&mut self) -> bool {
14003 self.client_rx.try_recv().is_err()
14004 }
14005
14006 fn stamp(&self) -> Option<ScopeStamp> {
14007 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
14008 ModuleControlRequest::RouteBind { scope, .. } => scope,
14009 other => panic!("expected a route.bind, got {other:?}"),
14010 }
14011 }
14012 }
14013
14014 #[tokio::test]
14015 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
14016 ) {
14017 let mut rig = rig().await;
14018 let mut record = session(1);
14019 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
14020 record.child_owners = vec![Principal::Reserved {
14021 module_id: MAGIC.to_string(),
14022 }];
14023 rig.sync(vec![record]).await;
14024 let scope = || Some(rig_selector("s", Some(1)));
14025
14026 rig.bound(Some(OWNER), PLEXUS, scope()).await;
14029 rig.bound(Some(AFT), OTHER, scope()).await;
14030 rig.bound(Some(BROCA), PLEXUS, scope()).await;
14031
14032 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
14036 assert_eq!(
14037 rig.refused(opener, target, scope()).await,
14038 error_codes::SCOPE_NOT_CARRIER,
14039 "{opener:?} -> {target}"
14040 );
14041 }
14042 }
14043
14044 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
14045 ScopeSelector {
14046 owner: Principal::Reserved {
14047 module_id: OWNER.to_string(),
14048 },
14049 scope_ref: scope_ref.to_string(),
14050 scope_epoch,
14051 }
14052 }
14053
14054 #[tokio::test]
14055 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
14056 let mut rig = rig().await;
14057 rig.sync(vec![session(1)]).await;
14058 for opener in [OWNER, AFT] {
14059 assert_eq!(
14060 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
14061 .await,
14062 error_codes::SCOPE_EPOCH_REQUIRED,
14063 "{opener}"
14064 );
14065 }
14066 }
14067
14068 #[tokio::test]
14069 async fn admission_separates_not_synced_not_live_and_ended() {
14070 let mut rig = rig().await;
14071 let code = rig
14073 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14074 .await;
14075 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
14076 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
14077
14078 let ghost = ScopeSelector {
14080 owner: Principal::Reserved {
14081 module_id: "ghost".to_string(),
14082 },
14083 scope_ref: "s".to_string(),
14084 scope_epoch: Some(1),
14085 };
14086 assert_eq!(
14087 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
14088 error_codes::SCOPE_NOT_LIVE
14089 );
14090
14091 rig.sync(vec![session(2)]).await;
14092 assert_eq!(
14093 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
14094 .await,
14095 error_codes::SCOPE_NOT_LIVE
14096 );
14097 for epoch in [1, 3] {
14098 assert_eq!(
14099 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
14100 .await,
14101 error_codes::SCOPE_ENDED,
14102 "epoch {epoch}"
14103 );
14104 }
14105 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
14107 .await;
14108 }
14109
14110 #[tokio::test]
14111 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
14112 let mut rig = rig().await;
14113 rig.sync(vec![session(1)]).await;
14114 let route = rig
14115 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14116 .await;
14117 let stamp = route.stamp().expect("a scoped bind carries the stamp");
14118 assert_eq!(stamp.scope_ref, "s");
14119 assert_eq!(stamp.scope_epoch, 1);
14120 assert_eq!(stamp.kind, ScopeKind::Head);
14121 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
14122 assert!(stamp.attributes.delegates);
14123 assert!(stamp.owner_authorized);
14124 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
14125 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
14126
14127 let (broca, mut broca_rx) = wide_ctx(50);
14130 hello_via_sink(
14131 &rig.handler,
14132 &broca,
14133 &mut broca_rx,
14134 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
14135 )
14136 .await;
14137 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
14138 .await
14139 .unwrap();
14140 let own = ScopeSelector {
14141 owner: Principal::Reserved {
14142 module_id: BROCA.to_string(),
14143 },
14144 scope_ref: "b".to_string(),
14145 scope_epoch: Some(1),
14146 };
14147 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
14148 assert!(!route.stamp().unwrap().owner_authorized);
14149 }
14150
14151 #[tokio::test]
14152 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
14153 let mut rig = rig().await;
14154 let mut record = head("s", 1);
14155 record.carriers = vec![carrier(AFT, None)];
14156 let flow_id = "Flow:run-7/step_2!~";
14157 record.attributes.flow_id = Some(flow_id.to_string());
14158 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
14159 .await
14160 .unwrap();
14161 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
14162 panic!("not a sync reply");
14163 };
14164 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
14165 let route = rig
14166 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14167 .await;
14168 assert!(rig.live(&route), "the stamped bind committed");
14169 let stamp = route.stamp().expect("a flow scope carries a stamp");
14170 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
14171 assert_eq!(stamp.attributes.agent_id, None);
14172 assert!(!stamp.attributes.delegates);
14173 assert!(stamp.owner_authorized);
14174 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
14175 assert_eq!(unscoped.stamp(), None);
14176 }
14177
14178 #[tokio::test]
14179 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
14180 let mut rig = rig_with_flow_support(false).await;
14181 let mut record = session(1);
14182 record.attributes.flow_id = Some("flow:7".to_string());
14183 rig.sync(vec![record]).await;
14184 for opener in [OWNER, AFT] {
14185 let body = rig
14186 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
14187 .await;
14188 assert_eq!(body["code"], "target_flow_unsupported");
14189 let message = body["message"].as_str().unwrap();
14190 for required in [PLEXUS, "flow-scopes/v1"] {
14191 assert!(message.contains(required), "{message}");
14192 }
14193 }
14194 }
14195
14196 #[tokio::test]
14197 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
14198 let mut rig = rig_with_flow_support(true).await;
14199 let mut record = session(1);
14200 record.attributes.flow_id = Some("flow:7".to_string());
14201 rig.sync(vec![record]).await;
14202 for opener in [OWNER, AFT] {
14203 let route = rig
14204 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
14205 .await;
14206 assert!(rig.live(&route));
14207 assert_eq!(
14208 route.stamp().unwrap().attributes.flow_id.as_deref(),
14209 Some("flow:7")
14210 );
14211 }
14212 }
14213
14214 #[tokio::test]
14215 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
14216 use std::future::Future;
14217
14218 let mut rig = rig_with_flow_support(true).await;
14219 let mut record = session(1);
14220 record.attributes.flow_id = Some("flow:7".to_string());
14221 rig.sync(vec![record]).await;
14222 let (client, mut client_rx, frame) =
14223 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
14224 for _ in 0..64 {
14227 client.egress.try_send(route_bind_ack(1)).unwrap();
14228 }
14229 let handler = rig.handler.clone();
14230 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
14231 std::future::poll_fn(|cx| {
14232 assert!(open.as_mut().poll(cx).is_pending());
14233 std::task::Poll::Ready(())
14234 })
14235 .await;
14236 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
14237
14238 let old_connection = rig.modules[PLEXUS].0.connection_id;
14239 rig.handler.cleanup_connection(old_connection).unwrap();
14240 let (replacement, mut replacement_rx) = wide_ctx(200);
14241 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
14242 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
14243 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
14244 let hello = Frame::build(
14245 FrameType::Hello,
14246 control_flags(),
14247 0,
14248 0,
14249 200,
14250 serde_json::to_vec(&body).unwrap(),
14251 )
14252 .unwrap();
14253 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
14254
14255 client_rx.try_recv().unwrap();
14256 let replies = tokio::time::timeout(Duration::from_secs(2), open)
14257 .await
14258 .expect("the replacement is refused without waiting for a bind ack")
14259 .unwrap();
14260 assert_eq!(replies.len(), 1);
14261 let body = parse_error(&replies[0]);
14262 assert_eq!(body["code"], "target_flow_unsupported");
14263 for required in [PLEXUS, "flow-scopes/v1"] {
14264 assert!(body["message"].as_str().unwrap().contains(required));
14265 }
14266 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
14267 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
14268 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
14269 assert!(rig
14270 .handler
14271 .registry
14272 .get_module_by_connection(replacement.connection_id)
14273 .unwrap()
14274 .is_some());
14275 }
14276
14277 #[tokio::test]
14278 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
14279 {
14280 let mut rig = rig_with_flow_support(false).await;
14281 rig.sync(vec![session(1)]).await;
14282 let route = rig
14283 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14284 .await;
14285 assert!(rig.live(&route));
14286 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
14287 let mut record = session(1);
14288 record.attributes.flow_id = Some("flow:7".to_string());
14289 rig.sync(vec![record]).await;
14290 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
14291 assert!(rig.live(&unscoped));
14292 assert_eq!(unscoped.stamp(), None);
14293 }
14294
14295 #[tokio::test]
14296 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
14297 let mut rig = rig().await;
14298 let mut record = session(1);
14299 record.attributes.flow_id = Some("flow:7".to_string());
14300 rig.sync(vec![record.clone()]).await;
14301 let before = rig.forwarding.published_scope_tag(OWNER, "s");
14302 let mut owner_route = rig
14303 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
14304 .await;
14305 let mut carrier_route = rig
14306 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
14307 .await;
14308 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
14309 rig.sync(vec![record.clone()]).await;
14310 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
14311 assert!(rig.live(&owner_route) && owner_route.untouched());
14312 assert!(rig.live(&carrier_route) && carrier_route.untouched());
14313
14314 record.attributes.flow_id = Some("flow:8".to_string());
14315 rig.sync(vec![record]).await;
14316 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
14317 let before = before.unwrap();
14318 assert_eq!(after.scope_epoch, before.scope_epoch);
14319 assert!(after.version > before.version);
14320 for route in [&mut owner_route, &mut carrier_route] {
14321 assert!(!rig.live(route));
14322 assert_eq!(
14323 route.closed_reason(),
14324 RouteCloseReason::ScopeDelegationChanged
14325 );
14326 }
14327 assert!(rig.live(&unscoped) && unscoped.untouched());
14328 for target in [PLEXUS, OTHER] {
14331 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
14332 let goodbye = module_rx
14333 .try_recv()
14334 .expect("the provider sees the drain")
14335 .frame;
14336 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14337 assert!(module_rx.try_recv().is_err());
14338 }
14339 let rebound = rig
14340 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14341 .await;
14342 assert_eq!(
14343 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
14344 Some("flow:8")
14345 );
14346 }
14347
14348 #[tokio::test]
14353 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
14354 let mut rig = rig().await;
14355 rig.sync(vec![session(1)]).await;
14356 let mut cotenant = rig
14357 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
14358 .await;
14359
14360 let mut changed = session(1);
14361 changed.child_owners.push(Principal::Reserved {
14362 module_id: MAGIC.to_string(),
14363 });
14364 let mut ended = None;
14365 for (code, next) in [
14366 (error_codes::SCOPE_CHANGED, vec![changed]),
14367 (error_codes::SCOPE_ENDED, Vec::new()),
14368 ] {
14369 let relayed = rig
14370 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14371 .await;
14372 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
14373 ended = Some(next.is_empty());
14374 rig.sync(next).await;
14375 rig.ack(&relayed).await;
14376 let replies = relayed.task.await.unwrap();
14377 assert_eq!(replies.len(), 1, "{replies:?}");
14378 assert_eq!(parse_error(&replies[0])["code"], code);
14379 assert_eq!(
14380 subc_protocol::error_codes::is_retryable_route_open(code),
14381 code == error_codes::SCOPE_CHANGED
14382 );
14383 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
14384 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
14386 let mut goodbyes = Vec::new();
14389 while let Ok(outbound) = plexus_rx.try_recv() {
14390 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
14391 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
14392 }
14393 assert!(
14394 goodbyes.contains(&(bind_channel, bind_epoch)),
14395 "{goodbyes:?}"
14396 );
14397 assert!(rig
14398 .handler
14399 .registry
14400 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
14401 .unwrap()
14402 .is_some());
14403 }
14404 assert_eq!(ended, Some(true));
14405 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
14408 }
14409
14410 #[tokio::test]
14413 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
14414 struct Case {
14415 name: &'static str,
14416 change: fn(&mut ScopeRecord),
14417 closed: [Option<RouteCloseReason>; 4],
14420 }
14421 use RouteCloseReason::*;
14422 let cases = [
14423 Case {
14424 name: "a carrier entry removed",
14425 change: |r| {
14426 r.carriers.retain(|c| {
14427 c.principal
14428 != Principal::Reserved {
14429 module_id: AFT.to_string(),
14430 }
14431 })
14432 },
14433 closed: [None, Some(ScopeCarrierRemoved), None, None],
14434 },
14435 Case {
14436 name: "a target removed from a carrier",
14437 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
14438 closed: [None, None, None, Some(ScopeCarrierRemoved)],
14439 },
14440 Case {
14441 name: "a bare carrier narrowed to targets",
14442 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
14443 closed: [None, Some(ScopeCarrierRemoved), None, None],
14444 },
14445 Case {
14446 name: "delegates turned off",
14447 change: |r| r.attributes.delegates = false,
14448 closed: [Some(ScopeDelegationChanged); 4],
14449 },
14450 Case {
14451 name: "agent_id changed",
14452 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
14453 closed: [Some(ScopeDelegationChanged); 4],
14454 },
14455 Case {
14456 name: "a carrier added, child owners changed, the record re-sent",
14457 change: |r| {
14458 r.carriers.push(carrier(MAGIC, None));
14459 r.child_owners.push(Principal::Reserved {
14460 module_id: MAGIC.to_string(),
14461 });
14462 },
14463 closed: [None; 4],
14464 },
14465 Case {
14466 name: "a target added",
14467 change: |r| {
14468 r.carriers[1]
14469 .targets
14470 .as_mut()
14471 .unwrap()
14472 .push("third".to_string())
14473 },
14474 closed: [None; 4],
14475 },
14476 Case {
14477 name: "delegates turned on",
14478 change: |r| r.attributes.delegates = true,
14479 closed: [None; 4],
14480 },
14481 ];
14482 for case in cases {
14483 let mut rig = rig().await;
14484 rig.sync(vec![session(1)]).await;
14485 let scope = || Some(rig_selector("s", Some(1)));
14486 let mut routes = [
14487 rig.bound(Some(OWNER), PLEXUS, scope()).await,
14488 rig.bound(Some(AFT), PLEXUS, scope()).await,
14489 rig.bound(Some(BROCA), PLEXUS, scope()).await,
14490 rig.bound(Some(BROCA), OTHER, scope()).await,
14491 ];
14492 let mut record = session(1);
14493 (case.change)(&mut record);
14494 rig.sync(vec![record]).await;
14495 for (index, expected) in case.closed.iter().enumerate() {
14496 let route = &mut routes[index];
14497 match expected {
14498 Some(reason) => {
14499 assert!(!rig.live(route), "{}: route {index} still live", case.name);
14500 assert_eq!(
14501 route.closed_reason(),
14502 *reason,
14503 "{}: route {index}",
14504 case.name
14505 );
14506 }
14507 None => {
14508 assert!(rig.live(route), "{}: route {index} closed", case.name);
14509 assert!(
14510 route.untouched(),
14511 "{}: route {index} was told something",
14512 case.name
14513 );
14514 }
14515 }
14516 }
14517 }
14518 }
14519
14520 #[tokio::test]
14521 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
14522 for next in [Vec::new(), vec![session(2)]] {
14524 let mut rig = rig().await;
14525 rig.sync(vec![session(1)]).await;
14526 let mut route = rig
14527 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14528 .await;
14529 rig.sync(next).await;
14530 assert!(!rig.live(&route));
14531 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
14532 }
14533
14534 let mut rig = rig().await;
14537 let mut child = session(1);
14538 child.scope_ref = "child".to_string();
14539 child.kind = ScopeKind::Worker;
14540 child.parent = Some(ScopeParent::new(
14541 Principal::Reserved {
14542 module_id: OWNER.to_string(),
14543 },
14544 "s".to_string(),
14545 1,
14546 ));
14547 rig.sync(vec![session(1), child.clone()]).await;
14548 let mut child_route = rig
14549 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
14550 .await;
14551 assert_eq!(
14552 child_route.stamp().unwrap().parent_state,
14553 Some(ParentState::Linked)
14554 );
14555 rig.sync(vec![child]).await;
14556 assert!(!rig.live(&child_route));
14557 assert_eq!(
14558 child_route.closed_reason(),
14559 RouteCloseReason::ScopeParentEnded
14560 );
14561 }
14562
14563 #[tokio::test]
14564 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
14565 ) {
14566 let mut rig = rig().await;
14567 rig.sync(vec![session(1)]).await;
14568 let mut route = rig
14569 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14570 .await;
14571 let before = rig.forwarding.published_scope_tag(OWNER, "s");
14572 rig.sync(vec![session(1)]).await;
14573 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
14574 assert!(rig.live(&route) && route.untouched());
14575
14576 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
14580 .forwarding
14581 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
14582 .unwrap()
14583 else {
14584 panic!("the route is bound");
14585 };
14586 binding.flow.acquire_tagged(9, false).await.unwrap();
14587 let mut widened = session(1);
14588 widened.carriers.push(carrier(MAGIC, None));
14589 rig.sync(vec![widened]).await;
14590 assert!(rig.live(&route) && route.untouched());
14591 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
14592 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
14593 assert_eq!(binding.flow.in_flight(), 1);
14596 binding
14597 .flow
14598 .acquire_tagged(10, false)
14599 .await
14600 .expect("the flow is still open");
14601 }
14602
14603 #[tokio::test]
14606 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
14607 let mut rig = rig().await;
14608 rig.sync(vec![session(1)]).await;
14609 let mut on_incumbent = rig
14610 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14611 .await;
14612
14613 let (candidate, _candidate_rx) = wide_ctx(9);
14616 let registration = rig
14617 .handler
14618 .registry
14619 .register_candidate_with_control_ops(
14620 manifest(PLEXUS, PROTOCOL_VERSION),
14621 PROTOCOL_VERSION,
14622 candidate.connection_id,
14623 module_baseline_control_ops(),
14624 )
14625 .unwrap();
14626 rig.forwarding
14627 .register_candidate_module_connection(
14628 candidate.connection_id,
14629 PLEXUS.to_string(),
14630 PROTOCOL_VERSION,
14631 manifest_concurrency(®istration.manifest),
14632 candidate.egress.clone(),
14633 )
14634 .unwrap();
14635 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
14636 rig.handler
14637 .registry
14638 .promote_candidate(PLEXUS)
14639 .unwrap()
14640 .unwrap();
14641 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
14642
14643 rig.sync(Vec::new()).await;
14644 assert!(!rig.live(&on_incumbent));
14645 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
14646 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
14647 let goodbye = incumbent_rx
14648 .try_recv()
14649 .expect("the superseded endpoint is told")
14650 .frame;
14651 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14652 }
14653 }
14654}
14655
14656#[cfg(test)]
14657mod concurrency_default_exposure_tests {
14658 use super::*;
14659
14660 fn hello_body(role_json: &str) -> Vec<u8> {
14661 format!(
14662 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
14663 )
14664 .into_bytes()
14665 }
14666
14667 fn manifest_from(body: &[u8]) -> ModuleManifest {
14668 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
14669 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
14670 .expect("manifest parses")
14671 }
14672
14673 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
14674
14675 #[test]
14676 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
14677 let body = hello_body(&format!(
14678 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
14679 ));
14680 let manifest = manifest_from(&body);
14681 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
14684 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
14685 }
14686
14687 #[test]
14688 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
14689 let body = hello_body(&format!(
14690 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
14691 ));
14692 let manifest = manifest_from(&body);
14693 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
14694 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
14695 }
14696
14697 #[test]
14698 fn non_management_roles_are_never_reported() {
14699 let body = hello_body(
14700 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
14701 );
14702 let manifest = manifest_from(&body);
14703 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
14704 }
14705}