1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex, RwLock},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 error_codes,
21 manifest::{
22 validate_hello_capability_grammar, validate_hello_event_declarations,
23 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
24 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
25 },
26 scope::{
27 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
28 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
29 SCOPE_SYNC_OP,
30 },
31 session::{
32 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
33 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
34 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
35 ROLE_VERSIONS_FIELD,
36 },
37 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
38 Priority, RouteTarget, PROTOCOL_VERSION,
39};
40use tokio::time::{timeout_at, Instant};
41use tracing::{debug, info, warn};
42
43use crate::{
44 capability_requirements::{
45 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
46 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
47 RuntimeModule,
48 },
49 daemon_config::RestartRequiredSection,
50 forwarding::{
51 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
52 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
53 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
54 },
55 observability::{
56 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
57 },
58 provenance::{
59 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
60 },
61 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
62 router::{RouteCtx, RouterError},
63 scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
64 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
65 stderr_tail::{CaptureState, TailEntry},
66 supervise::{
67 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
68 SupervisorHandle, SwapHelloAdmission,
69 },
70 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
71};
72
73pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
79
80const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
81const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
82const CAP_PING_PONG: &str = "ping_pong_v1";
83const CAP_SESSION_ATTACH: &str = "session_attach_v1";
84const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
85
86const SUBC_CONTROL_OPS: &[&str] = &[
87 ops::ACTIVITY_RECORD,
88 ops::SERVER_DESCRIBE,
89 ops::CATALOG_LIST,
90 ops::ROUTE_OPEN,
91 ops::ROUTE_POLL,
92 ops::ROUTE_CLOSING,
93 ops::ROUTE_CLOSED,
94 ops::SUPERVISOR_LIST,
95 ops::SUPERVISOR_RESTART,
96 ops::SUPERVISOR_SWAP,
97 ops::SUPERVISOR_RELOAD,
98 ops::SUPERVISOR_RESCAN,
99 ops::SUPERVISOR_RELEASE_RESERVED,
100 ops::SUPERVISOR_SET_ENABLED,
101 ops::SUPERVISOR_HEALTH_PROBE,
102 ops::SUPERVISOR_HEALTH,
103 ops::SUPERVISOR_STDERR_TAIL,
104 ops::SUPERVISOR_TERMINALS,
105 ops::SUPERVISOR_ROUTES,
106 ops::SUPERVISOR_PROVENANCE,
107 ops::SUPERVISOR_SPAWN_SNAPSHOT,
108 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
109];
110
111const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
112 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
113 "supervisor.live_roots",
114 SCOPE_SYNC_OP,
115 SCOPE_APPLY_OP,
116 SCOPE_DESCRIBE_OP,
117 "operator.confirm",
118];
119
120const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
124
125const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
126
127pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
135
136pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
151
152pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
170
171const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
172const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
173
174fn reload_verdict(
175 configured: &Path,
176 spawned_from: Option<&Path>,
177 image: subc_control::RunningImageAgreement,
178) -> PendingReloadVerdict {
179 let path = match spawned_from {
180 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
181 ReloadPathAgreement::Match
182 }
183 Some(spawned_from) => ReloadPathAgreement::Mismatch {
184 configured: configured.to_path_buf(),
185 spawned_from: spawned_from.to_path_buf(),
186 },
187 None => ReloadPathAgreement::Unavailable {
188 reason: if matches!(
189 image,
190 subc_control::RunningImageAgreement::Unavailable {
191 reason: subc_control::RunningImageUnavailableReason::NotRunning
192 }
193 ) {
194 ReloadPathUnavailableReason::NotRunning
195 } else {
196 ReloadPathUnavailableReason::SpawnedPathUnavailable
197 },
198 },
199 };
200 PendingReloadVerdict { path, image }
201}
202
203fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
204 if configured == spawned_from {
205 return true;
206 }
207 #[cfg(windows)]
208 {
209 match (
215 std::fs::canonicalize(configured),
216 std::fs::canonicalize(spawned_from),
217 ) {
218 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
219 _ => false,
220 }
221 }
222 #[cfg(not(windows))]
223 {
224 false
225 }
226}
227
228#[derive(Clone)]
229struct DaemonProvenanceFacts {
230 build: DaemonBuildProvenance,
231 pid: Option<u32>,
232 started_at_ms: Option<u64>,
233 start_clock: Option<crate::clock::StartClock>,
234 executable_path: Option<PathBuf>,
235 executable_identity: Option<SpawnedFileIdentity>,
236 process_start_time: Option<u64>,
237 probe: ExecutableIdentityProbe,
238}
239
240impl Default for DaemonProvenanceFacts {
241 fn default() -> Self {
242 Self {
243 build: DaemonBuildProvenance {
244 build_git_sha: None,
245 build_lock_digest: None,
246 },
247 pid: None,
248 started_at_ms: None,
249 start_clock: None,
250 executable_path: None,
251 executable_identity: None,
252 process_start_time: None,
253 probe: ExecutableIdentityProbe::default(),
254 }
255 }
256}
257
258#[derive(Debug, Clone)]
259struct SupervisorRescanContext {
260 supervisor: Supervisor,
261 config_path: PathBuf,
262 configured_port: Option<u16>,
263 storage_config: Option<crate::daemon_config::StorageConfig>,
264 admission_facts_carrier_module_id: Option<String>,
265 admission_facts_targets: Option<Vec<String>>,
266 scope_authority_owners: Vec<String>,
267}
268
269const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
290 "reloading",
291 "supervisor_not_live",
292 "registration_not_active",
293 "no_forwarding_connection",
294 "relay_send_failed",
295];
296
297#[derive(Clone)]
299pub struct ControlHandler {
300 registry: Arc<Registry>,
301 activity: Option<crate::activity::ActivityLedger>,
302 forwarding: Arc<ForwardingTable>,
303 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
304 supervisor: SupervisorHandle,
305 subc_capabilities: Arc<[String]>,
306 route_bind_relay_timeout: Duration,
310 route_bind_relay_timeouts: BTreeMap<String, Duration>,
314 route_bind_breakers: RouteBindBreakers,
317 route_bind_concurrency: RouteBindConcurrency,
320 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
325 route_bind_breaker_threshold: u32,
327 route_bind_breaker_cooldown: Duration,
329 health_probe_timeout: Duration,
330 storage_config: Option<crate::daemon_config::StorageConfig>,
333 machine_id: Option<crate::machine_id::MachineId>,
337 admission_facts_carrier_module_id: Option<String>,
338 admission_facts_targets: Option<Vec<String>>,
339 scopes: Arc<RwLock<ScopeTable>>,
343 scope_deadline_added: Arc<tokio::sync::Notify>,
347 scope_authority_owners: Vec<String>,
350 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
353 rescan: Option<SupervisorRescanContext>,
354 connected_clients: ConnectedClients,
355 counters: DaemonCounters,
356 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
357 daemon_provenance: DaemonProvenanceFacts,
358 #[cfg(test)]
359 control_dispatch_delay: Option<Duration>,
360 #[cfg(test)]
361 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
362 #[cfg(test)]
366 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
367}
368
369impl fmt::Debug for ControlHandler {
370 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
371 f.debug_struct("ControlHandler")
372 .field("registry", &self.registry)
373 .field("forwarding", &self.forwarding)
374 .field("process_liveness", &self.process_liveness.is_some())
375 .field("supervisor", &self.supervisor)
376 .field("subc_capabilities", &self.subc_capabilities)
377 .finish()
378 }
379}
380
381struct RouteOpenRequest {
382 target: RouteTarget,
383 identity: BindIdentity,
384 consumer_identity: Option<ConsumerIdentity>,
385 consumer_capabilities: Option<Vec<String>>,
386 role_versions: Option<BTreeMap<String, String>>,
387 admission_facts: Option<serde_json::Value>,
388 scope: Option<ScopeSelector>,
389}
390
391struct RouteBindReservationGuard {
392 forwarding: Arc<ForwardingTable>,
393 endpoint: ModuleEndpointId,
394 relay_corr: u64,
395 armed: bool,
396}
397
398struct ModuleControlRpcGuard {
399 forwarding: Arc<ForwardingTable>,
400 endpoint: ModuleEndpointId,
401 corr: u64,
402 armed: bool,
403}
404
405impl ModuleControlRpcGuard {
406 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
407 Self {
408 forwarding,
409 endpoint,
410 corr,
411 armed: true,
412 }
413 }
414
415 fn disarm(&mut self) {
416 self.armed = false;
417 }
418}
419
420impl Drop for ModuleControlRpcGuard {
421 fn drop(&mut self) {
422 if self.armed {
423 let _ = self
424 .forwarding
425 .cancel_module_control_rpc(self.endpoint, self.corr);
426 }
427 }
428}
429
430impl RouteBindReservationGuard {
431 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
432 Self {
433 forwarding,
434 endpoint,
435 relay_corr,
436 armed: true,
437 }
438 }
439
440 fn release_and_disarm(&mut self) {
441 if !self.armed {
442 return;
443 }
444 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
445 self.endpoint,
446 self.relay_corr,
447 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
448 ) {
449 send_goodbye_target_best_effort(
450 &self.forwarding.counters(),
451 &target,
452 "canceled route.bind",
453 );
454 }
455 self.armed = false;
456 }
457
458 fn disarm(&mut self) {
459 self.armed = false;
460 }
461}
462
463impl Drop for RouteBindReservationGuard {
464 fn drop(&mut self) {
465 self.release_and_disarm();
466 }
467}
468
469#[derive(Debug, Clone, Default)]
499pub(crate) struct RouteBindBreakers {
500 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
501}
502
503#[derive(Debug, Clone, Default)]
504pub(crate) struct RouteBindConcurrency {
505 modules: Arc<Mutex<HashMap<String, usize>>>,
506}
507
508struct RouteBindConcurrencyGuard {
509 concurrency: RouteBindConcurrency,
510 module_id: String,
511}
512
513impl RouteBindConcurrency {
514 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
517 let mut modules = self
518 .modules
519 .lock()
520 .expect("route.bind concurrency mutex poisoned");
521 let in_flight = modules.entry(module_id.to_string()).or_default();
522 if *in_flight >= limit {
523 return Err(*in_flight);
524 }
525 *in_flight += 1;
526 Ok(RouteBindConcurrencyGuard {
527 concurrency: self.clone(),
528 module_id: module_id.to_string(),
529 })
530 }
531}
532
533impl Drop for RouteBindConcurrencyGuard {
534 fn drop(&mut self) {
535 let mut modules = self
536 .concurrency
537 .modules
538 .lock()
539 .expect("route.bind concurrency mutex poisoned");
540 let remove = {
541 let in_flight = modules
542 .get_mut(&self.module_id)
543 .expect("admitted route.bind has a concurrency entry");
544 *in_flight -= 1;
545 *in_flight == 0
546 };
547 if remove {
548 modules.remove(&self.module_id);
549 }
550 }
551}
552
553#[derive(Debug, Default)]
554struct ModuleBreakerState {
555 consecutive_timeouts: u32,
557 cooldown_until: Option<Instant>,
560 probe_in_flight: Option<Arc<()>>,
565}
566
567enum RouteBindAdmission<'a> {
569 Admitted {
570 guard: RouteBindBreakerGuard<'a>,
571 probe: bool,
574 },
575 Refused {
576 consecutive_timeouts: u32,
577 retry_in: Duration,
581 probe_in_flight: bool,
582 },
583}
584
585struct RouteBindBreakerGuard<'a> {
592 breakers: RouteBindBreakers,
593 module_id: &'a str,
594 probe_token: Option<Arc<()>>,
595 settled: bool,
596}
597
598impl RouteBindBreakerGuard<'_> {
599 fn record_accepted(&mut self) -> bool {
603 self.settled = true;
604 self.breakers.record_accepted(self.module_id)
605 }
606
607 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
610 self.settled = true;
611 self.breakers.record_timeout(
612 self.module_id,
613 self.probe_token.as_ref(),
614 threshold,
615 cooldown,
616 )
617 }
618
619 fn record_inconclusive(&mut self) {
628 self.settled = true;
629 self.breakers
630 .record_inconclusive(self.module_id, self.probe_token.as_ref());
631 }
632}
633
634impl Drop for RouteBindBreakerGuard<'_> {
635 fn drop(&mut self) {
636 if !self.settled {
637 self.breakers
638 .record_inconclusive(self.module_id, self.probe_token.as_ref());
639 }
640 }
641}
642
643struct BreakerOpened {
647 consecutive_timeouts: u32,
648 reopened_after_probe: bool,
651}
652
653impl RouteBindBreakers {
654 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
655 self.modules
656 .lock()
657 .expect("route.bind breaker mutex poisoned")
658 }
659
660 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
663 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
664 probe: probe_token.is_some(),
665 guard: RouteBindBreakerGuard {
666 breakers: self.clone(),
667 module_id,
668 probe_token,
669 settled: false,
670 },
671 };
672
673 let mut modules = self.lock();
674 let Some(state) = modules.get_mut(module_id) else {
675 return admitted(None);
676 };
677 let Some(cooldown_until) = state.cooldown_until else {
678 return admitted(None);
679 };
680 if state.probe_in_flight.is_some() {
681 return RouteBindAdmission::Refused {
682 consecutive_timeouts: state.consecutive_timeouts,
683 retry_in: Duration::ZERO,
684 probe_in_flight: true,
685 };
686 }
687 let now = Instant::now();
688 if now < cooldown_until {
689 return RouteBindAdmission::Refused {
690 consecutive_timeouts: state.consecutive_timeouts,
691 retry_in: cooldown_until - now,
692 probe_in_flight: false,
693 };
694 }
695 let token = Arc::new(());
696 state.probe_in_flight = Some(Arc::clone(&token));
697 admitted(Some(token))
698 }
699
700 fn record_accepted(&self, module_id: &str) -> bool {
701 self.lock()
702 .remove(module_id)
703 .is_some_and(|state| state.cooldown_until.is_some())
704 }
705
706 fn record_timeout(
707 &self,
708 module_id: &str,
709 probe_token: Option<&Arc<()>>,
710 threshold: u32,
711 cooldown: Duration,
712 ) -> Option<BreakerOpened> {
713 let mut modules = self.lock();
714 let state = modules.entry(module_id.to_string()).or_default();
715 let was_open = state.cooldown_until.is_some();
716 let was_probe = Self::owns_probe(state, probe_token);
717 if was_probe {
718 state.probe_in_flight = None;
719 }
720 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
721 if state.consecutive_timeouts < threshold {
722 return None;
723 }
724 state.cooldown_until = Some(Instant::now() + cooldown);
725 Some(BreakerOpened {
726 consecutive_timeouts: state.consecutive_timeouts,
727 reopened_after_probe: was_open && was_probe,
728 })
729 }
730
731 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
732 state
736 .probe_in_flight
737 .as_ref()
738 .zip(token)
739 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
740 }
741
742 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
743 if let Some(state) = self.lock().get_mut(module_id) {
744 if Self::owns_probe(state, probe_token) {
745 state.probe_in_flight = None;
746 }
747 }
748 }
749
750 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
765 self.lock()
766 .remove(module_id)
767 .map(|state| state.consecutive_timeouts)
768 .filter(|discarded| *discarded > 0)
769 }
770
771 fn open_snapshot(&self) -> Option<serde_json::Value> {
778 let now = Instant::now();
779 let modules = self.lock();
780 let open = modules
781 .iter()
782 .filter_map(|(module_id, state)| {
783 let cooldown_until = state.cooldown_until?;
784 Some((
785 module_id.clone(),
786 serde_json::json!({
787 "consecutive_timeouts": state.consecutive_timeouts,
788 "cooldown_remaining_ms":
789 cooldown_until.saturating_duration_since(now).as_millis() as u64,
790 "probe_in_flight": state.probe_in_flight.is_some(),
791 }),
792 ))
793 })
794 .collect::<serde_json::Map<String, serde_json::Value>>();
795 (!open.is_empty()).then_some(serde_json::Value::Object(open))
796 }
797}
798
799impl ControlHandler {
800 pub fn new(registry: Arc<Registry>) -> Self {
801 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
802 }
803
804 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
805 let counters = forwarding.counters();
806 let route_bind_breakers = forwarding.route_bind_breakers();
810 let route_bind_concurrency = forwarding.route_bind_concurrency();
811 let route_outages = forwarding.route_outages();
812 Self {
813 registry,
814 activity: None,
815 forwarding,
816 process_liveness: None,
817 supervisor: SupervisorHandle::new(),
818 subc_capabilities: Arc::from([
819 CAP_MANIFEST_REGISTRATION.to_string(),
820 CAP_CHANNEL_LIFECYCLE.to_string(),
821 CAP_PING_PONG.to_string(),
822 CAP_SESSION_ATTACH.to_string(),
823 CAP_ADMISSION_FACTS_RELAY.to_string(),
824 CAP_SCOPES_V1.to_string(),
825 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
826 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
827 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
828 ]),
829 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
830 route_bind_relay_timeouts: BTreeMap::new(),
831 route_bind_breakers,
832 route_bind_concurrency,
833 route_outages,
834 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
835 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
836 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
837 storage_config: None,
838 machine_id: None,
839 admission_facts_carrier_module_id: None,
840 admission_facts_targets: None,
841 scopes: Arc::new(RwLock::new(ScopeTable::new(
842 crate::daemon_config::default_scope_authority_owners(),
843 ))),
844 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
845 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
846 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
847 rescan: None,
848 connected_clients: ConnectedClients::new(),
849 counters,
850 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
851 daemon_provenance: DaemonProvenanceFacts::default(),
852 #[cfg(test)]
853 control_dispatch_delay: None,
854 #[cfg(test)]
855 provenance_probe_override: None,
856 #[cfg(test)]
857 route_bind_relay_sent_hold: None,
858 }
859 }
860
861 pub fn with_storage_config(
864 mut self,
865 storage_config: Option<crate::daemon_config::StorageConfig>,
866 ) -> Self {
867 self.storage_config = storage_config;
868 self
869 }
870
871 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
874 self.activity = Some(crate::activity::ActivityLedger::open(
875 run_dir.join("activity.jsonl"),
876 incarnation,
877 ));
878 self
879 }
880
881 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
884 self.machine_id = machine_id;
885 self
886 }
887
888 pub fn with_admission_facts_config(
892 mut self,
893 carrier_module_id: Option<String>,
894 targets: Option<Vec<String>>,
895 ) -> Self {
896 self.admission_facts_carrier_module_id = carrier_module_id;
897 self.admission_facts_targets = targets;
898 self
899 }
900
901 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
905 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
906 self.scope_authority_owners = owners;
907 self
908 }
909
910 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
913 self.route_bind_relay_timeout = timeout;
914 self
915 }
916
917 pub fn with_route_bind_relay_timeouts(
923 mut self,
924 timeouts: impl IntoIterator<Item = (String, Duration)>,
925 ) -> Self {
926 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
927 self
928 }
929
930 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
936 self.route_bind_relay_timeouts
937 .get(module_id)
938 .copied()
939 .unwrap_or(self.route_bind_relay_timeout)
940 }
941
942 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
950 self.route_bind_breaker_threshold = threshold.max(1);
951 self.route_bind_breaker_cooldown = cooldown;
952 self
953 }
954
955 #[cfg(test)]
956 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
957 self.health_probe_timeout = timeout;
958 self
959 }
960
961 #[cfg(test)]
962 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
963 self.control_dispatch_delay = Some(delay);
964 self
965 }
966
967 #[cfg(test)]
971 pub(crate) fn with_route_bind_relay_sent_hold(
972 mut self,
973 hold: Arc<tokio::sync::Barrier>,
974 ) -> Self {
975 self.route_bind_relay_sent_hold = Some(hold);
976 self
977 }
978
979 pub fn with_process_liveness(
980 mut self,
981 process_liveness: Arc<dyn ModuleProcessLiveness>,
982 ) -> Self {
983 self.process_liveness = Some(process_liveness);
984 self
985 }
986
987 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
988 self.supervisor = supervisor;
989 self
990 }
991
992 pub fn with_daemon_provenance(
993 mut self,
994 pid: u32,
995 started_at_ms: u64,
996 executable_path: Option<PathBuf>,
997 build_git_sha: Option<String>,
998 build_lock_digest: Option<String>,
999 ) -> Self {
1000 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1001 let process_start_time = process_start_time(pid);
1002 self.daemon_provenance = DaemonProvenanceFacts {
1003 build: DaemonBuildProvenance {
1004 build_git_sha,
1005 build_lock_digest,
1006 },
1007 pid: Some(pid),
1008 started_at_ms: Some(started_at_ms),
1009 start_clock: None,
1010 executable_path,
1011 executable_identity,
1012 process_start_time,
1013 probe: ExecutableIdentityProbe::default(),
1014 };
1015 self
1016 }
1017
1018 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1019 self.daemon_provenance.start_clock = Some(clock);
1020 self
1021 }
1022
1023 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1025 self.scopes
1026 .write()
1027 .unwrap_or_else(|poisoned| poisoned.into_inner())
1028 .set_wall_clock(Arc::new(clock));
1029 self
1030 }
1031
1032 #[cfg(test)]
1033 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1034 self.provenance_probe_override = Some(result);
1035 self
1036 }
1037
1038 pub fn with_capability_config(
1042 self,
1043 modules: impl IntoIterator<Item = (String, bool)>,
1044 reserved_capabilities: BTreeMap<String, String>,
1045 ) -> Self {
1046 self.capability_evaluator
1047 .configure(modules, reserved_capabilities);
1048 self
1049 }
1050
1051 pub fn with_supervisor_rescan(
1052 mut self,
1053 supervisor: Supervisor,
1054 config_path: impl Into<PathBuf>,
1055 configured_port: Option<u16>,
1056 ) -> Self {
1057 self.rescan = Some(SupervisorRescanContext {
1058 supervisor,
1059 config_path: config_path.into(),
1060 configured_port,
1061 storage_config: self.storage_config.clone(),
1062 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1063 admission_facts_targets: self.admission_facts_targets.clone(),
1064 scope_authority_owners: self.scope_authority_owners.clone(),
1065 });
1066 self
1067 }
1068
1069 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1070 self.connected_clients = connected_clients;
1071 self
1072 }
1073
1074 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1075 Arc::clone(&self.forwarding)
1076 }
1077
1078 pub(crate) fn counters(&self) -> DaemonCounters {
1079 self.counters.clone()
1080 }
1081
1082 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1085 tokio::spawn(async move {
1086 loop {
1087 self.capability_evaluator
1088 .wait_for_change_or_deadline()
1089 .await;
1090 self.refresh_capability_requirements();
1091 }
1092 });
1093 }
1094
1095 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1101 tokio::spawn(async move {
1102 loop {
1103 let has_deadlines = self
1104 .scopes
1105 .read()
1106 .unwrap_or_else(|poisoned| poisoned.into_inner())
1107 .has_deadlines();
1108 if !has_deadlines {
1109 self.scope_deadline_added.notified().await;
1110 continue;
1111 }
1112 tokio::time::sleep(Duration::from_secs(1)).await;
1113 if let Err(error) = self.sweep_expired_scopes() {
1114 warn!(%error, "scope expiry sweep failed");
1115 }
1116 }
1117 });
1118 }
1119
1120 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1121 let mut table = self
1122 .scopes
1123 .write()
1124 .unwrap_or_else(|poisoned| poisoned.into_inner());
1125 let swept = table.sweep_expired();
1126 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1127 drop(table);
1128 self.close_scope_drained_routes(drained);
1129 Ok(())
1130 }
1131
1132 fn publish_scope_changes(
1133 &self,
1134 changes: &[crate::scopes::ScopeTagChange],
1135 expired: &[crate::scopes::ScopeExpired],
1136 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1137 if changes.is_empty() {
1138 return Ok(Vec::new());
1139 }
1140 let drained = self
1141 .forwarding
1142 .publish_scope_changes(changes)
1143 .map_err(RouterError::Forwarding)?;
1144 let mut counts = HashMap::new();
1145 for route in &drained {
1146 *counts
1147 .entry((
1148 route.scope.owner.as_str(),
1149 route.scope.scope_ref.as_str(),
1150 route.scope.tag.scope_epoch,
1151 ))
1152 .or_insert(0usize) += 1;
1153 }
1154 for scope in expired {
1155 let routes_closed = counts
1156 .get(&(
1157 scope.owner.as_str(),
1158 scope.scope_ref.as_str(),
1159 scope.scope_epoch,
1160 ))
1161 .copied()
1162 .unwrap_or(0);
1163 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1164 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1165 routes_closed, "scope expired");
1166 }
1167 Ok(drained)
1168 }
1169
1170 fn runtime_capability_snapshot(
1171 &self,
1172 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1173 let runtime = self
1174 .supervisor
1175 .list()
1176 .into_iter()
1177 .map(|module| {
1178 let status = module.status().map_err(|err| {
1179 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1180 })?;
1181 Ok(RuntimeModule {
1182 module_id: status.module_id,
1183 state: status.state,
1184 enabled: status.enabled,
1185 })
1186 })
1187 .collect::<Result<Vec<_>, RouterError>>()?;
1188 let (_, registrations) = self.registry.list_modules().map_err(|err| {
1189 RouterError::backend(
1190 0,
1191 0,
1192 format!("failed to list capability registrations: {err}"),
1193 )
1194 })?;
1195 let registrations = registrations
1196 .into_iter()
1197 .map(|registration| RegisteredModule {
1198 module_id: registration.manifest.module_id,
1199 module_version: registration.manifest.module_version,
1200 capabilities: registration.manifest.capabilities,
1201 })
1202 .collect();
1203 Ok((runtime, registrations))
1204 }
1205
1206 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1213 let cached_registration = RegisteredModule {
1214 module_id: registration.manifest.module_id.clone(),
1215 module_version: registration.manifest.module_version.clone(),
1216 capabilities: registration.manifest.capabilities.clone(),
1217 };
1218 if self.capability_evaluator.record_hello(&cached_registration) {
1219 warn!(
1220 module_id = %cached_registration.module_id,
1221 "capability claims drifted from the cached manifest"
1222 );
1223 }
1224 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1225 self.enforce_capability_denies();
1226 }
1227 self.refresh_capability_requirements();
1228 }
1229
1230 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1234 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1235 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1236 self.supervisor.set_swap_promotion_observer(observer);
1237 }
1238
1239 pub fn refresh_capability_requirements(&self) {
1240 match self.runtime_capability_snapshot() {
1241 Ok((runtime, registrations)) => {
1242 log_requirement_events(
1243 self.capability_evaluator
1244 .evaluate_now(&runtime, ®istrations),
1245 );
1246 }
1247 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1248 }
1249 }
1250
1251 fn enforce_capability_denies(&self) {
1255 let (_, registrations) = match self.registry.list_modules() {
1256 Ok(snapshot) => snapshot,
1257 Err(err) => {
1258 warn!(error = %err, "failed to read registrations for capability deny census");
1259 return;
1260 }
1261 };
1262 let manifests = registrations
1263 .into_iter()
1264 .map(|registration| {
1265 (
1266 registration.manifest.module_id.clone(),
1267 registration.manifest,
1268 )
1269 })
1270 .collect::<BTreeMap<_, _>>();
1271 let census = match self.forwarding.route_census(None) {
1272 Ok(census) => census,
1273 Err(err) => {
1274 warn!(error = %err, "failed to read route census for capability deny enforcement");
1275 return;
1276 }
1277 };
1278
1279 for (target_module_id, routes) in census {
1280 let Some(target_manifest) = manifests.get(&target_module_id) else {
1281 continue;
1282 };
1283 let mut closed_routes = Vec::new();
1284 let mut module_goodbyes = Vec::new();
1285 for route in routes {
1286 let Principal::Reserved {
1287 module_id: opening_module_id,
1288 } = &route.principal
1289 else {
1290 continue;
1291 };
1292 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1293 continue;
1294 };
1295 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1296 continue;
1297 };
1298
1299 match self.forwarding.release_client_route(
1300 route.goodbye_target.connection_id,
1301 route.goodbye_target.channel,
1302 route.goodbye_target.epoch,
1303 ) {
1304 Ok(RouteRelease::Removed(module_goodbye)) => {
1305 warn!(
1306 opening_module_id,
1307 target_module_id,
1308 capability,
1309 "force-closing route because an attested capability deny edge now matches"
1310 );
1311 closed_routes.push(route);
1312 module_goodbyes.push(module_goodbye);
1313 }
1314 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1315 Err(err) => warn!(
1316 opening_module_id,
1317 target_module_id,
1318 capability,
1319 error = %err,
1320 "failed to force-close capability-denied route"
1321 ),
1322 }
1323 }
1324
1325 if closed_routes.is_empty() {
1326 continue;
1327 }
1328 send_route_control_pushes(
1329 &self.forwarding,
1330 closed_routes,
1331 ClientControlPush::RouteClosed {
1332 module_id: target_module_id,
1333 channels: Vec::new(),
1334 reason: RouteCloseReason::CapabilityDenied,
1335 drained: false,
1336 abandoned: 0,
1337 excluded_subscriptions: 0,
1338 terminal: Some(false),
1339 },
1340 );
1341 self.emit_route_goodbyes(module_goodbyes);
1342 }
1343 }
1344
1345 fn not_ready_reason(
1351 &self,
1352 registration: &crate::registry::ModuleRegistration,
1353 ) -> Option<NotReadyReason> {
1354 if !registration.ready {
1355 return Some(NotReadyReason {
1356 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1357 capability: None,
1358 });
1359 }
1360 self.first_unprovided_required_capability(registration)
1361 .map(|capability| NotReadyReason {
1362 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1363 capability: Some(capability),
1364 })
1365 }
1366
1367 fn first_unprovided_required_capability(
1385 &self,
1386 registration: &crate::registry::ModuleRegistration,
1387 ) -> Option<String> {
1388 let required = registration
1389 .manifest
1390 .capabilities
1391 .iter()
1392 .flat_map(|declarations| declarations.requires.iter())
1393 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1394 .map(|requirement| requirement.capability.as_str())
1395 .collect::<BTreeSet<_>>();
1396 if required.is_empty() {
1397 return None;
1398 }
1399 let module_id = registration.manifest.module_id.as_str();
1400 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1401 if required
1402 .iter()
1403 .any(|capability| verdict(capability).is_none())
1404 {
1405 self.refresh_capability_requirements();
1406 }
1407 required
1408 .into_iter()
1409 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1410 .map(str::to_string)
1411 }
1412
1413 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1414 self.capability_evaluator
1415 .statuses()
1416 .into_iter()
1417 .map(capability_requirement_status)
1418 .collect()
1419 }
1420
1421 fn deregister_connection(
1428 &self,
1429 connection_id: ConnectionId,
1430 reason: RegistrationEndReason,
1431 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1432 self.registry
1433 .deregister_connection_with_reason(connection_id, reason)
1434 }
1435
1436 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1437 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1438 return None;
1439 }
1440 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1441 parse_client_control_request(&frame.body)
1442 else {
1443 return None;
1444 };
1445 Some(target_module_id(&target).to_string())
1446 }
1447
1448 pub(crate) fn route_open_capacity_refusal(
1449 &self,
1450 ctx: &RouteCtx,
1451 frame: &Frame,
1452 target_module_id: &str,
1453 in_flight: usize,
1454 limit: usize,
1455 ) -> Result<Frame, RouterError> {
1456 self.route_open_admission_refusal_frame(
1457 ctx,
1458 frame,
1459 target_module_id,
1460 "open_admission_full",
1461 (in_flight, limit),
1462 format!(
1463 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1464 ),
1465 )
1466 }
1467
1468 fn route_open_target_capacity_refusal(
1469 &self,
1470 ctx: &RouteCtx,
1471 frame: &Frame,
1472 target_module_id: &str,
1473 in_flight: usize,
1474 ) -> Result<Frame, RouterError> {
1475 self.route_open_admission_refusal_frame(
1476 ctx,
1477 frame,
1478 target_module_id,
1479 "target_binds_full",
1480 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1481 format!(
1482 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1483 ),
1484 )
1485 }
1486
1487 fn route_open_admission_refusal_frame(
1495 &self,
1496 ctx: &RouteCtx,
1497 frame: &Frame,
1498 target_module_id: &str,
1499 reason: &'static str,
1500 (in_flight, limit): (usize, usize),
1501 message: impl Into<String>,
1502 ) -> Result<Frame, RouterError> {
1503 let code = error_codes::TARGET_UNAVAILABLE;
1504 self.counters.increment_route_open_refused(code);
1505 info!(
1506 target: "control",
1507 code,
1508 reason,
1509 module_id = ?target_module_id,
1510 connection_id = ctx.connection_id.get(),
1511 in_flight,
1512 limit,
1513 "route.open refused"
1514 );
1515 control_error_frame(frame, code, message.into())
1516 }
1517
1518 #[cfg(test)]
1524 pub fn handle_control(
1525 &self,
1526 connection_id: ConnectionId,
1527 frame: Frame,
1528 ) -> Result<Vec<Frame>, RouterError> {
1529 match frame.header.ty {
1530 FrameType::Ping => Ok(vec![pong(&frame)?]),
1531 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1532 FrameType::Goodbye => self.handle_goodbye(connection_id),
1533 ty => Ok(vec![control_error_frame(
1534 &frame,
1535 "unsupported_control_frame",
1536 format!("unsupported channel-0 frame {ty:?}"),
1537 )?]),
1538 }
1539 }
1540
1541 pub async fn handle_control_frame(
1542 &self,
1543 ctx: &RouteCtx,
1544 frame: Frame,
1545 ) -> Result<Vec<Frame>, RouterError> {
1546 self.handle_control_frame_timed(ctx, frame, None).await
1547 }
1548
1549 pub(crate) async fn handle_control_frame_timed(
1550 &self,
1551 ctx: &RouteCtx,
1552 frame: Frame,
1553 dispatch_started_at: Option<StdInstant>,
1554 ) -> Result<Vec<Frame>, RouterError> {
1555 match frame.header.ty {
1556 FrameType::Ping => Ok(vec![pong(&frame)?]),
1557 FrameType::Hello => {
1558 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1559 }
1560 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1561 FrameType::Cancel => {
1562 if self
1565 .forwarding
1566 .operator_confirms()
1567 .cancel(ctx.connection_id, frame.header.corr)
1568 || self
1569 .supervisor
1570 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1571 {
1572 Ok(Vec::new())
1573 } else {
1574 Ok(vec![control_error_frame(
1575 &frame,
1576 "unknown_subscription",
1577 "no supervisor spawn subscription has this correlation id",
1578 )?])
1579 }
1580 }
1581 FrameType::Request => {
1582 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1585 if op
1586 .as_ref()
1587 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1588 {
1589 return self.handle_activity_record(ctx, frame).await;
1590 }
1591 if op
1592 .as_ref()
1593 .is_some_and(|probe| probe.op == "operator.confirm")
1594 {
1595 return self.handle_operator_confirm(ctx, frame);
1596 }
1597 if self
1598 .forwarding
1599 .module_endpoint_for_connection(ctx.connection_id)
1600 .map_err(RouterError::Forwarding)?
1601 .is_some()
1602 {
1603 if !is_known_module_request_op(&frame.body) {
1604 return Ok(vec![control_error_frame(
1605 &frame,
1606 "unsupported_control_frame",
1607 "module-originated channel-0 REQUEST is not supported",
1608 )?]);
1609 }
1610 let request = match parse_module_control_request_from_module(&frame.body) {
1611 Ok(request) => request,
1612 Err((err, ControlRequestBodyError::UnknownOp)) => {
1613 return Ok(vec![control_error_frame(
1614 &frame,
1615 "unsupported_control_frame",
1616 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1617 )?])
1618 }
1619 Err((err, ControlRequestBodyError::InvalidBody)) => {
1620 return Ok(vec![control_error_frame(
1621 &frame,
1622 "invalid_control_body",
1623 format!("malformed module control body: {err}"),
1624 )?])
1625 }
1626 };
1627 let op = module_control_request_op(&request);
1628 let corr = frame.header.corr;
1629 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1630 let result =
1631 self.handle_module_control_request(ctx.connection_id, frame, request);
1632 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1633 return result;
1634 }
1635
1636 if is_known_module_request_op(&frame.body) {
1637 return Ok(vec![control_error_frame(
1638 &frame,
1639 "not_registered",
1640 "catalog.update requires an active module registration owned by this connection",
1641 )?]);
1642 }
1643
1644 let request = match parse_client_control_request(&frame.body) {
1645 Ok(request) => request,
1646 Err((err, ControlRequestBodyError::UnknownOp)) => {
1647 return Ok(vec![control_error_frame(
1648 &frame,
1649 "unknown_control_op",
1650 format!("unknown client control op: {err}"),
1651 )?])
1652 }
1653 Err((err, ControlRequestBodyError::InvalidBody)) => {
1654 return Ok(vec![control_error_frame(
1655 &frame,
1656 "invalid_control_body",
1657 format!("malformed client control body: {err}"),
1658 )?])
1659 }
1660 };
1661 let op = client_control_request_op(&request);
1662 let corr = frame.header.corr;
1663 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1664 #[cfg(test)]
1665 if let Some(delay) = self.control_dispatch_delay {
1666 tokio::time::sleep(delay).await;
1667 }
1668 let result = self
1669 .handle_client_control_request(ctx, frame, request)
1670 .await;
1671 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1672 result
1673 }
1674 FrameType::Push => {
1675 let Some(endpoint) = self
1676 .forwarding
1677 .module_endpoint_for_connection(ctx.connection_id)
1678 .map_err(RouterError::Forwarding)?
1679 else {
1680 return Ok(vec![control_error_frame(
1681 &frame,
1682 "unsupported_control_frame",
1683 "client-originated channel-0 PUSH is not supported",
1684 )?]);
1685 };
1686 self.handle_status_update(endpoint, frame)
1687 }
1688 FrameType::Response | FrameType::Error
1689 if self
1690 .forwarding
1691 .module_endpoint_for_connection(ctx.connection_id)
1692 .map_err(RouterError::Forwarding)?
1693 .is_some() =>
1694 {
1695 self.handle_module_relay_response(ctx.connection_id, frame)
1696 }
1697 ty => Ok(vec![control_error_frame(
1698 &frame,
1699 "unsupported_control_frame",
1700 format!("unsupported channel-0 frame {ty:?}"),
1701 )?]),
1702 }
1703 }
1704
1705 pub fn cleanup_connection(
1706 &self,
1707 connection_id: ConnectionId,
1708 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1709 self.cleanup_connection_with_end_reason(
1710 connection_id,
1711 RegistrationEndReason::ConnectionClosed,
1712 )
1713 }
1714
1715 fn cleanup_connection_with_end_reason(
1716 &self,
1717 connection_id: ConnectionId,
1718 requested_reason: RegistrationEndReason,
1719 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1720 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1721 self.registry
1722 .get_module_by_connection(connection_id)?
1723 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
1724 .and_then(|module| module.registration_end_reason().ok().flatten())
1725 .unwrap_or(requested_reason)
1726 } else {
1727 requested_reason
1728 };
1729 let crash_closed = self
1730 .registry
1731 .get_module_by_connection(connection_id)?
1732 .and_then(|registration| {
1733 self.forwarding
1734 .module_endpoint_for_connection(connection_id)
1735 .ok()
1736 .flatten()
1737 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1738 .map(|routes| (registration.manifest.module_id, routes))
1739 });
1740 let crash_closed = crash_closed.map(|(module_id, routes)| {
1741 let terminal = match self.supervisor.get(&module_id) {
1742 None => false,
1743 Some(module) => match module.will_recover_after_connection_loss() {
1744 Ok(will_recover) => !will_recover,
1745 Err(err) => {
1746 warn!(
1747 %module_id,
1748 error = %err,
1749 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1750 );
1751 false
1752 }
1753 },
1754 };
1755 let (reason, drained) = match self
1759 .forwarding
1760 .daemon_drain_result_for_connection(connection_id)
1761 {
1762 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
1763 Ok(None) => (RouteCloseReason::Crash, false),
1764 Err(err) => {
1765 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1766 (RouteCloseReason::Crash, false)
1767 }
1768 };
1769 (module_id, routes, reason, drained, terminal)
1770 });
1771 let registrations = self.deregister_connection(connection_id, end_reason);
1772 let cleanup = if crash_closed.is_some() {
1773 self.forwarding.cleanup_connection_counted(connection_id)
1774 } else {
1775 self.forwarding
1781 .cleanup_connection(connection_id)
1782 .map(|released| crate::forwarding::ConnectionCleanup {
1783 released,
1784 abandoned_relays: 0,
1785 })
1786 };
1787 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
1792 let abandoned = cleanup
1793 .as_ref()
1794 .map_or(0, |cleanup| cleanup.abandoned_relays);
1795 send_route_control_pushes(
1796 &self.forwarding,
1797 routes,
1798 ClientControlPush::RouteClosed {
1799 module_id,
1800 channels: Vec::new(),
1801 reason,
1802 drained,
1803 abandoned,
1804 excluded_subscriptions: 0,
1805 terminal: Some(terminal),
1806 },
1807 );
1808 }
1809 if let Ok(cleanup) = cleanup {
1810 self.emit_route_goodbyes(cleanup.released);
1811 }
1812 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1816 crate::supervise::notify_registration_release();
1817 self.capability_evaluator.wake_deadline_loop();
1818 self.refresh_capability_requirements();
1819 }
1820 self.supervisor.remove_spawn_subscribers(connection_id);
1821 self.hello_launch_nonces
1824 .lock()
1825 .unwrap_or_else(|poisoned| poisoned.into_inner())
1826 .forget(connection_id);
1827 self.scopes
1828 .write()
1829 .unwrap_or_else(|poisoned| poisoned.into_inner())
1830 .release_connection(connection_id);
1831 registrations
1832 }
1833
1834 pub(crate) fn handle_route_goodbye(
1835 &self,
1836 connection_id: ConnectionId,
1837 route_channel: u16,
1838 route_epoch: u32,
1839 ) -> Result<bool, RouterError> {
1840 debug!(
1841 connection_id = connection_id.get(),
1842 route_channel, route_epoch, "handling route GOODBYE"
1843 );
1844 let RouteRelease::Removed(released_route) = self
1845 .forwarding
1846 .release_client_route(connection_id, route_channel, route_epoch)
1847 .map_err(RouterError::Forwarding)?
1848 else {
1849 return Ok(false);
1850 };
1851 self.emit_route_goodbyes(vec![released_route]);
1852 Ok(true)
1853 }
1854
1855 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1856 for released in released_routes {
1857 let frame = match Frame::build_with_version(
1858 released.negotiated_ver,
1859 FrameType::Goodbye,
1860 control_flags(),
1861 released.channel,
1862 released.epoch,
1863 0,
1864 Vec::new(),
1865 ) {
1866 Ok(frame) => frame,
1867 Err(err) => {
1868 warn!(
1869 route_channel = released.channel,
1870 error = %err,
1871 "failed to build route GOODBYE frame"
1872 );
1873 continue;
1874 }
1875 };
1876 if !released.close_on_delivery_failure() {
1877 crate::forwarding::send_module_route_goodbye(
1878 &self.counters,
1879 &released.sink,
1880 frame,
1881 released.module_id.as_deref(),
1882 "client route released",
1883 );
1884 continue;
1885 }
1886 if let Err(err) = released.sink.try_send(frame) {
1887 warn!(
1888 target_connection_id = released.connection_id.get(),
1889 route_channel = released.channel,
1890 error = %err,
1891 "route GOODBYE was not delivered to client; closing target connection"
1892 );
1893 if self
1894 .forwarding
1895 .escalate_client_delivery_failure(
1896 released.connection_id,
1897 released.channel,
1898 released.epoch,
1899 CloseReason::new(
1900 "route_goodbye_delivery_failed",
1901 format!(
1902 "failed to enqueue route GOODBYE for channel {}: {err}",
1903 released.channel
1904 ),
1905 ),
1906 crate::forwarding::UndeliveredFrame {
1907 module_id: released.module_id.as_deref(),
1908 sink: &released.sink,
1909 },
1910 )
1911 .unwrap_or(false)
1912 {
1913 self.counters.increment_goodbye_relay_client_failed();
1914 }
1915 }
1916 }
1917 }
1918
1919 fn send_abandoned_route_bind_goodbye(
1931 &self,
1932 module_sink: &crate::FrameSink,
1933 negotiated_ver: u8,
1934 module_channel: u16,
1935 module_epoch: u32,
1936 ) {
1937 let frame = match Frame::build_with_version(
1938 negotiated_ver,
1939 FrameType::Goodbye,
1940 control_flags(),
1941 module_channel,
1942 module_epoch,
1943 0,
1944 Vec::new(),
1945 ) {
1946 Ok(frame) => frame,
1947 Err(err) => {
1948 warn!(
1949 route_channel = module_channel,
1950 error = %err,
1951 "failed to build GOODBYE for abandoned route.bind"
1952 );
1953 return;
1954 }
1955 };
1956 crate::forwarding::send_module_route_goodbye(
1957 &self.counters,
1958 module_sink,
1959 frame,
1960 None,
1961 "abandoned route.bind",
1962 );
1963 }
1964
1965 fn handle_hello(
1966 &self,
1967 connection_id: ConnectionId,
1968 sink: Option<crate::FrameSink>,
1969 frame: Frame,
1970 ) -> Result<Vec<Frame>, RouterError> {
1971 debug!(
1972 connection_id = connection_id.get(),
1973 corr = frame.header.corr,
1974 "handling HELLO"
1975 );
1976 if self
1980 .registry
1981 .get_module_by_connection(connection_id)
1982 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
1983 .is_some()
1984 {
1985 return Ok(vec![control_error_frame(
1986 &frame,
1987 "invalid_hello",
1988 "connection is already registered as a module",
1989 )?]);
1990 }
1991 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1992 Ok(value) => value,
1993 Err(err) => {
1994 return Ok(vec![control_error_frame(
1995 &frame,
1996 "invalid_hello",
1997 format!("malformed HELLO body: {err}"),
1998 )?])
1999 }
2000 };
2001 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2002 return Ok(vec![control_error_frame(
2003 &frame,
2004 "invalid_capability_grammar",
2005 err.to_string(),
2006 )?]);
2007 }
2008 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2009 return Ok(vec![control_error_frame(
2010 &frame,
2011 "invalid_manifest",
2012 err.to_string(),
2013 )?]);
2014 }
2015 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2016 return Ok(vec![control_error_frame(
2017 &frame,
2018 "invalid_event_declaration",
2019 err.to_string(),
2020 )?]);
2021 }
2022 if let Some(provenance) = hello_value
2023 .get("manifest")
2024 .and_then(|manifest| manifest.get("provenance"))
2025 {
2026 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2027 return Ok(vec![control_error_frame(
2028 &frame,
2029 "invalid_manifest",
2030 format!("malformed manifest provenance: {err}"),
2031 )?]);
2032 }
2033 }
2034 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2035 Ok(hello) => hello,
2036 Err(err) => {
2037 return Ok(vec![control_error_frame(
2038 &frame,
2039 "invalid_hello",
2040 format!("malformed HELLO body: {err}"),
2041 )?])
2042 }
2043 };
2044
2045 if hello.protocol_ver != hello.manifest.protocol_ver {
2046 return Ok(vec![control_error_frame(
2047 &frame,
2048 "invalid_manifest",
2049 format!(
2050 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2051 hello.protocol_ver, hello.manifest.protocol_ver
2052 ),
2053 )?]);
2054 }
2055
2056 if hello.manifest.module_id.trim().is_empty() {
2057 return Ok(vec![control_error_frame(
2058 &frame,
2059 "invalid_manifest",
2060 "manifest module_id must not be empty",
2061 )?]);
2062 }
2063
2064 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2065 Ok(negotiated_ver) => negotiated_ver,
2066 Err(message) => {
2067 return Ok(vec![control_error_frame(
2068 &frame,
2069 "version_unsupported",
2070 message,
2071 )?])
2072 }
2073 };
2074
2075 let swap_admission = self
2083 .supervisor
2084 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2085 if swap_admission == SwapHelloAdmission::Refused {
2086 warn!(
2087 module_id = %hello.manifest.module_id,
2088 connection_id = connection_id.get(),
2089 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2090 );
2091 return Ok(vec![control_error_frame(
2092 &frame,
2093 "swap_token_invalid",
2094 format!(
2095 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2096 hello.manifest.module_id
2097 ),
2098 )?]);
2099 }
2100 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2101
2102 if let Some(rejection) = (!swap_candidate)
2110 .then(|| {
2111 self.supervisor.reserved_hello_rejection(
2112 &hello.manifest.module_id,
2113 hello.launch_nonce.as_deref(),
2114 )
2115 })
2116 .flatten()
2117 {
2118 let message = match rejection {
2119 ReservedHelloRejection::Exact { module_id } => format!(
2120 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2121 ),
2122 ReservedHelloRejection::Prefix {
2123 prefix,
2124 owner_module_id,
2125 } => format!(
2126 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2127 hello.manifest.module_id
2128 ),
2129 };
2130 return Ok(vec![control_error_frame(
2131 &frame,
2132 "reserved_module",
2133 message,
2134 )?]);
2135 }
2136
2137 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2138 &hello.manifest.module_id,
2139 hello.manifest.capabilities.as_ref(),
2140 );
2141 if let Some(refusal) = reserved_capability_refusals.first() {
2142 let capability = refusal.capability.clone();
2143 let bound_module = refusal.claimants[0].clone();
2144 log_duplicate_claim_events(reserved_capability_refusals);
2145 return Ok(vec![control_error_frame(
2146 &frame,
2147 "reserved_capability",
2148 format!(
2149 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2150 capability, bound_module, hello.manifest.module_id
2151 ),
2152 )?]);
2153 }
2154
2155 if self
2158 .forwarding
2159 .connection_has_client_routes(connection_id)
2160 .map_err(RouterError::Forwarding)?
2161 {
2162 return Ok(vec![control_error_frame(
2163 &frame,
2164 "invalid_hello",
2165 "connection has open client routes and cannot also register as a module",
2166 )?]);
2167 }
2168
2169 self.hello_launch_nonces
2174 .lock()
2175 .unwrap_or_else(|poisoned| poisoned.into_inner())
2176 .record(connection_id, hello.launch_nonce.as_deref());
2177 let control_ops = effective_module_control_ops(hello.control_ops);
2178 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2181 if swap_candidate {
2182 return self.register_swap_candidate(
2183 connection_id,
2184 sink,
2185 &frame,
2186 hello.manifest,
2187 negotiated_ver,
2188 control_ops,
2189 hello_ack,
2190 );
2191 }
2192 let registration = match self.registry.register_with_control_ops(
2193 hello.manifest,
2194 negotiated_ver,
2195 connection_id,
2196 control_ops,
2197 ) {
2198 Ok(registration) => registration,
2199 Err(RegistryError::DuplicateModuleId { module_id }) => {
2200 return Ok(vec![control_error_frame(
2201 &frame,
2202 "duplicate_module_id",
2203 format!(
2204 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2205 ),
2206 )?])
2207 }
2208 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2209 return Ok(vec![control_error_frame(
2210 &frame,
2211 "invalid_module_id",
2212 err.to_string(),
2213 )?])
2214 }
2215 Err(err) => {
2216 return Ok(vec![control_error_frame(
2217 &frame,
2218 "registry_error",
2219 err.to_string(),
2220 )?])
2221 }
2222 };
2223
2224 let reply = if let Some(sink) = sink {
2225 let concurrency = manifest_concurrency(®istration.manifest);
2241 if let Err(err) = self.forwarding.register_module_connection_acked(
2242 connection_id,
2243 registration.manifest.module_id.clone(),
2244 negotiated_ver,
2245 concurrency,
2246 sink,
2247 hello_ack,
2248 ) {
2249 if matches!(
2253 self.deregister_connection(
2254 connection_id,
2255 RegistrationEndReason::RegistrationFailed,
2256 ),
2257 Ok(r) if !r.is_empty()
2258 ) {
2259 crate::supervise::notify_registration_release();
2260 }
2261 return Ok(vec![control_error_frame(
2262 &frame,
2263 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2264 "invalid_hello"
2265 } else {
2266 forwarding_error_code(&err)
2267 },
2268 err.to_string(),
2269 )?]);
2270 }
2271 Vec::new()
2272 } else {
2273 vec![hello_ack]
2276 };
2277
2278 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2287 info!(
2288 module_id = %registration.manifest.module_id,
2289 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2290 );
2291 }
2292
2293 self.apply_registration_capabilities(®istration);
2294
2295 info!(
2296 module_id = %registration.manifest.module_id,
2297 module_version = %registration.manifest.module_version,
2298 negotiated_ver,
2299 routable_provider = manifest_provides_routable_role(®istration.manifest),
2300 connection_id = connection_id.get(),
2301 "module registered"
2302 );
2303
2304 Ok(reply)
2305 }
2306
2307 #[allow(clippy::too_many_arguments)]
2317 fn register_swap_candidate(
2318 &self,
2319 connection_id: ConnectionId,
2320 sink: Option<crate::FrameSink>,
2321 frame: &Frame,
2322 manifest: ModuleManifest,
2323 negotiated_ver: u8,
2324 control_ops: Vec<String>,
2325 hello_ack: Frame,
2326 ) -> Result<Vec<Frame>, RouterError> {
2327 let module_id = manifest.module_id.clone();
2328 let registration = match self.registry.register_candidate_with_control_ops(
2329 manifest,
2330 negotiated_ver,
2331 connection_id,
2332 control_ops,
2333 ) {
2334 Ok(registration) => registration,
2335 Err(RegistryError::DuplicateModuleId { module_id }) => {
2336 return Ok(vec![control_error_frame(
2337 frame,
2338 "duplicate_module_id",
2339 format!(
2340 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2341 ),
2342 )?])
2343 }
2344 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2345 return Ok(vec![control_error_frame(
2346 frame,
2347 "invalid_module_id",
2348 err.to_string(),
2349 )?])
2350 }
2351 Err(err) => {
2352 return Ok(vec![control_error_frame(
2353 frame,
2354 "registry_error",
2355 err.to_string(),
2356 )?])
2357 }
2358 };
2359 let reply = if let Some(sink) = sink {
2360 let concurrency = manifest_concurrency(®istration.manifest);
2364 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2365 connection_id,
2366 module_id.clone(),
2367 negotiated_ver,
2368 concurrency,
2369 sink,
2370 hello_ack,
2371 ) {
2372 if matches!(
2373 self.deregister_connection(
2374 connection_id,
2375 RegistrationEndReason::RegistrationFailed,
2376 ),
2377 Ok(r) if !r.is_empty()
2378 ) {
2379 crate::supervise::notify_registration_release();
2380 }
2381 return Ok(vec![control_error_frame(
2382 frame,
2383 forwarding_error_code(&err),
2384 err.to_string(),
2385 )?]);
2386 }
2387 Vec::new()
2388 } else {
2389 vec![hello_ack]
2390 };
2391 self.supervisor.mark_swap_candidate_admitted(&module_id);
2392 info!(
2393 module_id = %module_id,
2394 module_version = %registration.manifest.module_version,
2395 negotiated_ver,
2396 ready = registration.ready,
2397 connection_id = connection_id.get(),
2398 "swap candidate registered; not routable until cutover"
2399 );
2400 Ok(reply)
2401 }
2402
2403 fn build_hello_ack(
2404 &self,
2405 frame: &Frame,
2406 negotiated_ver: u8,
2407 module_id: &str,
2408 ) -> Result<Frame, RouterError> {
2409 let ack = ModuleHelloAckBody {
2410 negotiated_ver,
2411 subc_ops: module_subc_ops(),
2412 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2413 storage: self
2414 .storage_config
2415 .as_ref()
2416 .map(|cfg| cfg.descriptor_for(module_id)),
2417 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2418 };
2419 let body = serde_json::to_vec(&ack).map_err(|err| {
2420 RouterError::backend(
2421 0,
2422 frame.header.corr,
2423 format!("failed to encode HELLO_ACK: {err}"),
2424 )
2425 })?;
2426
2427 Frame::build_with_version(
2428 negotiated_ver,
2429 FrameType::HelloAck,
2430 control_flags(),
2431 0,
2432 0,
2433 frame.header.corr,
2434 body,
2435 )
2436 .map_err(RouterError::FrameBuild)
2437 }
2438
2439 async fn handle_client_control_request(
2440 &self,
2441 ctx: &RouteCtx,
2442 frame: Frame,
2443 request: ClientControlRequest,
2444 ) -> Result<Vec<Frame>, RouterError> {
2445 match request {
2446 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2447 ClientControlRequest::CatalogList { module_id } => {
2448 self.handle_catalog_list(frame, module_id)
2449 }
2450 ClientControlRequest::RouteOpen {
2451 target,
2452 identity,
2453 consumer_identity,
2454 consumer_capabilities,
2455 role_versions,
2456 admission_facts,
2457 scope,
2458 } => {
2459 self.handle_route_open(
2460 ctx,
2461 frame,
2462 RouteOpenRequest {
2463 target,
2464 identity,
2465 consumer_identity,
2466 consumer_capabilities,
2467 role_versions,
2468 admission_facts,
2469 scope,
2470 },
2471 )
2472 .await
2473 }
2474 ClientControlRequest::RoutePoll {
2475 route_channel,
2476 route_epoch,
2477 kind,
2478 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2479 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2480 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2481 self.handle_supervisor_spawn_snapshot(frame)
2482 }
2483 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2484 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2485 }
2486 ClientControlRequest::SupervisorRestart {
2487 module_id,
2488 drain_timeout_ms,
2489 } => {
2490 self.log_supervisor_request_received(
2491 ctx,
2492 frame.header.corr,
2493 ops::SUPERVISOR_RESTART,
2494 Some(&module_id),
2495 None,
2496 )?;
2497 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2498 .await
2499 }
2500 ClientControlRequest::SupervisorSwap {
2501 module_id,
2502 ready_timeout_ms,
2503 } => {
2504 self.log_supervisor_request_received(
2505 ctx,
2506 frame.header.corr,
2507 ops::SUPERVISOR_SWAP,
2508 Some(&module_id),
2509 None,
2510 )?;
2511 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2512 .await
2513 }
2514 ClientControlRequest::SupervisorReload { module_id } => {
2515 self.log_supervisor_request_received(
2516 ctx,
2517 frame.header.corr,
2518 ops::SUPERVISOR_RELOAD,
2519 Some(&module_id),
2520 None,
2521 )?;
2522 self.handle_supervisor_reload(frame, module_id).await
2523 }
2524 ClientControlRequest::SupervisorRescan { preview } => {
2525 if !preview {
2526 self.log_supervisor_request_received(
2527 ctx,
2528 frame.header.corr,
2529 ops::SUPERVISOR_RESCAN,
2530 None,
2531 None,
2532 )?;
2533 }
2534 self.handle_supervisor_rescan(frame, preview).await
2535 }
2536 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2537 self.log_supervisor_request_received(
2538 ctx,
2539 frame.header.corr,
2540 ops::SUPERVISOR_RELEASE_RESERVED,
2541 Some(&module_id),
2542 None,
2543 )?;
2544 self.handle_supervisor_release_reserved(frame, module_id)
2545 .await
2546 }
2547 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2548 self.log_supervisor_request_received(
2549 ctx,
2550 frame.header.corr,
2551 ops::SUPERVISOR_SET_ENABLED,
2552 Some(&module_id),
2553 Some(enabled),
2554 )?;
2555 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2556 .await
2557 }
2558 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2559 self.handle_supervisor_health_probe(frame, module_id).await
2560 }
2561 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2562 ClientControlRequest::SupervisorRoutes { module_id } => {
2563 self.handle_supervisor_routes(frame, module_id)
2564 }
2565 ClientControlRequest::SupervisorProvenance { module_id } => {
2566 self.handle_supervisor_provenance(frame, module_id).await
2567 }
2568 ClientControlRequest::SupervisorStderrTail {
2569 module_id,
2570 max_lines,
2571 max_bytes,
2572 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2573 ClientControlRequest::SupervisorTerminals { module_id } => {
2574 self.handle_supervisor_terminals(frame, module_id).await
2575 }
2576 }
2577 }
2578
2579 fn log_supervisor_request_received(
2580 &self,
2581 ctx: &RouteCtx,
2582 corr: u64,
2583 op: &'static str,
2584 module_id: Option<&str>,
2585 enabled: Option<bool>,
2586 ) -> Result<(), RouterError> {
2587 let caller = self
2588 .registry
2589 .get_module_by_connection(ctx.connection_id)
2590 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2591 .map(|registration| Principal::Reserved {
2592 module_id: registration.manifest.module_id,
2593 })
2594 .unwrap_or(Principal::Direct);
2595 let caller = principal_label(&caller);
2596 info!(
2599 target: "control",
2600 op,
2601 module_id,
2602 enabled,
2603 connection_id = ctx.connection_id.get(),
2604 caller = %caller,
2605 "supervisor request received"
2606 );
2607 Ok(())
2608 }
2609
2610 fn handle_module_control_request(
2611 &self,
2612 connection_id: ConnectionId,
2613 frame: Frame,
2614 request: ModuleControlRequestFromModule,
2615 ) -> Result<Vec<Frame>, RouterError> {
2616 match request {
2617 ModuleControlRequestFromModule::CatalogUpdate {
2618 provides,
2619 capabilities,
2620 ready,
2621 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2622 ModuleControlRequestFromModule::LiveRoots {} => {
2623 let registered = self
2624 .registry
2625 .get_module_by_connection(connection_id)
2626 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2627 let Some(registration) = registered else {
2628 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2629 };
2630 let response = self
2631 .forwarding
2632 .live_roots(®istration.manifest.module_id)
2633 .map_err(RouterError::Forwarding)?;
2634 Ok(vec![control_response_body_frame(
2635 &frame,
2636 &response,
2637 "ModuleControlResponseToModule::LiveRoots",
2638 )?])
2639 }
2640 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2641 self.handle_scope_sync(connection_id, frame, generation, scopes)
2642 }
2643 ModuleControlRequestFromModule::ScopeApply {
2644 generation,
2645 upsert,
2646 end,
2647 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
2648 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2649 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2650 }
2651 }
2652 }
2653
2654 fn handle_operator_confirm(
2655 &self,
2656 ctx: &RouteCtx,
2657 frame: Frame,
2658 ) -> Result<Vec<Frame>, RouterError> {
2659 use crate::operator_confirm::{audit, Outcome};
2660 let registration = self
2661 .registry
2662 .get_module_by_connection(ctx.connection_id)
2663 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2664 let Some(registration) = registration else {
2665 let outcome = Outcome::refusal("not_registered");
2666 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2667 return Ok(vec![outcome.frame(&frame)]);
2668 };
2669 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
2670 Ok(request) => request,
2671 Err(_) => {
2672 let outcome = Outcome::refusal("invalid_control_body");
2673 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2674 return Ok(vec![outcome.frame(&frame)]);
2675 }
2676 };
2677 let module_id = registration.manifest.module_id;
2678 let nonce = self
2682 .hello_launch_nonces
2683 .lock()
2684 .unwrap_or_else(|p| p.into_inner())
2685 .nonce(ctx.connection_id)
2686 .map(str::to_owned);
2687 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
2688 self.supervisor
2689 .spawned_consumer_authorized(&module_id, nonce)
2690 });
2691 let confirms = self.forwarding.operator_confirms();
2692 self.forwarding
2693 .with_operator_route(
2694 ctx.connection_id,
2695 request.route_channel,
2696 request.route_epoch,
2697 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
2698 )
2699 .map_err(RouterError::Forwarding)
2700 }
2701
2702 fn handle_scope_sync(
2706 &self,
2707 connection_id: ConnectionId,
2708 frame: Frame,
2709 generation: u64,
2710 scopes: Vec<ScopeRecord>,
2711 ) -> Result<Vec<Frame>, RouterError> {
2712 self.handle_scope_change(connection_id, frame, generation, scopes, None)
2713 }
2714
2715 fn handle_scope_change(
2716 &self,
2717 connection_id: ConnectionId,
2718 frame: Frame,
2719 generation: u64,
2720 scopes: Vec<ScopeRecord>,
2721 end: Option<Vec<ScopeEnd>>,
2722 ) -> Result<Vec<Frame>, RouterError> {
2723 let is_apply = end.is_some();
2724 let Some(registration) = self
2725 .registry
2726 .get_module_by_connection(connection_id)
2727 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2728 else {
2729 return Ok(vec![control_error_frame(
2730 &frame,
2731 "not_registered",
2732 "scope.sync and scope.apply require an active module registration owned by this connection",
2733 )?]);
2734 };
2735 let owner = registration.manifest.module_id;
2736 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2737 let is_current_launch = |connection: ConnectionId| {
2738 self.hello_launch_nonces
2739 .lock()
2740 .unwrap_or_else(|poisoned| poisoned.into_inner())
2741 .presented(connection, current_nonce.as_deref())
2742 };
2743 let mut table = self
2748 .scopes
2749 .write()
2750 .unwrap_or_else(|poisoned| poisoned.into_inner());
2751 let outcome = match end {
2752 Some(end) => table.apply(
2753 &owner,
2754 connection_id,
2755 is_current_launch,
2756 generation,
2757 scopes,
2758 end,
2759 ),
2760 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
2761 };
2762 let drained = match &outcome {
2763 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
2764 Err(_) => Vec::new(),
2765 };
2766 let has_deadlines = outcome.is_ok() && table.has_deadlines();
2767 drop(table);
2768 if has_deadlines {
2769 self.scope_deadline_added.notify_one();
2770 }
2771 match outcome {
2772 Ok(applied) => {
2773 let counts = ScopeOutcomeCounts::of(&applied.results);
2774 info!(
2775 owner = %owner,
2776 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
2777 generation,
2778 records = applied.results.len(),
2779 created = counts.created,
2780 replaced = counts.replaced,
2781 updated = counts.updated,
2782 unchanged = counts.unchanged,
2783 refused = counts.refused,
2784 ended = applied.ended.len(),
2785 tag_changes = applied.tag_changes.len(),
2786 routes_closed = drained.len(),
2787 "scope change accepted"
2788 );
2789 for refused in applied
2795 .results
2796 .iter()
2797 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
2798 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
2799 {
2800 warn!(
2801 owner = %owner,
2802 generation,
2803 scope_ref = %refused.scope_ref,
2804 scope_epoch = refused.scope_epoch,
2805 code = refused.code.as_deref().unwrap_or(""),
2806 "scope record refused"
2807 );
2808 }
2809 self.close_scope_drained_routes(drained);
2810 let response = if is_apply {
2811 ModuleControlResponseToModule::ScopeApply {
2812 generation,
2813 results: applied.results,
2814 end_results: applied.end_results,
2815 ended: applied.ended,
2816 }
2817 } else {
2818 ModuleControlResponseToModule::ScopeSync {
2819 generation,
2820 results: applied.results,
2821 ended: applied.ended,
2822 }
2823 };
2824 Ok(vec![control_response_body_frame(
2825 &frame,
2826 &response,
2827 "scope change response",
2828 )?])
2829 }
2830 Err(refusal) => {
2831 info!(
2832 owner = %owner,
2833 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
2834 generation,
2835 code = refusal.code,
2836 "scope change refused"
2837 );
2838 Ok(vec![control_error_frame(
2839 &frame,
2840 refusal.code,
2841 refusal.message,
2842 )?])
2843 }
2844 }
2845 }
2846
2847 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2853 if drained.is_empty() {
2854 return;
2855 }
2856 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2857 BTreeMap::new();
2858 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2859 for route in drained {
2860 warn!(
2861 module_id = %route.module_id,
2862 reason = ?route.reason,
2863 client_connection_id = route.client.connection_id.get(),
2864 route_channel = route.client.channel,
2865 "closing route because its scope changed"
2866 );
2867 pushes
2868 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2869 .or_insert_with(|| (route.reason, Vec::new()))
2870 .1
2871 .push(EndpointRoute {
2872 goodbye_target: route.client.clone(),
2873 principal: Principal::Unverified,
2874 bound_at: Instant::now(),
2875 draining: false,
2876 drain_reason: None,
2877 });
2878 goodbyes.push(route.module);
2879 goodbyes.push(route.client);
2880 }
2881 for ((module_id, _), (reason, routes)) in pushes {
2882 send_route_control_pushes(
2883 &self.forwarding,
2884 routes,
2885 ClientControlPush::RouteClosed {
2886 module_id,
2887 channels: Vec::new(),
2888 reason,
2889 drained: false,
2890 abandoned: 0,
2891 excluded_subscriptions: 0,
2892 terminal: Some(false),
2893 },
2894 );
2895 }
2896 self.emit_route_goodbyes(goodbyes);
2897 }
2898
2899 fn handle_scope_describe(
2902 &self,
2903 connection_id: ConnectionId,
2904 frame: Frame,
2905 owner: Principal,
2906 scope_ref: String,
2907 ) -> Result<Vec<Frame>, RouterError> {
2908 let registered = self
2909 .registry
2910 .get_module_by_connection(connection_id)
2911 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2912 if registered.is_none() {
2913 return Ok(vec![control_error_frame(
2914 &frame,
2915 "not_registered",
2916 "scope.describe requires an active module registration owned by this connection",
2917 )?]);
2918 }
2919 let description = self
2920 .scopes
2921 .read()
2922 .unwrap_or_else(|poisoned| poisoned.into_inner())
2923 .describe(&owner, &scope_ref);
2924 let owner_configured = match &owner {
2925 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
2930 _ => false,
2931 };
2932 let response = ModuleControlResponseToModule::ScopeDescribe {
2933 status: description.status,
2934 scope_epoch: description.scope_epoch,
2935 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2936 owner_synced: description.owner_synced,
2937 owner_configured,
2938 scope: description.stamp,
2939 };
2940 Ok(vec![control_response_body_frame(
2941 &frame,
2942 &response,
2943 "ModuleControlResponseToModule::ScopeDescribe",
2944 )?])
2945 }
2946
2947 fn handle_catalog_update(
2948 &self,
2949 connection_id: ConnectionId,
2950 frame: Frame,
2951 provides: Vec<ProviderRole>,
2952 capabilities: Option<CapabilityDeclarations>,
2953 ready: Option<bool>,
2954 ) -> Result<Vec<Frame>, RouterError> {
2955 self.refresh_capability_requirements();
2956 let Some(registration) = self
2957 .registry
2958 .get_module_by_connection(connection_id)
2959 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2960 else {
2961 return Ok(vec![control_error_frame(
2962 &frame,
2963 "not_registered",
2964 "catalog.update requires an active module registration owned by this connection",
2965 )?]);
2966 };
2967
2968 if let Some(message) =
2969 catalog_update_frozen_field_message(®istration.manifest, &provides)
2970 {
2971 return Ok(vec![control_error_frame(
2972 &frame,
2973 "catalog_update_frozen_field",
2974 message,
2975 )?]);
2976 }
2977
2978 let mut candidate = registration.manifest.clone();
2979 candidate.provides = provides.clone();
2980 candidate.capabilities = capabilities
2981 .clone()
2982 .or_else(|| registration.manifest.capabilities.clone());
2983 if let Err(err) = candidate.validate_capability_grammar() {
2984 return Ok(vec![control_error_frame(
2985 &frame,
2986 "invalid_capability_grammar",
2987 err.to_string(),
2988 )?]);
2989 }
2990
2991 let mut conflicts = self
2994 .capability_evaluator
2995 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
2996 if let Some(conflict) = conflicts.first() {
2997 let message = format!(
2998 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2999 conflict.capability, conflict.claimants[0], candidate.module_id
3000 );
3001 for conflict in &mut conflicts {
3002 conflict.source = DuplicateClaimSource::CatalogUpdate;
3003 }
3004 log_duplicate_claim_events(conflicts);
3005 return Ok(vec![control_error_frame(
3006 &frame,
3007 "reserved_capability",
3008 message,
3009 )?]);
3010 }
3011
3012 let updated = self
3013 .registry
3014 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3015 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3016 if updated.is_none() {
3017 return Ok(vec![control_error_frame(
3018 &frame,
3019 "not_registered",
3020 "catalog.update requires an active module registration owned by this connection",
3021 )?]);
3022 }
3023 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3024 log_duplicate_claim_events(
3025 self.capability_evaluator
3026 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3027 );
3028 }
3029 if capability_census_trigger(
3030 registration.manifest.capabilities.as_ref(),
3031 updated
3032 .as_ref()
3033 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3034 ) {
3035 self.enforce_capability_denies();
3036 }
3037 self.refresh_capability_requirements();
3038
3039 let response = ModuleControlResponseToModule::CatalogUpdate {};
3040 control_response_body_frame(
3041 &frame,
3042 &response,
3043 "ModuleControlResponseToModule::CatalogUpdate",
3044 )
3045 .map(|frame| vec![frame])
3046 }
3047
3048 async fn handle_activity_record(
3049 &self,
3050 ctx: &RouteCtx,
3051 frame: Frame,
3052 ) -> Result<Vec<Frame>, RouterError> {
3053 use subc_control::{
3054 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3055 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3056 };
3057 let result = async {
3058 let module = self
3059 .forwarding
3060 .module_id_for_connection(ctx.connection_id)
3061 .ok()
3062 .flatten()
3063 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3064 let request: ActivityRecordRequest =
3065 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3066 crate::activity::validate(&module, &request)?;
3067 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3068 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3069 let at_ms = std::time::SystemTime::now()
3070 .duration_since(std::time::UNIX_EPOCH)
3071 .unwrap_or_default()
3072 .as_millis() as u64;
3073 ledger.record(module, request, actor, at_ms).await
3074 }
3075 .await;
3076 match result {
3077 Ok(entry) => Ok(vec![control_response_body_frame(
3078 &frame,
3079 &ActivityRecordResponse { seq: entry.seq },
3080 "activity.record",
3081 )?]),
3082 Err(code) => Ok(vec![control_error_frame(
3083 &frame,
3084 code,
3085 "activity recording refused",
3086 )?]),
3087 }
3088 }
3089
3090 fn activity_actor(
3091 &self,
3092 connection: ConnectionId,
3093 module: &str,
3094 on: Option<&subc_control::ActivityOn>,
3095 ) -> Result<subc_control::ActivityActor, &'static str> {
3096 use crate::forwarding::{DataRoute, DataRouteState};
3097 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3098 let principal = |id: &str| Principal::Reserved {
3099 module_id: id.to_owned(),
3100 };
3101 let mut actor = ActivityActor {
3102 principal: principal(module),
3103 scope_owner: None,
3104 agent_id: None,
3105 run_id: None,
3106 flow_id: None,
3107 project_root: None,
3108 };
3109 let scope = match on {
3110 None => return Ok(actor),
3111 Some(ActivityOn::Scope(scope)) => {
3112 let selector = subc_protocol::scope::ScopeSelector {
3113 owner: principal(module),
3114 scope_ref: scope.scope_ref.clone(),
3115 scope_epoch: Some(scope.epoch),
3116 };
3117 self.scopes
3118 .read()
3119 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3120 .admit(&principal(module), module, &selector, true)
3121 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3122 }
3123 Some(ActivityOn::Route(route)) => {
3124 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3125 .forwarding
3126 .lookup_data_route(connection, route.channel, route.epoch)
3127 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3128 else {
3129 return Err(ACTIVITY_ACTOR_UNKNOWN);
3130 };
3131 if binding.module_id != module {
3132 return Err(ACTIVITY_ACTOR_UNKNOWN);
3133 }
3134 actor.principal = binding.principal.clone();
3135 actor.project_root = binding
3136 .project_root
3137 .as_ref()
3138 .map(|root| root.as_path().to_path_buf());
3139 let Some(bound) = &binding.scope else {
3140 return Ok(actor);
3141 };
3142 let owner = principal(&bound.owner);
3143 let selector = subc_protocol::scope::ScopeSelector {
3144 owner: owner.clone(),
3145 scope_ref: bound.scope_ref.clone(),
3146 scope_epoch: Some(bound.tag.scope_epoch),
3147 };
3148 let admitted = self
3149 .scopes
3150 .read()
3151 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3152 .admit(&owner, module, &selector, true)
3153 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3154 if admitted.tag != bound.tag {
3155 return Err(ACTIVITY_ACTOR_UNKNOWN);
3156 }
3157 admitted
3158 }
3159 };
3160 actor.scope_owner = Some(scope.stamp.owner);
3161 actor.agent_id = scope.stamp.attributes.agent_id;
3162 actor.run_id = scope.stamp.attributes.run_id;
3163 actor.flow_id = scope.stamp.attributes.flow_id;
3164 Ok(actor)
3165 }
3166
3167 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3168 self.refresh_capability_requirements();
3169 let mut counters = self.counters.snapshot();
3175 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3176 obj.insert(
3177 "activity_unparseable_lines".into(),
3178 ledger.read_errors().into(),
3179 );
3180 }
3181 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3182 self.forwarding.client_route_concentration(),
3183 counters.as_object_mut(),
3184 ) {
3185 obj.insert(
3186 "client_connections_with_routes".into(),
3187 connections_with_routes.into(),
3188 );
3189 obj.insert("max_routes_on_one_connection".into(), max.into());
3190 }
3191 if let (Some(open_breakers), Some(obj)) = (
3197 self.route_bind_breakers.open_snapshot(),
3198 counters.as_object_mut(),
3199 ) {
3200 obj.insert("route_bind_breakers_open".into(), open_breakers);
3201 }
3202 let response = ClientControlResponse::ServerDescribe {
3203 protocol_ver: PROTOCOL_VERSION,
3204 subc_ops: subc_ops(),
3205 capabilities: self.subc_capabilities.as_ref().to_vec(),
3206 connected_clients: self.connected_clients.count(),
3207 counters: Some(counters),
3208 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3209 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3210 capability_requirements: self.capability_requirement_statuses(),
3211 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3212 };
3213 Ok(vec![control_response_body_frame(
3214 &frame,
3215 &response,
3216 "ClientControlResponse::ServerDescribe",
3217 )?])
3218 }
3219
3220 fn handle_catalog_list(
3221 &self,
3222 frame: Frame,
3223 module_id: Option<String>,
3224 ) -> Result<Vec<Frame>, RouterError> {
3225 let (generation, modules) = self.registry.list_modules().map_err(|err| {
3226 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3227 })?;
3228 let entries = modules
3229 .into_iter()
3230 .filter(|registration| {
3231 module_id
3232 .as_deref()
3233 .map(|wanted| registration.manifest.module_id == wanted)
3234 .unwrap_or(true)
3235 })
3236 .map(|registration| {
3237 let not_ready = self.not_ready_reason(®istration);
3238 let roles = registration.manifest.provides;
3239 CatalogEntry::new(
3240 registration.manifest.module_id,
3241 roles,
3242 registration.control_ops,
3243 )
3244 .with_ready(not_ready.is_none())
3245 .with_not_ready(not_ready)
3246 .with_module_version(Some(registration.manifest.module_version))
3247 .with_capabilities(registration.manifest.capabilities)
3248 .with_self_signals(registration.manifest.self_signals)
3249 .with_events(registration.manifest.events)
3250 })
3251 .collect();
3252 let response = ClientControlResponse::CatalogList {
3253 generation,
3254 modules: entries,
3255 subc_ops: subc_ops(),
3256 };
3257 Ok(vec![control_response_body_frame(
3258 &frame,
3259 &response,
3260 "ClientControlResponse::CatalogList",
3261 )?])
3262 }
3263
3264 fn route_open_principal(
3265 &self,
3266 frame: &Frame,
3267 consumer_identity: Option<ConsumerIdentity>,
3268 ) -> Result<Result<Principal, Frame>, RouterError> {
3269 let Some(consumer_identity) = consumer_identity else {
3270 return Ok(Ok(Principal::Direct));
3271 };
3272
3273 if self.supervisor.spawned_consumer_authorized(
3274 &consumer_identity.module_id,
3275 &consumer_identity.launch_nonce,
3276 ) {
3277 return Ok(Ok(Principal::Reserved {
3278 module_id: consumer_identity.module_id,
3279 }));
3280 }
3281
3282 Ok(Err(control_error_frame(
3283 frame,
3284 "bad_consumer_identity",
3285 format!(
3286 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3287 consumer_identity.module_id
3288 ),
3289 )?))
3290 }
3291
3292 fn route_open_refusal_frame(
3299 &self,
3300 ctx: &RouteCtx,
3301 frame: &Frame,
3302 module_id: &str,
3303 reason: &'static str,
3304 code: &'static str,
3305 message: impl Into<String>,
3306 ) -> Result<Frame, RouterError> {
3307 self.observe_route_open_refusal(ctx, module_id, reason, code);
3308 control_error_frame(frame, code, message.into())
3309 }
3310
3311 fn route_open_breaker_refusal_frame(
3329 &self,
3330 ctx: &RouteCtx,
3331 frame: &Frame,
3332 module_id: &str,
3333 consecutive_timeouts: u32,
3334 retry_in: Duration,
3335 probe_in_flight: bool,
3336 ) -> Result<Frame, RouterError> {
3337 self.counters
3338 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
3339 debug!(
3340 target: "control",
3341 code = "module_timeout",
3342 module_id = ?module_id,
3343 connection_id = ctx.connection_id.get(),
3344 consecutive_timeouts,
3345 retry_in_ms = retry_in.as_millis() as u64,
3346 probe_in_flight,
3347 "route.open refused by open bind-relay breaker"
3348 );
3349 let detail = if probe_in_flight {
3356 "one test route is already being tried; retry once it settles".to_string()
3357 } else {
3358 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
3359 };
3360 control_error_frame(
3361 frame,
3362 "module_timeout",
3363 format!(
3364 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
3365 timed out in a row); {detail}; its established routes are unaffected"
3366 ),
3367 )
3368 }
3369
3370 fn observe_route_open_refusal(
3383 &self,
3384 ctx: &RouteCtx,
3385 module_id: &str,
3386 reason: &'static str,
3387 code: &'static str,
3388 ) {
3389 self.counters.increment_route_open_refused(code);
3390 info!(
3391 target: "control",
3392 code,
3393 reason,
3394 module_id = ?module_id,
3395 connection_id = ctx.connection_id.get(),
3396 "route.open refused"
3397 );
3398 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
3399 self.route_outages.record_not_serving(module_id, reason);
3400 }
3401 }
3402
3403 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
3448 self.route_outages.record_accepted(module_id);
3449 self.counters.increment_route_open_accepted(principal);
3450 info!(
3451 target: "control",
3452 principal,
3453 module_id,
3454 connection_id = ctx.connection_id.get(),
3455 "route.open accepted"
3456 );
3457 }
3458
3459 fn supervised_absent_route_open_refusal_frame(
3460 &self,
3461 ctx: &RouteCtx,
3462 frame: &Frame,
3463 module_id: &str,
3464 code: &'static str,
3465 status: &crate::supervise::ModuleStatus,
3466 ) -> Result<Frame, RouterError> {
3467 self.counters.increment_route_open_refused(code);
3468 info!(
3469 target: "control",
3470 code,
3471 reason = "supervised_not_registered",
3472 module_id = ?module_id,
3473 connection_id = ctx.connection_id.get(),
3474 state = %status.state,
3475 enabled = status.enabled,
3476 live = status.live,
3477 "route.open refused"
3478 );
3479 self.route_outages
3483 .record_not_serving(module_id, "supervised_not_registered");
3484 let message = if let Some(pid) = status.waiting_for_previous_pid {
3485 format!("waiting: previous instance (pid {pid}) is still running but hidden; it cannot be identified")
3486 } else {
3487 format!(
3488 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
3489 status.state, status.enabled, status.live
3490 )
3491 };
3492 control_error_frame(frame, code, message)
3493 }
3494
3495 async fn handle_route_open(
3496 &self,
3497 ctx: &RouteCtx,
3498 frame: Frame,
3499 request: RouteOpenRequest,
3500 ) -> Result<Vec<Frame>, RouterError> {
3501 let RouteOpenRequest {
3502 target,
3503 mut identity,
3504 consumer_identity,
3505 consumer_capabilities,
3506 role_versions,
3507 admission_facts,
3508 scope,
3509 } = request;
3510 let target_module_id = target_module_id(&target).to_string();
3511 if self
3512 .registry
3513 .get_module_by_connection(ctx.connection_id)
3514 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3515 .is_some()
3516 {
3517 return Ok(vec![control_error_frame(
3518 &frame,
3519 "invalid_request",
3520 "module connections cannot open client routes",
3521 )?]);
3522 }
3523 debug!(
3524 connection_id = ctx.connection_id.get(),
3525 corr = frame.header.corr,
3526 module_id = %target_module_id,
3527 "handling route.open"
3528 );
3529
3530 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
3536 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
3537 self.observe_route_open_refusal(
3538 ctx,
3539 &target_module_id,
3540 "invalid_role_versions",
3541 error_codes::INVALID_REQUEST,
3542 );
3543 return Ok(vec![control_error_body_frame(
3544 &frame,
3545 ErrorBody {
3546 code: error_codes::INVALID_REQUEST.to_string(),
3547 message: error.to_string(),
3548 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
3549 },
3550 )?]);
3551 }
3552
3553 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
3554 if status.waiting_for_previous_pid.is_some() {
3555 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3556 ctx,
3557 &frame,
3558 &target_module_id,
3559 "module_warming",
3560 &status,
3561 )?]);
3562 }
3563 }
3564
3565 let Some(registration) = self
3584 .registry
3585 .get_module(&target_module_id)
3586 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3587 else {
3588 if let Some((status, warming)) =
3589 self.supervisor_status(&target_module_id, frame.header.corr)?
3590 {
3591 if status.protocol == ModuleProtocol::None {
3598 return Ok(vec![self.route_open_refusal_frame(
3599 ctx,
3600 &frame,
3601 &target_module_id,
3602 "protocol_none",
3603 error_codes::MODULE_NO_PROTOCOL,
3604 format!(
3605 "module_id '{target_module_id}' is declared protocol: none; \
3606 it speaks no subc wire and serves no routes"
3607 ),
3608 )?]);
3609 }
3610 let code = if warming {
3611 "module_warming"
3612 } else {
3613 "target_unavailable"
3614 };
3615 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3616 ctx,
3617 &frame,
3618 &target_module_id,
3619 code,
3620 &status,
3621 )?]);
3622 }
3623 if let Some(removed_ago_ms) =
3624 self.supervisor.removal_tombstone_age_ms(&target_module_id)
3625 {
3626 return Ok(vec![self.route_open_refusal_frame(
3627 ctx,
3628 &frame,
3629 &target_module_id,
3630 "removed",
3631 error_codes::MODULE_REMOVED,
3632 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
3633 )?]);
3634 }
3635 return Ok(vec![self.route_open_refusal_frame(
3636 ctx,
3637 &frame,
3638 &target_module_id,
3639 "not_registered",
3640 error_codes::UNKNOWN_MODULE,
3641 format!("module_id '{target_module_id}' is not registered"),
3642 )?]);
3643 };
3644
3645 if !registration.ready {
3649 self.counters
3650 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
3651 info!(
3652 target: "control",
3653 code = error_codes::MODULE_WARMING,
3654 module_id = ?target_module_id,
3655 connection_id = ctx.connection_id.get(),
3656 reason = "declared_not_ready",
3657 "route.open refused"
3658 );
3659 self.route_outages
3662 .record_not_serving(&target_module_id, "declared_not_ready");
3663 return Ok(vec![control_error_body_frame(
3664 &frame,
3665 ErrorBody {
3666 code: error_codes::MODULE_WARMING.to_string(),
3667 message: format!(
3668 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
3669 ),
3670 detail: Some(serde_json::json!({
3671 "reason": "declared_not_ready"
3672 })),
3673 },
3674 )?]);
3675 }
3676
3677 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
3696 self.counters
3697 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3698 info!(
3699 target: "control",
3700 code = error_codes::MODULE_WARMING,
3701 module_id = ?target_module_id,
3702 connection_id = ctx.connection_id.get(),
3703 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3704 capability = %capability,
3705 "route.open refused"
3706 );
3707 return Ok(vec![control_error_body_frame(
3708 &frame,
3709 ErrorBody {
3710 code: error_codes::MODULE_WARMING.to_string(),
3711 message: format!(
3712 "module_id '{target_module_id}' requires capability '{capability}', \
3713 which no registered module provides; retry"
3714 ),
3715 detail: Some(serde_json::json!({
3716 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3717 "capability": capability,
3718 })),
3719 },
3720 )?]);
3721 }
3722
3723 if !target_has_required_role(&target, ®istration.manifest.provides) {
3724 return Ok(vec![self.route_open_refusal_frame(
3725 ctx,
3726 &frame,
3727 &target_module_id,
3728 "role_not_provided",
3729 "target_unavailable",
3730 format!("module_id '{target_module_id}' does not provide the requested target"),
3731 )?]);
3732 }
3733
3734 if registration.state != ChannelState::Active {
3735 return Ok(vec![self.route_open_refusal_frame(
3736 ctx,
3737 &frame,
3738 &target_module_id,
3739 "registration_not_active",
3740 "target_unavailable",
3741 format!("module_id '{target_module_id}' is not active"),
3742 )?]);
3743 }
3744
3745 if self
3746 .forwarding
3747 .module_is_draining(&target_module_id)
3748 .map_err(RouterError::Forwarding)?
3749 {
3750 return Ok(vec![self.route_open_refusal_frame(
3751 ctx,
3752 &frame,
3753 &target_module_id,
3754 "reloading",
3755 "module_reloading",
3756 format!("module_id '{target_module_id}' is reloading"),
3757 )?]);
3758 }
3759
3760 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
3761 process_liveness.process_live(&target_module_id) == Some(false)
3762 }) {
3763 if process_liveness.process_replacing(&target_module_id) {
3770 return Ok(vec![self.route_open_refusal_frame(
3771 ctx,
3772 &frame,
3773 &target_module_id,
3774 "reloading",
3775 "module_reloading",
3776 format!("module_id '{target_module_id}' is reloading"),
3777 )?]);
3778 }
3779 return Ok(vec![self.route_open_refusal_frame(
3780 ctx,
3781 &frame,
3782 &target_module_id,
3783 "supervisor_not_live",
3784 "target_unavailable",
3785 format!("module_id '{target_module_id}' is not live"),
3786 )?]);
3787 }
3788
3789 if !self
3790 .forwarding
3791 .has_live_module_connection(&target_module_id)
3792 .map_err(RouterError::Forwarding)?
3793 {
3794 return Ok(vec![self.route_open_refusal_frame(
3795 ctx,
3796 &frame,
3797 &target_module_id,
3798 "no_forwarding_connection",
3799 "target_unavailable",
3800 format!("module_id '{target_module_id}' has no live forwarding connection"),
3801 )?]);
3802 }
3803
3804 if let Some(error) =
3805 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
3806 {
3807 self.observe_route_open_refusal(
3808 ctx,
3809 &target_module_id,
3810 "op_not_allowed",
3811 "op_not_allowed",
3812 );
3813 return Ok(vec![error]);
3814 }
3815
3816 let principal = match self.route_open_principal(&frame, consumer_identity)? {
3817 Ok(principal) => principal,
3818 Err(error) => {
3819 self.observe_route_open_refusal(
3820 ctx,
3821 &target_module_id,
3822 "bad_consumer_identity",
3823 "bad_consumer_identity",
3824 );
3825 return Ok(vec![error]);
3826 }
3827 };
3828
3829 if let Principal::Reserved {
3833 module_id: opening_module_id,
3834 } = &principal
3835 {
3836 if let Some(opening_registration) = self
3837 .registry
3838 .get_module(opening_module_id)
3839 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3840 {
3841 if let Some(capability) =
3842 denied_capability(&opening_registration.manifest, ®istration.manifest)
3843 {
3844 warn!(
3845 opening_module_id,
3846 target_module_id,
3847 capability,
3848 "refusing route.open because an attested capability deny edge matches"
3849 );
3850 return Ok(vec![self.route_open_refusal_frame(
3851 ctx,
3852 &frame,
3853 &target_module_id,
3854 "capability_deny_edge",
3855 "capability_forbidden",
3856 format!(
3857 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
3858 ),
3859 )?]);
3860 }
3861 }
3862 }
3863
3864 if admission_facts.is_some() {
3865 let carrier_matches = matches!(
3866 &principal,
3867 Principal::Reserved { module_id }
3868 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
3869 );
3870 if !carrier_matches {
3871 return Ok(vec![self.route_open_refusal_frame(
3872 ctx,
3873 &frame,
3874 &target_module_id,
3875 "admission_facts_carrier_not_permitted",
3876 "admission_facts_not_permitted",
3877 "admission facts may only be carried by the configured reserved module",
3878 )?]);
3879 }
3880
3881 let target_allowed = self
3882 .admission_facts_targets
3883 .as_ref()
3884 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
3885 if !target_allowed {
3886 return Ok(vec![self.route_open_refusal_frame(
3887 ctx,
3888 &frame,
3889 &target_module_id,
3890 "admission_facts_target_not_listed",
3891 "admission_facts_target_not_allowed",
3892 format!(
3893 "admission facts are not permitted for target module_id '{target_module_id}'"
3894 ),
3895 )?]);
3896 }
3897
3898 }
3902
3903 let (bound_scope, scope_stamp) = match scope {
3909 None => (None, None),
3910 Some(selector) => {
3911 let owner_configured = match &selector.owner {
3912 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3918 _ => false,
3919 };
3920 let admitted = self
3921 .scopes
3922 .read()
3923 .unwrap_or_else(|poisoned| poisoned.into_inner())
3924 .admit(&principal, &target_module_id, &selector, owner_configured)
3925 .and_then(|admission| {
3926 crate::scopes::check_target_flow_support(
3927 &admission.stamp,
3928 &target_module_id,
3929 registration.manifest.capabilities.as_ref(),
3930 )?;
3931 crate::scopes::check_target_agent_run_support(
3932 &admission.stamp,
3933 &target_module_id,
3934 registration.manifest.capabilities.as_ref(),
3935 )?;
3936 Ok(admission)
3937 });
3938 match admitted {
3939 Ok(admission) => (
3940 Some(BoundScope {
3941 owner: admission.owner,
3942 scope_ref: admission.stamp.scope_ref.clone(),
3943 tag: admission.tag,
3944 }),
3945 Some(admission.stamp),
3946 ),
3947 Err(refusal) => {
3948 return Ok(vec![self.route_open_refusal_frame(
3949 ctx,
3950 &frame,
3951 &target_module_id,
3952 refusal.code,
3953 refusal.code,
3954 refusal.message,
3955 )?]);
3956 }
3957 }
3958 }
3959 };
3960
3961 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
3983 Ok(project_root) => project_root,
3984 Err(err) => {
3985 return Ok(vec![control_error_frame(
3986 &frame,
3987 "invalid_project_root",
3988 err.to_string(),
3989 )?])
3990 }
3991 };
3992 identity.project_root = project_root.as_path().to_path_buf();
3993
3994 let _concurrency_guard = match self
4005 .route_bind_concurrency
4006 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4007 {
4008 Ok(guard) => guard,
4009 Err(in_flight) => {
4010 return Ok(vec![self.route_open_target_capacity_refusal(
4011 ctx,
4012 &frame,
4013 &target_module_id,
4014 in_flight,
4015 )?]);
4016 }
4017 };
4018
4019 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4022 RouteBindAdmission::Admitted { guard, probe } => {
4023 if probe {
4024 info!(
4025 module_id = %target_module_id,
4026 connection_id = ctx.connection_id.get(),
4027 "route.bind breaker half-open: admitting one probe"
4028 );
4029 }
4030 guard
4031 }
4032 RouteBindAdmission::Refused {
4033 consecutive_timeouts,
4034 retry_in,
4035 probe_in_flight,
4036 } => {
4037 return Ok(vec![self.route_open_breaker_refusal_frame(
4038 ctx,
4039 &frame,
4040 &target_module_id,
4041 consecutive_timeouts,
4042 retry_in,
4043 probe_in_flight,
4044 )?]);
4045 }
4046 };
4047
4048 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4053 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4054 let pending = match self
4055 .forwarding
4056 .begin_route_bind_relay_for(
4057 ctx.connection_id,
4058 ctx.egress.clone(),
4059 response_version(&frame),
4060 frame.header.corr,
4061 &target_module_id,
4062 principal.clone(),
4063 bound_scope,
4064 Some(project_root),
4065 relay_deadline,
4066 )
4067 .await
4068 {
4069 Ok(pending) => pending,
4070 Err(err) => {
4071 return Ok(vec![self.route_open_refusal_frame(
4072 ctx,
4073 &frame,
4074 &target_module_id,
4075 "relay_reservation_failed",
4076 forwarding_error_code(&err),
4077 err.to_string(),
4078 )?])
4079 }
4080 };
4081 let crate::forwarding::PendingRouteBindRelay {
4082 endpoint,
4083 module_sink,
4084 negotiated_ver,
4085 client_channel,
4086 client_epoch,
4087 module_channel,
4088 module_epoch,
4089 corr: relay_corr,
4090 receiver,
4091 } = pending;
4092 let mut reservation =
4093 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4094
4095 if let Some(stamp) = scope_stamp
4100 .as_ref()
4101 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4102 {
4103 let relay_registration = self
4104 .registry
4105 .get_module_by_connection(endpoint.connection_id)
4106 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4107 if let Err(refusal) = crate::scopes::check_target_flow_support(
4108 stamp,
4109 &target_module_id,
4110 relay_registration
4111 .as_ref()
4112 .and_then(|registration| registration.manifest.capabilities.as_ref()),
4113 )
4114 .and_then(|()| {
4115 crate::scopes::check_target_agent_run_support(
4116 stamp,
4117 &target_module_id,
4118 relay_registration
4119 .as_ref()
4120 .and_then(|registration| registration.manifest.capabilities.as_ref()),
4121 )
4122 }) {
4123 reservation.release_and_disarm();
4124 return Ok(vec![self.route_open_refusal_frame(
4125 ctx,
4126 &frame,
4127 &target_module_id,
4128 refusal.code,
4129 refusal.code,
4130 refusal.message,
4131 )?]);
4132 }
4133 }
4134
4135 debug!(
4136 connection_id = ctx.connection_id.get(),
4137 client_channel,
4138 client_epoch,
4139 module_channel,
4140 module_epoch,
4141 "reserved route handle pair"
4142 );
4143 let principal_label = principal_label(&principal);
4146 let relay = ModuleControlRequest::RouteBind {
4147 route_channel: module_channel,
4148 epoch: module_epoch,
4149 target,
4150 identity,
4151 principal: Some(principal),
4152 consumer_capabilities,
4153 role_versions,
4154 admission_facts,
4155 scope: scope_stamp,
4156 };
4157 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4158 RouterError::backend(
4159 0,
4160 frame.header.corr,
4161 format!("failed to encode route.bind request: {err}"),
4162 )
4163 })?;
4164 let relay_frame = Frame::build_with_version(
4165 negotiated_ver,
4166 FrameType::Request,
4167 control_flags(),
4168 0,
4169 0,
4170 relay_corr,
4171 relay_body,
4172 )
4173 .map_err(RouterError::FrameBuild)?;
4174
4175 if let Err(err) = module_sink.send(relay_frame).await {
4176 reservation.release_and_disarm();
4177 return Ok(vec![self.route_open_refusal_frame(
4178 ctx,
4179 &frame,
4180 &target_module_id,
4181 "relay_send_failed",
4182 "target_unavailable",
4183 err.to_string(),
4184 )?]);
4185 }
4186
4187 #[cfg(test)]
4188 if let Some(hold) = &self.route_bind_relay_sent_hold {
4189 hold.wait().await;
4193 hold.wait().await;
4194 }
4195
4196 if self
4201 .forwarding
4202 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4203 .map_err(RouterError::Forwarding)?
4204 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4205 {
4206 self.send_abandoned_route_bind_goodbye(
4207 &module_sink,
4208 negotiated_ver,
4209 module_channel,
4210 module_epoch,
4211 );
4212 }
4213
4214 match timeout_at(relay_deadline, receiver).await {
4215 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4216 reservation.disarm();
4217 if breaker.record_accepted() {
4218 info!(
4219 module_id = %target_module_id,
4220 "route.bind breaker closed: the probe was accepted"
4221 );
4222 }
4223 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
4224 Ok(Vec::new())
4225 }
4226 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4227 reservation.release_and_disarm();
4228 breaker.record_inconclusive();
4232 let scope_code = match body.code.as_str() {
4236 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4237 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4238 _ => None,
4239 };
4240 if let Some(code) = scope_code {
4241 self.observe_route_open_refusal(
4242 ctx,
4243 &target_module_id,
4244 "scope_changed_before_commit",
4245 code,
4246 );
4247 return Ok(vec![control_error_body_frame(&frame, body)?]);
4248 }
4249 self.counters
4250 .increment_route_open_refused("module_rejected");
4251 info!(
4252 target: "control",
4253 code = "module_rejected",
4254 module_code = ?body.code,
4255 module_id = ?target_module_id,
4256 connection_id = ctx.connection_id.get(),
4257 "route.open refused"
4258 );
4259 Ok(vec![control_error_body_frame(&frame, body)?])
4260 }
4261 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
4262 reservation.release_and_disarm();
4263 breaker.record_inconclusive();
4264 tracing::warn!(
4271 module_id = %target_module_id,
4272 "route.bind relay abandoned: {message}"
4273 );
4274 Ok(vec![self.route_open_refusal_frame(
4275 ctx,
4276 &frame,
4277 &target_module_id,
4278 "relay_abandoned",
4279 "target_unavailable",
4280 message,
4281 )?])
4282 }
4283 Ok(Err(_)) => {
4284 reservation.release_and_disarm();
4285 breaker.record_inconclusive();
4286 Ok(vec![self.route_open_refusal_frame(
4287 ctx,
4288 &frame,
4289 &target_module_id,
4290 "relay_waiter_canceled",
4291 "target_unavailable",
4292 "route.bind relay waiter was canceled before the module responded",
4293 )?])
4294 }
4295 Err(_) => {
4296 reservation.release_and_disarm();
4297 if let Some(opened) = breaker.record_timeout(
4301 self.route_bind_breaker_threshold,
4302 self.route_bind_breaker_cooldown,
4303 ) {
4304 warn!(
4305 module_id = %target_module_id,
4306 consecutive_timeouts = opened.consecutive_timeouts,
4307 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
4308 reopened_after_probe = opened.reopened_after_probe,
4309 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
4310 );
4311 }
4312 tracing::warn!(
4320 module_id = %target_module_id,
4321 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
4322 "route.bind relay timed out: module did not ack within budget"
4323 );
4324 Ok(vec![self.route_open_refusal_frame(
4325 ctx,
4326 &frame,
4327 &target_module_id,
4328 "relay_timed_out",
4329 "module_timeout",
4330 format!(
4331 "module_id '{target_module_id}' did not answer route.bind within {:?}",
4332 route_bind_relay_timeout
4333 ),
4334 )?])
4335 }
4336 }
4337 }
4338
4339 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4340 let response = ClientControlResponse::SupervisorSpawnSnapshot {
4341 snapshot: self.supervisor.spawn_snapshot(),
4342 };
4343 Ok(vec![control_response_body_frame(
4344 &frame,
4345 &response,
4346 "ClientControlResponse::SupervisorSpawnSnapshot",
4347 )?])
4348 }
4349
4350 fn handle_supervisor_spawn_subscribe(
4351 &self,
4352 ctx: &RouteCtx,
4353 frame: Frame,
4354 since: Option<SpawnCursor>,
4355 ) -> Result<Vec<Frame>, RouterError> {
4356 match self.supervisor.subscribe_spawns(
4357 ctx.connection_id,
4358 frame.header.corr,
4359 response_version(&frame),
4360 since,
4361 ctx.egress.clone(),
4362 ) {
4363 Ok(()) => Ok(Vec::new()),
4364 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
4365 Ok(vec![control_error_body_frame(
4366 &frame,
4367 ErrorBody {
4368 code: "spawn_cursor_incarnation_mismatch".to_string(),
4369 message: "spawn cursor belongs to a different daemon incarnation"
4370 .to_string(),
4371 detail: Some(serde_json::json!({
4372 "current_daemon_incarnation": current
4373 })),
4374 },
4375 )?])
4376 }
4377 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
4378 &frame,
4379 ErrorBody {
4380 code: "spawn_cursor_too_old".to_string(),
4381 message: "spawn cursor predates the retained event ring".to_string(),
4382 detail: Some(serde_json::json!({
4383 "oldest_retained_cursor": oldest
4384 })),
4385 },
4386 )?]),
4387 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
4388 0,
4389 frame.header.corr,
4390 format!("failed to open supervisor spawn subscription: {error}"),
4391 )),
4392 }
4393 }
4394
4395 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4396 let generation = self
4397 .registry
4398 .generation()
4399 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4400 let mut modules = Vec::new();
4401 for module in self.supervisor.list() {
4402 let status = module.status_for_control("list").map_err(|err| {
4403 RouterError::backend(
4404 0,
4405 frame.header.corr,
4406 format!("failed to read supervisor status: {err}"),
4407 )
4408 })?;
4409 let (configured, _) = module.configuration().map_err(|err| {
4410 RouterError::backend(
4411 0,
4412 frame.header.corr,
4413 format!("failed to read module configuration: {err}"),
4414 )
4415 })?;
4416 let image = module.running_image_agreement().await;
4418 let resources = Some(module.child_resource_usage());
4421 let pending_reload = Some(reload_verdict(
4422 &configured.program,
4423 status.spawned_from.as_deref(),
4424 image,
4425 ));
4426 modules.push(
4429 SupervisorEntry::new(
4430 status.module_id,
4431 status.state.to_string(),
4432 status.enabled,
4433 status.live,
4434 status.health.status,
4435 )
4436 .with_launch_nonce_env(Some(!cfg!(unix)))
4437 .with_launch_nonce_source(status.launch_nonce_source)
4438 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
4439 .with_protocol(status.protocol)
4440 .with_pending_reload(pending_reload)
4441 .with_last_probe_ms(status.health.last_probe_ms)
4442 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
4443 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
4444 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
4445 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
4446 .with_restart_count(Some(status.restart_count))
4447 .with_max_restarts(Some(status.max_restarts))
4448 .with_lifetime_restarts(Some(status.lifetime_restarts))
4449 .with_spawn_generation(Some(status.spawn_generation))
4450 .with_restart_window_secs(Some(status.restart_window.as_secs()))
4451 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
4452 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
4453 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
4454 .with_resources(resources),
4455 );
4456 }
4457 let response = ClientControlResponse::SupervisorList {
4458 generation,
4459 modules,
4460 };
4461 Ok(vec![control_response_body_frame(
4462 &frame,
4463 &response,
4464 "ClientControlResponse::SupervisorList",
4465 )?])
4466 }
4467
4468 fn handle_supervisor_stderr_tail(
4469 &self,
4470 frame: Frame,
4471 module_id: String,
4472 max_lines: Option<u32>,
4473 max_bytes: Option<u32>,
4474 ) -> Result<Vec<Frame>, RouterError> {
4475 let Some(module) = self.supervisor.get(&module_id) else {
4476 return Ok(vec![control_error_frame(
4477 &frame,
4478 "unknown_module",
4479 format!("module_id '{module_id}' is not supervised"),
4480 )?]);
4481 };
4482
4483 let snapshot = module.stderr_tail(
4484 max_lines.map(|value| value as usize),
4485 max_bytes.map(|value| value as usize),
4486 );
4487
4488 let response = ClientControlResponse::SupervisorStderrTail {
4489 module_id,
4490 tail: StderrTail {
4491 capture: match snapshot.capture {
4492 CaptureState::Captured => StderrCaptureState::Captured,
4493 CaptureState::Incomplete { reason } => {
4494 StderrCaptureState::Incomplete { reason }
4495 }
4496 CaptureState::NotCaptured { reason } => {
4497 StderrCaptureState::NotCaptured { reason }
4498 }
4499 },
4500 entries: snapshot
4501 .entries
4502 .into_iter()
4503 .map(|entry| match entry {
4504 TailEntry::Line {
4505 text,
4506 truncated,
4507 at_ms,
4508 } => StderrTailEntry::Line {
4509 text,
4510 truncated,
4511 at_ms,
4512 },
4513 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
4514 })
4515 .collect(),
4516 dropped_lines: snapshot.dropped_lines,
4517 },
4518 };
4519 Ok(vec![control_response_body_frame(
4520 &frame,
4521 &response,
4522 "ClientControlResponse::SupervisorStderrTail",
4523 )?])
4524 }
4525
4526 async fn handle_supervisor_terminals(
4527 &self,
4528 frame: Frame,
4529 module_id: String,
4530 ) -> Result<Vec<Frame>, RouterError> {
4531 let Some(module) = self.supervisor.get(&module_id) else {
4532 return Ok(vec![control_error_frame(
4533 &frame,
4534 "unknown_module",
4535 format!("module_id '{module_id}' is not supervised"),
4536 )?]);
4537 };
4538
4539 let terminals = module
4542 .read_durable_terminal_history()
4543 .await
4544 .map_err(|error| {
4545 RouterError::backend(
4546 0,
4547 frame.header.corr,
4548 format!("failed to read terminal history: {error}"),
4549 )
4550 })?;
4551 let response = ClientControlResponse::SupervisorTerminals {
4552 module_id,
4553 terminals,
4554 };
4555 Ok(vec![control_response_body_frame(
4556 &frame,
4557 &response,
4558 "ClientControlResponse::SupervisorTerminals",
4559 )?])
4560 }
4561
4562 fn handle_supervisor_routes(
4563 &self,
4564 frame: Frame,
4565 module_id: Option<String>,
4566 ) -> Result<Vec<Frame>, RouterError> {
4567 let modules = self
4568 .forwarding
4569 .route_census(module_id.as_deref())
4570 .map_err(RouterError::Forwarding)?
4571 .into_iter()
4572 .map(|(module_id, routes)| SupervisorRouteModule {
4573 module_id,
4574 routes: routes
4575 .into_iter()
4576 .map(|route| SupervisorRoute {
4577 consumer: match route.principal {
4578 Principal::Reserved { module_id } => {
4579 SupervisorRouteConsumer::Reserved { module_id }
4580 }
4581 Principal::Direct | Principal::Unverified => {
4582 SupervisorRouteConsumer::Direct {
4583 connection_id: route.goodbye_target.connection_id.get(),
4584 }
4585 }
4586 },
4587 age_ms: Instant::now()
4588 .saturating_duration_since(route.bound_at)
4589 .as_millis()
4590 .try_into()
4591 .unwrap_or(u64::MAX),
4592 draining: route.draining,
4593 drain_reason: route.drain_reason,
4594 })
4595 .collect(),
4596 })
4597 .collect();
4598 let response = ClientControlResponse::SupervisorRoutes { modules };
4599 Ok(vec![control_response_body_frame(
4600 &frame,
4601 &response,
4602 "ClientControlResponse::SupervisorRoutes",
4603 )?])
4604 }
4605
4606 async fn handle_supervisor_provenance(
4607 &self,
4608 frame: Frame,
4609 module_id: Option<String>,
4610 ) -> Result<Vec<Frame>, RouterError> {
4611 let mut selected = if let Some(module_id) = module_id {
4612 let Some(module) = self.supervisor.get(&module_id) else {
4613 return Ok(vec![control_error_frame(
4614 &frame,
4615 "unknown_module",
4616 format!("module_id '{module_id}' is not supervised"),
4617 )?]);
4618 };
4619 vec![module]
4620 } else {
4621 self.supervisor.list()
4622 };
4623
4624 let mut modules = Vec::with_capacity(selected.len());
4625 for module in selected.drain(..) {
4626 let (status, observed_image) = module
4627 .status_and_running_image_agreement()
4628 .await
4629 .map_err(|err| {
4630 RouterError::backend(
4631 0,
4632 frame.header.corr,
4633 format!("failed to read supervisor status: {err}"),
4634 )
4635 })?;
4636 let module_declared = self
4637 .registry
4638 .get_module(&status.module_id)
4639 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4640 .and_then(|registration| registration.manifest.provenance)
4641 .map(|build| ModuleDeclaredProvenance::Reported { build })
4642 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
4643 #[cfg(test)]
4644 let running_image = match &self.provenance_probe_override {
4645 Some(result) => result.clone(),
4646 None => observed_image,
4647 };
4648 #[cfg(not(test))]
4649 let running_image = observed_image;
4650 modules.push(SupervisorModuleProvenance {
4651 module_id: status.module_id,
4652 module_declared,
4653 daemon_observed: SupervisorObservedProcess {
4654 launch_nonce_source: status.launch_nonce_source,
4655 pid: status.pid,
4656 spawned_at_ms: status.spawned_at_ms,
4657 spawned_from: status.spawned_from,
4658 running_image,
4659 },
4660 });
4661 }
4662 let daemon = SupervisorDaemonProvenance {
4663 daemon_build: self.daemon_provenance.build.clone(),
4664 daemon_observed: DaemonObservedProcess {
4665 pid: self.daemon_provenance.pid,
4666 started_at_ms: self
4667 .daemon_provenance
4668 .start_clock
4669 .map(|clock| clock.started_at_ms())
4670 .or(self.daemon_provenance.started_at_ms),
4671 running_image: self
4672 .daemon_provenance
4673 .probe
4674 .observe(
4675 self.daemon_provenance.pid,
4676 self.daemon_provenance.executable_path.as_deref(),
4677 self.daemon_provenance.executable_identity.clone(),
4678 self.daemon_provenance.process_start_time,
4679 )
4680 .await,
4681 },
4682 };
4683 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
4684 Ok(vec![control_response_body_frame(
4685 &frame,
4686 &response,
4687 "ClientControlResponse::SupervisorProvenance",
4688 )?])
4689 }
4690
4691 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4692 self.refresh_capability_requirements();
4693 let generation = self
4694 .registry
4695 .generation()
4696 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4697 let modules = self
4698 .supervisor
4699 .list()
4700 .into_iter()
4701 .map(|module| {
4702 let status = module.status_for_control("health").map_err(|err| {
4703 RouterError::backend(
4704 0,
4705 frame.header.corr,
4706 format!("failed to read supervisor health: {err}"),
4707 )
4708 })?;
4709 let module_id = status.module_id;
4710 let capability_detail = self
4711 .capability_evaluator
4712 .required_problem_detail(&module_id);
4713 Ok(SupervisorHealthEntry {
4714 module_id,
4715 status: status.health.status,
4716 detail: append_capability_problem_detail(
4717 status.health.detail,
4718 capability_detail,
4719 ),
4720 metrics: status.health.metrics,
4721 consecutive_failures: status.health.consecutive_failures,
4722 late_answer_count: status.health.late_answer_count,
4723 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
4724 last_action: status.health.last_action,
4725 last_action_ms: status.health.last_action_ms,
4726 last_probe_ms: status.health.last_probe_ms,
4727 })
4728 })
4729 .collect::<Result<Vec<_>, RouterError>>()?;
4730 let response = ClientControlResponse::SupervisorHealth {
4731 generation,
4732 modules,
4733 };
4734 Ok(vec![control_response_body_frame(
4735 &frame,
4736 &response,
4737 "ClientControlResponse::SupervisorHealth",
4738 )?])
4739 }
4740
4741 async fn handle_supervisor_restart(
4742 &self,
4743 frame: Frame,
4744 module_id: String,
4745 drain_timeout_ms: Option<u64>,
4746 ) -> Result<Vec<Frame>, RouterError> {
4747 let operation_lock = self.supervisor.operation_lock();
4748 let _operation_guard = operation_lock.lock().await;
4749 let Some(module) = self.supervisor.get(&module_id) else {
4750 return Ok(vec![control_error_frame(
4751 &frame,
4752 "unknown_module",
4753 format!("module_id '{module_id}' is not supervised"),
4754 )?]);
4755 };
4756
4757 self.route_outages.mark_operator_action(&module_id);
4758 if let Err(err) = module.restart(drain_timeout_ms).await {
4759 self.route_outages
4760 .operator_action_ended_unrefused(&module_id);
4761 let (code, message) = match err {
4762 crate::supervise::SuperviseError::Disabled { .. } => {
4763 ("module_disabled", err.to_string())
4764 }
4765 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4766 ("swap_in_progress", err.to_string())
4767 }
4768 _ => (
4769 "target_unavailable",
4770 format!("failed to restart module_id '{module_id}': {err}"),
4771 ),
4772 };
4773 return Ok(vec![control_error_frame(&frame, code, message)?]);
4774 }
4775
4776 let response = ClientControlResponse::SupervisorAck {
4777 module_id,
4778 applied: true,
4779 };
4780 Ok(vec![control_response_body_frame(
4781 &frame,
4782 &response,
4783 "ClientControlResponse::SupervisorAck",
4784 )?])
4785 }
4786
4787 async fn handle_supervisor_swap(
4791 &self,
4792 frame: Frame,
4793 module_id: String,
4794 ready_timeout_ms: Option<u64>,
4795 ) -> Result<Vec<Frame>, RouterError> {
4796 let module = {
4803 let operation_lock = self.supervisor.operation_lock();
4804 let _operation_guard = operation_lock.lock().await;
4805 self.supervisor.get(&module_id)
4806 };
4807 let Some(module) = module else {
4808 return Ok(vec![control_error_frame(
4809 &frame,
4810 "unknown_module",
4811 format!("module_id '{module_id}' is not supervised"),
4812 )?]);
4813 };
4814
4815 self.route_outages.mark_operator_action(&module_id);
4816 if let Err(err) = module
4817 .swap(ready_timeout_ms.map(Duration::from_millis))
4818 .await
4819 {
4820 self.route_outages
4821 .operator_action_ended_unrefused(&module_id);
4822 use crate::supervise::SuperviseError;
4823 let message = err.to_string();
4824 let error = match err {
4825 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
4826 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
4827 code: "swap_refused".to_string(),
4828 message,
4829 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
4830 },
4831 SuperviseError::SwapFailed {
4832 arm,
4833 candidate_exit,
4834 ..
4835 } => ErrorBody {
4836 code: "swap_failed".to_string(),
4837 message,
4838 detail: Some(serde_json::json!({
4839 "arm": arm.as_str(),
4840 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
4841 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
4842 })),
4843 },
4844 _ => ErrorBody::new(
4845 "target_unavailable",
4846 format!("failed to swap module_id '{module_id}': {message}"),
4847 ),
4848 };
4849 return Ok(vec![control_error_body_frame(&frame, error)?]);
4850 }
4851 self.route_outages
4855 .operator_action_ended_unrefused(&module_id);
4856
4857 let response = ClientControlResponse::SupervisorAck {
4858 module_id,
4859 applied: true,
4860 };
4861 Ok(vec![control_response_body_frame(
4862 &frame,
4863 &response,
4864 "ClientControlResponse::SupervisorAck",
4865 )?])
4866 }
4867
4868 async fn handle_supervisor_reload(
4869 &self,
4870 frame: Frame,
4871 module_id: String,
4872 ) -> Result<Vec<Frame>, RouterError> {
4873 let operation_lock = self.supervisor.operation_lock();
4874 let _operation_guard = operation_lock.lock().await;
4875 let Some(module) = self.supervisor.get(&module_id) else {
4876 return Ok(vec![control_error_frame(
4877 &frame,
4878 "unknown_module",
4879 format!("module_id '{module_id}' is not supervised"),
4880 )?]);
4881 };
4882
4883 self.route_outages.mark_operator_action(&module_id);
4884 if let Err(err) = module.reload().await {
4885 self.route_outages
4886 .operator_action_ended_unrefused(&module_id);
4887 let (code, message) = match err {
4888 crate::supervise::SuperviseError::Disabled { .. } => {
4889 ("module_disabled", err.to_string())
4890 }
4891 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4892 ("swap_in_progress", err.to_string())
4893 }
4894 _ => (
4895 "reload_failed",
4896 format!("failed to reload module_id '{module_id}': {err}"),
4897 ),
4898 };
4899 return Ok(vec![control_error_frame(&frame, code, message)?]);
4900 }
4901
4902 let response = ClientControlResponse::SupervisorAck {
4903 module_id,
4904 applied: true,
4905 };
4906 Ok(vec![control_response_body_frame(
4907 &frame,
4908 &response,
4909 "ClientControlResponse::SupervisorAck",
4910 )?])
4911 }
4912
4913 async fn handle_supervisor_rescan(
4914 &self,
4915 frame: Frame,
4916 preview: bool,
4917 ) -> Result<Vec<Frame>, RouterError> {
4918 let Some(context) = self.rescan.clone() else {
4919 return Ok(vec![control_error_frame(
4920 &frame,
4921 "rescan_unavailable",
4922 "the daemon was not started with a reloadable config path".to_string(),
4923 )?]);
4924 };
4925
4926 let operation_lock = self.supervisor.operation_lock();
4927 let _operation_guard = operation_lock.lock().await;
4928 let loaded = match crate::daemon_config::load(&context.config_path) {
4929 Ok(config) => config,
4930 Err(err) => {
4931 return Ok(vec![control_error_frame(
4932 &frame,
4933 "invalid_daemon_config",
4934 format!("supervisor rescan rejected daemon config: {err}"),
4935 )?])
4936 }
4937 };
4938 let Some(config) = loaded else {
4946 return Ok(vec![control_error_frame(
4947 &frame,
4948 "invalid_daemon_config",
4949 format!(
4950 "daemon config not found at {}; refusing to rescan (an absent config would \
4951 retire every supervised module)",
4952 context.config_path.display()
4953 ),
4954 )?]);
4955 };
4956 let (
4957 configured_port,
4958 storage_config,
4959 admission_facts_carrier_module_id,
4960 admission_facts_targets,
4961 scope_authority_owners,
4962 modules,
4963 reserved_capabilities,
4964 ) = (
4965 config.port,
4966 config.storage,
4967 config.admission_facts_carrier_module_id,
4968 config.admission_facts_targets,
4969 config.scope_authority_owners,
4970 config.modules,
4971 config.reserved_capabilities,
4972 );
4973
4974 let mut restart_required = Vec::new();
4983 for section in RestartRequiredSection::ALL {
4984 let changed = match section {
4985 RestartRequiredSection::Port => configured_port != context.configured_port,
4986 RestartRequiredSection::Storage => storage_config != context.storage_config,
4987 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
4988 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
4989 }
4990 RestartRequiredSection::AdmissionFactsTargets => {
4991 admission_facts_targets != context.admission_facts_targets
4992 }
4993 RestartRequiredSection::ScopeAuthorityOwners => {
4994 scope_authority_owners != context.scope_authority_owners
4995 }
4996 };
4997 if changed {
4998 restart_required.push(section.label().to_string());
4999 }
5000 }
5001 if !restart_required.is_empty() {
5002 warn!(
5003 config_path = %context.config_path.display(),
5004 sections = %restart_required.join(", "),
5005 "daemon config changed outside the modules section; restart the daemon to apply those changes"
5006 );
5007 }
5008
5009 for configured in &modules {
5010 if let Err(err) = validate_spec(&configured.module_spec()) {
5011 return Ok(vec![control_error_frame(
5012 &frame,
5013 "invalid_daemon_config",
5014 format!("supervisor rescan rejected daemon config: {err}"),
5015 )?]);
5016 }
5017 }
5018
5019 let configured_capabilities = modules
5020 .iter()
5021 .map(|module| (module.module_id.clone(), module.enabled))
5022 .collect::<Vec<_>>();
5023 let preview_capability_warnings = if preview {
5024 let (_, registrations) = self.runtime_capability_snapshot()?;
5025 let current_modules = self
5026 .supervisor
5027 .list()
5028 .into_iter()
5029 .map(|module| module.module_id().to_string())
5030 .collect::<BTreeSet<_>>();
5031 let resulting_modules = configured_capabilities.clone();
5032 let removed = current_modules
5033 .into_iter()
5034 .filter(|module_id| {
5035 !resulting_modules
5036 .iter()
5037 .any(|(configured_id, _)| configured_id == module_id)
5038 })
5039 .collect::<Vec<_>>();
5040 self.capability_evaluator.preview_removal_warnings(
5041 resulting_modules,
5042 &removed,
5043 ®istrations,
5044 )
5045 } else {
5046 Vec::new()
5047 };
5048 let result = match self
5049 .reconcile_supervised_modules(&context.supervisor, modules, preview)
5050 .await
5051 {
5052 Ok(result) => result,
5053 Err(message) => {
5054 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5055 }
5056 };
5057 if !preview {
5058 self.capability_evaluator
5059 .configure(configured_capabilities, reserved_capabilities);
5060 self.capability_evaluator.wake_deadline_loop();
5061 self.refresh_capability_requirements();
5062 }
5063 let mut result = result;
5064 result.restart_required = restart_required;
5065 result.capability_warnings = preview_capability_warnings;
5066 let response = ClientControlResponse::SupervisorRescan { result };
5067 Ok(vec![control_response_body_frame(
5068 &frame,
5069 &response,
5070 "ClientControlResponse::SupervisorRescan",
5071 )?])
5072 }
5073
5074 async fn handle_supervisor_release_reserved(
5075 &self,
5076 frame: Frame,
5077 module_id: String,
5078 ) -> Result<Vec<Frame>, RouterError> {
5079 let Some(context) = self.rescan.clone() else {
5080 return Ok(vec![control_error_frame(
5081 &frame,
5082 "release_unavailable",
5083 "reserved-id release requires a daemon started with a reloadable config path",
5084 )?]);
5085 };
5086 let operation_lock = self.supervisor.operation_lock();
5087 let _operation_guard = operation_lock.lock().await;
5088 let loaded = match crate::daemon_config::load(&context.config_path) {
5089 Ok(Some(config)) => config,
5090 Ok(None) => {
5091 return Ok(vec![control_error_frame(
5092 &frame,
5093 "invalid_daemon_config",
5094 format!(
5095 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5096 context.config_path.display()
5097 ),
5098 )?])
5099 }
5100 Err(err) => {
5101 return Ok(vec![control_error_frame(
5102 &frame,
5103 "invalid_daemon_config",
5104 format!("unable to verify reserved-id release against daemon config: {err}"),
5105 )?])
5106 }
5107 };
5108 if loaded
5109 .modules
5110 .iter()
5111 .any(|configured| configured.module_id == module_id)
5112 {
5113 return Ok(vec![control_error_frame(
5114 &frame,
5115 "reserved_module_configured",
5116 format!(
5117 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5118 ),
5119 )?]);
5120 }
5121 if !self.supervisor.release_retained_reserved_gate(&module_id) {
5122 return Ok(vec![control_error_frame(
5123 &frame,
5124 "reserved_gate_not_retained",
5125 format!(
5126 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
5127 ),
5128 )?]);
5129 }
5130
5131 let response = ClientControlResponse::SupervisorAck {
5132 module_id,
5133 applied: true,
5134 };
5135 Ok(vec![control_response_body_frame(
5136 &frame,
5137 &response,
5138 "ClientControlResponse::SupervisorAck",
5139 )?])
5140 }
5141
5142 async fn reconcile_supervised_modules(
5151 &self,
5152 supervisor: &Supervisor,
5153 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
5154 preview: bool,
5155 ) -> Result<SupervisorRescanResult, String> {
5156 let mut current = BTreeMap::new();
5157 for module in self.supervisor.list() {
5158 let (spec, health) = module.configuration().map_err(|err| {
5159 format!(
5160 "failed to read configuration for module_id '{}': {err}",
5161 module.module_id()
5162 )
5163 })?;
5164 let enabled = module
5165 .status()
5166 .map_err(|err| {
5167 format!(
5168 "failed to read status for module_id '{}': {err}",
5169 module.module_id()
5170 )
5171 })?
5172 .enabled;
5173 current.insert(
5174 module.module_id().to_string(),
5175 (module, spec, health, enabled),
5176 );
5177 }
5178 let configured = configured_modules
5179 .into_iter()
5180 .map(|module| (module.module_id.clone(), module))
5181 .collect::<BTreeMap<_, _>>();
5182
5183 let added = configured
5184 .keys()
5185 .filter(|module_id| !current.contains_key(*module_id))
5186 .cloned()
5187 .collect::<Vec<_>>();
5188 let removed = current
5189 .keys()
5190 .filter(|module_id| !configured.contains_key(*module_id))
5191 .cloned()
5192 .collect::<Vec<_>>();
5193 let mut changed_pending_reload = Vec::new();
5194 let mut configuration_changes = BTreeSet::new();
5195 let mut enabled_changes = BTreeSet::new();
5196 let mut unchanged = 0_u32;
5197
5198 for (module_id, configured_module) in &configured {
5199 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5200 else {
5201 continue;
5202 };
5203 let launch_changed = *current_spec != configured_module.module_spec();
5207 let configuration_changed =
5208 launch_changed || *current_health != configured_module.health;
5209 let enabled_changed = *current_enabled != configured_module.enabled;
5210 if configuration_changed {
5211 configuration_changes.insert(module_id.clone());
5212 }
5213 if launch_changed {
5214 changed_pending_reload.push(module_id.clone());
5215 }
5216 if enabled_changed {
5217 enabled_changes.insert(module_id.clone());
5218 }
5219 if !configuration_changed && !enabled_changed {
5220 unchanged = unchanged.saturating_add(1);
5221 }
5222 }
5223
5224 if preview {
5230 return Ok(SupervisorRescanResult {
5231 added,
5232 removed,
5233 changed_pending_reload,
5234 enabled_changes: enabled_changes.iter().cloned().collect(),
5235 unchanged,
5236 preview: true,
5237 restart_required: Vec::new(),
5241 capability_warnings: Vec::new(),
5242 });
5243 }
5244
5245 for module_id in &removed {
5246 let module = ¤t
5247 .get(module_id)
5248 .expect("removed module came from current supervisor state")
5249 .0;
5250 module.retire().await.map_err(|err| {
5251 format!("failed to retire module_id '{module_id}' during rescan: {err}")
5252 })?;
5253 self.supervisor.record_rescan_removal(module_id);
5287 self.supervisor.retire(module_id);
5288 self.route_outages.forget(module_id);
5289 }
5290
5291 for module_id in configured.keys() {
5292 let Some((module, _, _, _)) = current.get(module_id) else {
5293 continue;
5294 };
5295 let configured_module = configured
5296 .get(module_id)
5297 .expect("configured module id came from configured map");
5298 if configuration_changes.contains(module_id) {
5299 module
5300 .update_configuration(
5301 configured_module.module_spec(),
5302 configured_module.health.clone(),
5303 configured_module.drain_timeout_ms,
5304 )
5305 .await
5306 .map_err(|err| {
5307 format!(
5308 "failed to update module_id '{module_id}' configuration during rescan: {err}"
5309 )
5310 })?;
5311 }
5312 if enabled_changes.contains(module_id) {
5313 self.route_outages.mark_operator_action(module_id);
5316 module
5317 .set_enabled(configured_module.enabled)
5318 .await
5319 .map_err(|err| {
5320 self.route_outages.operator_action_ended_unrefused(module_id);
5321 format!(
5322 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
5323 configured_module.enabled
5324 )
5325 })?;
5326 }
5327 }
5328
5329 for module_id in &added {
5330 let configured_module = configured
5331 .get(module_id)
5332 .expect("added module id came from configured map");
5333 supervisor
5334 .supervise_configured_with_health(
5335 configured_module.module_spec(),
5336 configured_module.enabled,
5337 configured_module.health.clone(),
5338 configured_module.drain_timeout_ms,
5339 configured_module.restart,
5340 )
5341 .map_err(|err| {
5342 format!("failed to add module_id '{module_id}' during rescan: {err}")
5343 })?;
5344 }
5345
5346 Ok(SupervisorRescanResult {
5347 added,
5348 removed,
5349 changed_pending_reload,
5350 enabled_changes: enabled_changes.iter().cloned().collect(),
5351 unchanged,
5352 preview: false,
5353 restart_required: Vec::new(),
5356 capability_warnings: Vec::new(),
5357 })
5358 }
5359
5360 async fn handle_supervisor_set_enabled(
5361 &self,
5362 frame: Frame,
5363 module_id: String,
5364 enabled: bool,
5365 ) -> Result<Vec<Frame>, RouterError> {
5366 let operation_lock = self.supervisor.operation_lock();
5367 let _operation_guard = operation_lock.lock().await;
5368 let Some(module) = self.supervisor.get(&module_id) else {
5369 return Ok(vec![control_error_frame(
5370 &frame,
5371 "unknown_module",
5372 format!("module_id '{module_id}' is not supervised"),
5373 )?]);
5374 };
5375
5376 self.route_outages.mark_operator_action(&module_id);
5379 let applied = match module.set_enabled(enabled).await {
5380 Ok(applied) => applied,
5381 Err(err) => {
5382 self.route_outages
5383 .operator_action_ended_unrefused(&module_id);
5384 return Ok(vec![control_error_frame(
5385 &frame,
5386 "target_unavailable",
5387 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
5388 )?]);
5389 }
5390 };
5391 if !applied {
5392 self.route_outages
5395 .operator_action_ended_unrefused(&module_id);
5396 }
5397
5398 self.capability_evaluator.wake_deadline_loop();
5399 self.refresh_capability_requirements();
5400 let response = ClientControlResponse::SupervisorAck { module_id, applied };
5401 Ok(vec![control_response_body_frame(
5402 &frame,
5403 &response,
5404 "ClientControlResponse::SupervisorAck",
5405 )?])
5406 }
5407
5408 async fn handle_supervisor_health_probe(
5409 &self,
5410 frame: Frame,
5411 module_id: String,
5412 ) -> Result<Vec<Frame>, RouterError> {
5413 self.refresh_capability_requirements();
5414 let Some(registration) = self
5415 .registry
5416 .get_module(&module_id)
5417 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5418 else {
5419 return Ok(vec![control_error_frame(
5420 &frame,
5421 "unknown_module",
5422 format!("module_id '{module_id}' is not registered"),
5423 )?]);
5424 };
5425
5426 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
5439 {
5440 return Ok(vec![control_error_frame(
5441 &frame,
5442 "health_not_advertised",
5443 format!("module_id '{module_id}' did not advertise health.check"),
5444 )?]);
5445 }
5446
5447 let deadline = Instant::now() + self.health_probe_timeout;
5448 let pending = match self.forwarding.begin_module_control_rpc_for(
5449 &module_id,
5450 MODULE_CONTROL_OP_HEALTH_CHECK,
5451 deadline,
5452 ) {
5453 Ok(pending) => pending,
5454 Err(err) => {
5455 return Ok(vec![control_error_frame(
5456 &frame,
5457 forwarding_error_code(&err),
5458 err.to_string(),
5459 )?])
5460 }
5461 };
5462
5463 let PendingModuleControlRpc {
5464 endpoint,
5465 module_sink,
5466 negotiated_ver,
5467 corr: probe_corr,
5468 receiver,
5469 } = pending;
5470 let mut guard =
5471 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
5472 let probe_body =
5473 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5474 RouterError::backend(
5475 0,
5476 frame.header.corr,
5477 format!("failed to encode health.check request: {err}"),
5478 )
5479 })?;
5480 let probe_frame = Frame::build_with_version(
5481 negotiated_ver,
5482 FrameType::Request,
5483 control_flags(),
5484 0,
5485 0,
5486 probe_corr,
5487 probe_body,
5488 )
5489 .map_err(RouterError::FrameBuild)?;
5490
5491 if let Err(err) = module_sink.send(probe_frame).await {
5492 return Ok(vec![control_error_frame(
5493 &frame,
5494 "target_unavailable",
5495 err.to_string(),
5496 )?]);
5497 }
5498
5499 match timeout_at(deadline, receiver).await {
5500 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5501 guard.disarm();
5502 let Some(report) = response.health_report() else {
5503 return Ok(vec![control_error_frame(
5504 &frame,
5505 "invalid_control_body",
5506 "health.check RPC returned a non-health response",
5507 )?]);
5508 };
5509 let HealthReport {
5514 status,
5515 detail,
5516 metrics,
5517 } = report;
5518 let capability_detail = self
5519 .capability_evaluator
5520 .required_problem_detail(&module_id);
5521 let response = ClientControlResponse::SupervisorHealthProbe {
5522 module_id,
5523 status,
5524 detail: append_capability_problem_detail(detail, capability_detail),
5525 metrics,
5526 };
5527 Ok(vec![control_response_body_frame(
5528 &frame,
5529 &response,
5530 "ClientControlResponse::SupervisorHealthProbe",
5531 )?])
5532 }
5533 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
5534 guard.disarm();
5535 Ok(vec![control_error_body_frame(&frame, body)?])
5536 }
5537 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5538 guard.disarm();
5539 Ok(vec![control_error_frame(
5540 &frame,
5541 "target_unavailable",
5542 message,
5543 )?])
5544 }
5545 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5546 guard.disarm();
5547 Ok(vec![control_error_frame(
5548 &frame,
5549 "invalid_control_body",
5550 message,
5551 )?])
5552 }
5553 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5554 guard.disarm();
5555 Ok(vec![control_error_frame(
5556 &frame,
5557 "invalid_control_body",
5558 format!("expected module-control op '{expected}', got '{actual}'"),
5559 )?])
5560 }
5561 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
5562 guard.disarm();
5563 Ok(vec![control_error_frame(
5564 &frame,
5565 "module_timeout",
5566 format!(
5567 "module_id '{module_id}' answered health.check after {:?}",
5568 self.health_probe_timeout
5569 ),
5570 )?])
5571 }
5572 Ok(Err(_)) => Ok(vec![control_error_frame(
5573 &frame,
5574 "target_unavailable",
5575 "health.check waiter was canceled before the module responded",
5576 )?]),
5577 Err(_) => Ok(vec![control_error_frame(
5578 &frame,
5579 "module_timeout",
5580 format!(
5581 "module_id '{module_id}' did not answer health.check within {:?}",
5582 self.health_probe_timeout
5583 ),
5584 )?]),
5585 }
5586 }
5587
5588 fn supervisor_status(
5589 &self,
5590 module_id: &str,
5591 corr: u64,
5592 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
5593 self.supervisor
5594 .get(module_id)
5595 .map(|module| {
5596 let warming = module.is_warming_for_control("status").map_err(|err| {
5597 RouterError::backend(
5598 0,
5599 corr,
5600 format!(
5601 "failed to read supervisor warming state for module_id '{module_id}': {err}"
5602 ),
5603 )
5604 })?;
5605 module.status_for_control("status").map_err(|err| {
5606 RouterError::backend(
5607 0,
5608 corr,
5609 format!(
5610 "failed to read supervisor status for module_id '{module_id}': {err}"
5611 ),
5612 )
5613 }).map(|status| (status, warming))
5614 })
5615 .transpose()
5616 }
5617
5618 fn guard_module_control_op(
5619 &self,
5620 frame: &Frame,
5621 module_id: &str,
5622 op: &str,
5623 ) -> Result<Option<Frame>, RouterError> {
5624 if self.module_grants_op(module_id, op, frame.header.corr)? {
5625 return Ok(None);
5626 }
5627
5628 Ok(Some(control_error_frame(
5629 frame,
5630 "op_not_allowed",
5631 format!("module_id '{module_id}' did not grant control op '{op}'"),
5632 )?))
5633 }
5634
5635 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
5636 let Some(registration) = self
5637 .registry
5638 .get_module(module_id)
5639 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
5640 else {
5641 return Ok(false);
5642 };
5643 Ok(module_registration_grants_op(®istration.control_ops, op))
5644 }
5645
5646 fn handle_status_update(
5647 &self,
5648 endpoint: ModuleEndpointId,
5649 frame: Frame,
5650 ) -> Result<Vec<Frame>, RouterError> {
5651 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
5652 Ok(update) => update,
5653 Err(err) => {
5654 if is_known_module_push_op(&frame.body) {
5659 return Ok(vec![control_error_frame(
5660 &frame,
5661 "invalid_control_body",
5662 format!("malformed module control push body: {err}"),
5663 )?]);
5664 }
5665 return Ok(Vec::new());
5666 }
5667 };
5668
5669 match update {
5670 ModuleControlPush::RouteStatus {
5671 route_channel,
5672 route_epoch,
5673 status,
5674 } => {
5675 self.forwarding
5676 .cache_status(endpoint, route_channel, route_epoch, status)
5677 .map_err(RouterError::Forwarding)?;
5678 }
5679 }
5680 Ok(Vec::new())
5681 }
5682
5683 fn handle_route_poll(
5684 &self,
5685 ctx: &RouteCtx,
5686 frame: Frame,
5687 route_channel: u16,
5688 route_epoch: u32,
5689 kind: PollKind,
5690 ) -> Result<Vec<Frame>, RouterError> {
5691 let snapshot = self
5692 .forwarding
5693 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
5694 .map_err(RouterError::Forwarding)?;
5695 let response = match (kind, snapshot) {
5696 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
5697 ClientControlResponse::RoutePoll {
5698 route_channel,
5699 route_epoch,
5700 status,
5701 live: None,
5702 }
5703 }
5704 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5705 route_channel,
5706 route_epoch,
5707 status: None,
5708 live: None,
5709 },
5710 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
5711 let live = self
5737 .process_liveness
5738 .as_ref()
5739 .and_then(|source| source.process_live(&module_id))
5740 .unwrap_or(true);
5741 ClientControlResponse::RoutePoll {
5742 route_channel,
5743 route_epoch,
5744 status: None,
5745 live: Some(live),
5746 }
5747 }
5748 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5749 route_channel,
5750 route_epoch,
5751 status: None,
5752 live: Some(false),
5753 },
5754 };
5755
5756 Ok(vec![control_response_body_frame(
5757 &frame,
5758 &response,
5759 "ClientControlResponse::RoutePoll",
5760 )?])
5761 }
5762
5763 pub(crate) fn observe_module_control_completion(
5764 &self,
5765 completion: ModuleControlRpcCompletion,
5766 ) -> bool {
5767 match completion {
5768 ModuleControlRpcCompletion::Unknown => false,
5769 ModuleControlRpcCompletion::Settled => true,
5770 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
5771 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
5772 info!(
5773 module_id = %module_id,
5774 latency_ms,
5775 "late health.check answer proves the module is alive"
5776 );
5777 match self
5778 .supervisor
5779 .record_late_health_answer(&module_id, latency_ms)
5780 {
5781 Ok(true) => {}
5782 Ok(false) => debug!(
5783 module_id = %module_id,
5784 latency_ms,
5785 "late health.check answer has no active supervisor snapshot"
5786 ),
5787 Err(err) => warn!(
5788 module_id = %module_id,
5789 latency_ms,
5790 error = %err,
5791 "failed to record late health.check answer"
5792 ),
5793 }
5794 true
5795 }
5796 }
5797 }
5798
5799 fn refuse_to_end_module_connection_for_a_client(
5820 &self,
5821 module_connection_id: ConnectionId,
5822 corr: u64,
5823 err: ForwardingError,
5824 ) -> Result<(), RouterError> {
5825 if let ForwardingError::ConnectionClosing { connection_id } = err {
5826 if connection_id != module_connection_id {
5827 warn!(
5828 module_connection_id = module_connection_id.get(),
5829 client_connection_id = connection_id.get(),
5830 corr,
5831 "dropping a route.bind response for a closing client; the module connection keeps serving"
5832 );
5833 return Ok(());
5834 }
5835 }
5836 Err(RouterError::Forwarding(err))
5837 }
5838
5839 fn handle_module_relay_response(
5840 &self,
5841 connection_id: ConnectionId,
5842 frame: Frame,
5843 ) -> Result<Vec<Frame>, RouterError> {
5844 let mut secondary_error = None;
5845 let outcome = match frame.header.ty {
5846 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
5847 Ok(probe) if probe.op == "route.bind" => {
5848 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
5849 Ok(ModuleControlResponse::RouteBindAck {}) => {
5850 RouteBindRelayOutcome::Accepted
5851 }
5852 Ok(other) => {
5853 let message =
5854 format!("route.bind response carried unexpected body: {other:?}");
5855 secondary_error = Some(control_error_frame(
5856 &frame,
5857 "invalid_control_body",
5858 message.clone(),
5859 )?);
5860 RouteBindRelayOutcome::ModuleGone(message)
5861 }
5862 Err(err) => {
5863 let message = format!("malformed route.bind response body: {err}");
5864 secondary_error = Some(control_error_frame(
5865 &frame,
5866 "invalid_control_body",
5867 message.clone(),
5868 )?);
5869 RouteBindRelayOutcome::ModuleGone(message)
5870 }
5871 }
5872 }
5873 Ok(probe) => {
5874 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
5875 {
5876 Ok(response) => ModuleControlRpcOutcome::Response(response),
5877 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5878 "malformed {} response body: {err}",
5879 probe.op
5880 )),
5881 };
5882 let completion = self
5883 .forwarding
5884 .complete_module_control_rpc(
5885 connection_id,
5886 frame.header.corr,
5887 Some(&probe.op),
5888 outcome,
5889 )
5890 .map_err(RouterError::Forwarding)?;
5891 if !self.observe_module_control_completion(completion) {
5892 debug!(
5893 connection_id = connection_id.get(),
5894 corr = frame.header.corr,
5895 op = %probe.op,
5896 "dropping late or unknown module-control RPC response"
5897 );
5898 }
5899 return Ok(Vec::new());
5900 }
5901 Err(err) => {
5902 if let Some(expected_op) = self
5903 .forwarding
5904 .pending_module_control_op(connection_id, frame.header.corr)
5905 .map_err(RouterError::Forwarding)?
5906 {
5907 let completion = self
5908 .forwarding
5909 .complete_module_control_rpc(
5910 connection_id,
5911 frame.header.corr,
5912 None,
5913 ModuleControlRpcOutcome::MalformedResponse(format!(
5914 "malformed {expected_op} response body: {err}"
5915 )),
5916 )
5917 .map_err(RouterError::Forwarding)?;
5918 if !self.observe_module_control_completion(completion) {
5919 debug!(
5920 connection_id = connection_id.get(),
5921 corr = frame.header.corr,
5922 "dropping late malformed module-control RPC response"
5923 );
5924 }
5925 return Ok(Vec::new());
5926 }
5927 let message = format!("malformed route.bind response body: {err}");
5928 secondary_error = Some(control_error_frame(
5929 &frame,
5930 "invalid_control_body",
5931 message.clone(),
5932 )?);
5933 RouteBindRelayOutcome::ModuleGone(message)
5934 }
5935 },
5936 FrameType::Error => {
5937 if self
5938 .forwarding
5939 .pending_module_control_op(connection_id, frame.header.corr)
5940 .map_err(RouterError::Forwarding)?
5941 .is_some()
5942 {
5943 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
5944 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
5945 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5946 "malformed module-control ERROR body: {err}"
5947 )),
5948 };
5949 let completion = self
5950 .forwarding
5951 .complete_module_control_rpc(
5952 connection_id,
5953 frame.header.corr,
5954 None,
5955 outcome,
5956 )
5957 .map_err(RouterError::Forwarding)?;
5958 if !self.observe_module_control_completion(completion) {
5959 debug!(
5960 connection_id = connection_id.get(),
5961 corr = frame.header.corr,
5962 "dropping late or unknown module-control RPC error"
5963 );
5964 }
5965 return Ok(Vec::new());
5966 }
5967 match serde_json::from_slice::<ErrorBody>(&frame.body) {
5968 Ok(body) => RouteBindRelayOutcome::Rejected(body),
5969 Err(err) => {
5970 let message = format!("malformed route.bind ERROR body: {err}");
5971 secondary_error = Some(control_error_frame(
5972 &frame,
5973 "invalid_control_body",
5974 message.clone(),
5975 )?);
5976 RouteBindRelayOutcome::ModuleGone(message)
5977 }
5978 }
5979 }
5980 ty => {
5981 return Ok(vec![control_error_frame(
5982 &frame,
5983 "unsupported_control_frame",
5984 format!("unsupported module channel-0 frame {ty:?}"),
5985 )?])
5986 }
5987 };
5988
5989 let settled =
5990 self.forwarding
5991 .complete_pending_relay(connection_id, frame.header.corr, outcome);
5992 let completion = match settled {
5993 Ok(completion) => completion,
5994 Err(err) => {
5995 self.refuse_to_end_module_connection_for_a_client(
5996 connection_id,
5997 frame.header.corr,
5998 err,
5999 )?;
6000 return Ok(secondary_error.into_iter().collect());
6001 }
6002 };
6003 if let Some(target) = completion.abandoned.as_ref() {
6004 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6005 }
6006 if !completion.settled {
6007 debug!(
6008 connection_id = connection_id.get(),
6009 corr = frame.header.corr,
6010 frame_type = ?frame.header.ty,
6011 "dropping late or unknown route.bind relay response"
6012 );
6013 }
6014 Ok(secondary_error.into_iter().collect())
6015 }
6016
6017 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6018 debug!(connection_id = connection_id.get(), "handling GOODBYE");
6019 self.cleanup_connection_with_end_reason(
6023 connection_id,
6024 RegistrationEndReason::ExplicitGoodbye,
6025 )
6026 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6027 Ok(Vec::new())
6028 }
6029}
6030
6031impl Default for ControlHandler {
6032 fn default() -> Self {
6033 Self::new(Arc::new(Registry::default()))
6034 }
6035}
6036
6037impl crate::supervise::SwapPromotionObserver for ControlHandler {
6038 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6039 self.apply_registration_capabilities(registration);
6040 }
6041}
6042
6043fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6044 CapabilityRequirementStatus {
6045 consumer: status.consumer,
6046 capability: status.capability,
6047 need: match status.need {
6048 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6049 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6050 },
6051 verdict: status.verdict.as_str().to_string(),
6052 episode_seq: status.episode_seq,
6053 config_satisfiable: status.config_satisfiable,
6054 runtime_available: status.runtime_available,
6055 detail: status.detail,
6056 }
6057}
6058
6059fn append_capability_problem_detail(
6060 detail: Option<String>,
6061 capability_detail: Option<String>,
6062) -> Option<String> {
6063 match (detail, capability_detail) {
6064 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
6065 (Some(detail), None) => Some(detail),
6066 (None, Some(capability_detail)) => Some(capability_detail),
6067 (None, None) => None,
6068 }
6069}
6070
6071fn subc_ops() -> Vec<String> {
6072 SUBC_CONTROL_OPS
6073 .iter()
6074 .map(|op| (*op).to_string())
6075 .collect()
6076}
6077
6078fn module_subc_ops() -> Vec<String> {
6079 SUBC_CONTROL_OPS
6080 .iter()
6081 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
6082 .map(|op| (*op).to_string())
6083 .collect()
6084}
6085
6086#[cfg(test)]
6087fn module_baseline_control_ops() -> Vec<String> {
6088 MODULE_BASELINE_CONTROL_OPS
6089 .iter()
6090 .map(|op| (*op).to_string())
6091 .collect()
6092}
6093
6094fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
6095 let mut seen = HashSet::new();
6096 let mut effective = Vec::new();
6097 for op in MODULE_BASELINE_CONTROL_OPS {
6098 if seen.insert((*op).to_string()) {
6099 effective.push((*op).to_string());
6100 }
6101 }
6102 for op in declared.unwrap_or_default() {
6103 if seen.insert(op.clone()) {
6104 effective.push(op);
6105 }
6106 }
6107 effective
6108}
6109
6110fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
6111 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
6112}
6113
6114fn target_module_id(target: &RouteTarget) -> &str {
6115 match target {
6116 RouteTarget::ToolProvider { module_id }
6117 | RouteTarget::ManagementSurface { module_id }
6118 | RouteTarget::InternalService { module_id, .. } => module_id,
6119 }
6120}
6121
6122fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
6123 roles.iter().any(|role| match (target, role) {
6124 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
6125 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
6126 (
6127 RouteTarget::InternalService { service_id, .. },
6128 ProviderRole::InternalService {
6129 service_id: provided,
6130 ..
6131 },
6132 ) => service_id == provided,
6133 _ => false,
6134 })
6135}
6136
6137fn is_routable_role(role: &ProviderRole) -> bool {
6138 matches!(
6139 role,
6140 ProviderRole::ToolProvider { .. }
6141 | ProviderRole::ManagementSurface { .. }
6142 | ProviderRole::InternalService { .. }
6143 )
6144}
6145
6146#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6147enum ControlRequestBodyError {
6148 UnknownOp,
6149 InvalidBody,
6150}
6151
6152#[derive(Debug, Deserialize)]
6153struct ControlOpProbe {
6154 op: String,
6155}
6156
6157const MODULE_PUSH_OPS: &[&str] = &["route.status"];
6160
6161fn is_known_module_push_op(body: &[u8]) -> bool {
6162 serde_json::from_slice::<ControlOpProbe>(body)
6163 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
6164 .unwrap_or(false)
6165}
6166
6167fn is_known_module_request_op(body: &[u8]) -> bool {
6168 serde_json::from_slice::<ControlOpProbe>(body)
6169 .map(|probe| is_module_to_subc_op(&probe.op))
6170 .unwrap_or(false)
6171}
6172
6173fn is_module_to_subc_op(op: &str) -> bool {
6174 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
6175}
6176
6177fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
6178 debug!(
6179 op = %op,
6180 connection_id = connection_id.get(),
6181 corr,
6182 "control dispatch"
6183 );
6184}
6185
6186fn log_slow_control_dispatch(
6187 dispatch_started_at: Option<StdInstant>,
6188 op: &'static str,
6189 connection_id: ConnectionId,
6190 corr: u64,
6191) {
6192 let Some(dispatch_started_at) = dispatch_started_at else {
6193 return;
6194 };
6195 let elapsed = dispatch_started_at.elapsed();
6196 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
6197 warn!(
6198 op = %op,
6199 connection_id = connection_id.get(),
6200 corr,
6201 elapsed_ms = elapsed.as_millis() as u64,
6202 "slow control dispatch"
6203 );
6204 }
6205}
6206
6207fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
6208 match request {
6209 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
6210 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
6211 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
6212 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
6213 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
6214 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
6215 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
6216 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
6217 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
6218 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
6219 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
6220 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
6221 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
6222 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
6223 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
6224 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
6225 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
6226 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
6227 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
6228 }
6229}
6230
6231fn principal_label(principal: &Principal) -> String {
6232 match principal {
6233 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
6234 Principal::Direct => "direct".to_string(),
6235 other => format!("{other:?}"),
6236 }
6237}
6238
6239fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
6240 match request {
6241 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
6242 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
6243 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
6244 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
6245 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
6246 }
6247}
6248
6249fn parse_client_control_request(
6250 body: &[u8],
6251) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
6252 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
6253 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6254 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
6255 ControlRequestBodyError::InvalidBody
6256 }
6257 Ok(_) => ControlRequestBodyError::UnknownOp,
6258 Err(_) => ControlRequestBodyError::InvalidBody,
6259 };
6260 (err, classification)
6261 })
6262}
6263
6264fn parse_module_control_request_from_module(
6265 body: &[u8],
6266) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
6267 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
6268 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6269 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
6270 Ok(_) => ControlRequestBodyError::UnknownOp,
6271 Err(_) => ControlRequestBodyError::InvalidBody,
6272 };
6273 (err, classification)
6274 })
6275}
6276
6277#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
6278enum ProviderRoleKind {
6279 ToolProvider,
6280 PipelineStage,
6281 ManagementSurface,
6282 InternalService,
6283}
6284
6285fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
6286 match role {
6287 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
6288 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
6289 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
6290 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
6291 }
6292}
6293
6294fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
6295 roles.iter().map(provider_role_kind).collect()
6296}
6297
6298const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
6301
6302#[derive(Debug, Default, PartialEq, Eq)]
6304struct ScopeOutcomeCounts {
6305 created: usize,
6306 replaced: usize,
6307 updated: usize,
6308 unchanged: usize,
6309 refused: usize,
6310}
6311
6312impl ScopeOutcomeCounts {
6313 fn of(results: &[ScopeRecordResult]) -> Self {
6314 let mut counts = Self::default();
6315 for result in results {
6316 let slot = match result.outcome {
6317 ScopeRecordOutcome::Created => &mut counts.created,
6318 ScopeRecordOutcome::Replaced => &mut counts.replaced,
6319 ScopeRecordOutcome::Updated => &mut counts.updated,
6320 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
6321 ScopeRecordOutcome::Refused => &mut counts.refused,
6322 };
6323 *slot += 1;
6324 }
6325 counts
6326 }
6327}
6328
6329#[cfg(test)]
6330mod scope_outcome_count_tests {
6331 use super::*;
6332
6333 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
6334 ScopeRecordResult {
6335 scope_ref: "r".to_string(),
6336 scope_epoch: 1,
6337 outcome,
6338 code: None,
6339 message: None,
6340 version: None,
6341 parent_state: None,
6342 }
6343 }
6344
6345 #[test]
6348 fn every_outcome_is_counted_in_its_own_field() {
6349 let results = [
6350 result(ScopeRecordOutcome::Created),
6351 result(ScopeRecordOutcome::Created),
6352 result(ScopeRecordOutcome::Replaced),
6353 result(ScopeRecordOutcome::Updated),
6354 result(ScopeRecordOutcome::Unchanged),
6355 result(ScopeRecordOutcome::Refused),
6356 result(ScopeRecordOutcome::Refused),
6357 result(ScopeRecordOutcome::Refused),
6358 ];
6359 assert_eq!(
6360 ScopeOutcomeCounts::of(&results),
6361 ScopeOutcomeCounts {
6362 created: 2,
6363 replaced: 1,
6364 updated: 1,
6365 unchanged: 1,
6366 refused: 3,
6367 }
6368 );
6369 }
6370}
6371
6372fn capability_census_trigger(
6376 old: Option<&CapabilityDeclarations>,
6377 new: Option<&CapabilityDeclarations>,
6378) -> bool {
6379 let old_provides = old
6380 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
6381 .unwrap_or_default();
6382 let old_denies = old
6383 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
6384 .unwrap_or_default();
6385 let new = new.cloned().unwrap_or(CapabilityDeclarations {
6386 provides: Vec::new(),
6387 requires: Vec::new(),
6388 must_never_reach: Vec::new(),
6389 });
6390
6391 new.provides
6392 .iter()
6393 .any(|capability| !old_provides.contains(capability))
6394 || new
6395 .must_never_reach
6396 .iter()
6397 .any(|capability| !old_denies.contains(capability))
6398}
6399
6400fn denied_capability<'a>(
6403 opening_manifest: &'a ModuleManifest,
6404 target_manifest: &ModuleManifest,
6405) -> Option<&'a str> {
6406 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
6407 let target_capabilities = target_manifest.capabilities.as_ref()?;
6408 opening_capabilities
6409 .must_never_reach
6410 .iter()
6411 .find(|denied| {
6412 target_capabilities
6413 .provides
6414 .iter()
6415 .any(|provided| provided == *denied)
6416 })
6417 .map(String::as_str)
6418}
6419
6420fn catalog_update_frozen_field_message(
6421 registered: &ModuleManifest,
6422 provides: &[ProviderRole],
6423) -> Option<String> {
6424 let old_has_provides = !registered.provides.is_empty();
6425 let new_has_provides = !provides.is_empty();
6426 if old_has_provides != new_has_provides {
6427 return Some(format!(
6428 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
6429 registered.module_id
6430 ));
6431 }
6432
6433 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
6434 return Some(format!(
6435 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
6436 registered.module_id
6437 ));
6438 }
6439
6440 let registered_concurrency = manifest_concurrency(registered);
6441 let mut candidate = registered.clone();
6442 candidate.provides = provides.to_vec();
6443 let candidate_concurrency = manifest_concurrency(&candidate);
6444 if candidate_concurrency != registered_concurrency {
6445 return Some(format!(
6446 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
6447 registered.module_id, registered_concurrency, candidate_concurrency
6448 ));
6449 }
6450
6451 None
6454}
6455
6456fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
6457 manifest.provides.iter().any(is_routable_role)
6458}
6459
6460fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
6466 manifest
6467 .provides
6468 .iter()
6469 .find_map(|provider| match provider {
6470 ProviderRole::ToolProvider { concurrency, .. }
6471 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
6472 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
6473 })
6474 .unwrap_or(Concurrency::ModuleManaged)
6475}
6476
6477fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
6484 let has_management_surface = manifest
6485 .provides
6486 .iter()
6487 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
6488 if !has_management_surface {
6489 return false;
6490 }
6491 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
6492 return false;
6493 };
6494 let Some(provides) = raw
6495 .get("manifest")
6496 .and_then(|manifest| manifest.get("provides"))
6497 .and_then(serde_json::Value::as_array)
6498 else {
6499 return false;
6500 };
6501 provides.iter().any(|role| {
6506 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
6507 && role.get("concurrency").is_none()
6508 })
6509}
6510
6511fn negotiate_version(peer_version: u8) -> Result<u8, String> {
6512 if peer_version != PROTOCOL_VERSION {
6513 return Err(format!(
6514 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
6515 ));
6516 }
6517 Ok(PROTOCOL_VERSION)
6518}
6519
6520fn pong(frame: &Frame) -> Result<Frame, RouterError> {
6521 Frame::build_with_version(
6522 response_version(frame),
6523 FrameType::Pong,
6524 frame.header.flags,
6525 0,
6526 0,
6527 frame.header.corr,
6528 Vec::new(),
6529 )
6530 .map_err(RouterError::FrameBuild)
6531}
6532
6533pub(crate) fn control_error_frame(
6534 frame: &Frame,
6535 code: &'static str,
6536 message: impl Into<String>,
6537) -> Result<Frame, RouterError> {
6538 control_error_body_frame(
6539 frame,
6540 ErrorBody {
6541 code: code.to_string(),
6542 message: message.into(),
6543 detail: None,
6544 },
6545 )
6546}
6547
6548fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
6549 let body = serde_json::to_vec(&error).map_err(|err| {
6550 RouterError::backend(
6551 0,
6552 frame.header.corr,
6553 format!("failed to encode control ERROR: {err}"),
6554 )
6555 })?;
6556
6557 Frame::build_with_version(
6558 response_version(frame),
6559 FrameType::Error,
6560 control_flags(),
6561 0,
6562 0,
6563 frame.header.corr,
6564 body,
6565 )
6566 .map_err(RouterError::FrameBuild)
6567}
6568
6569fn control_response_body_frame<T: Serialize>(
6570 frame: &Frame,
6571 reply: &T,
6572 label: &'static str,
6573) -> Result<Frame, RouterError> {
6574 let body = serde_json::to_vec(reply).map_err(|err| {
6575 RouterError::backend(
6576 0,
6577 frame.header.corr,
6578 format!("failed to encode {label}: {err}"),
6579 )
6580 })?;
6581
6582 Frame::build_with_version(
6583 response_version(frame),
6584 FrameType::Response,
6585 control_flags(),
6586 0,
6587 0,
6588 frame.header.corr,
6589 body,
6590 )
6591 .map_err(RouterError::FrameBuild)
6592}
6593
6594fn forwarding_error_code(err: &ForwardingError) -> &'static str {
6624 match err {
6625 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
6626 ForwardingError::NoModuleConnection => "target_unavailable",
6627 ForwardingError::ModuleReloading { .. } => "module_reloading",
6628 ForwardingError::ClientRouteChannelExhausted { .. }
6629 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
6630 ForwardingError::StaleModuleEndpoint
6631 | ForwardingError::UnknownReservation { .. }
6632 | ForwardingError::ConnectionClosing { .. }
6633 | ForwardingError::ClientEgressClosed { .. }
6634 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
6635 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
6638 ForwardingError::RelayCorrelationExhausted
6639 | ForwardingError::RouteOpenBuild(_)
6640 | ForwardingError::Poisoned => "forwarding_error",
6641 }
6642}
6643
6644fn response_version(frame: &Frame) -> u8 {
6645 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
6646 frame.header.ver
6647 } else {
6648 PROTOCOL_VERSION
6649 }
6650}
6651
6652fn control_flags() -> Flags {
6653 Flags::new(false, Priority::Passive, false)
6654}
6655
6656fn send_goodbye_target_best_effort(
6661 counters: &DaemonCounters,
6662 target: &GoodbyeTarget,
6663 context: &'static str,
6664) {
6665 let Ok(frame) = Frame::build_with_version(
6666 target.negotiated_ver,
6667 FrameType::Goodbye,
6668 control_flags(),
6669 target.channel,
6670 target.epoch,
6671 0,
6672 Vec::new(),
6673 ) else {
6674 return;
6675 };
6676 crate::forwarding::send_module_route_goodbye(
6677 counters,
6678 &target.sink,
6679 frame,
6680 target.module_id.as_deref(),
6681 context,
6682 );
6683}
6684
6685pub(crate) fn send_route_control_pushes(
6686 forwarding: &ForwardingTable,
6687 routes: Vec<EndpointRoute>,
6688 push: ClientControlPush,
6689) {
6690 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
6691 for route in routes {
6692 let target = route.goodbye_target;
6693 if let Some((existing, channels)) = targets
6694 .iter_mut()
6695 .find(|(existing, _)| existing.connection_id == target.connection_id)
6696 {
6697 debug_assert_eq!(
6698 existing.negotiated_ver, target.negotiated_ver,
6699 "one connection cannot negotiate multiple frame versions"
6700 );
6701 if !channels.contains(&target.channel) {
6702 channels.push(target.channel);
6703 }
6704 continue;
6705 }
6706 let channel = target.channel;
6707 targets.push((target, vec![channel]));
6708 }
6709 for (target, mut channels) in targets {
6710 channels.sort_unstable();
6711 let mut push = push.clone();
6712 match &mut push {
6713 ClientControlPush::RouteClosing {
6714 channels: covered, ..
6715 }
6716 | ClientControlPush::RouteClosed {
6717 channels: covered, ..
6718 } => *covered = channels,
6719 }
6720 let body = match serde_json::to_vec(&push) {
6721 Ok(body) => body,
6722 Err(err) => {
6723 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
6724 continue;
6725 }
6726 };
6727 let frame = match Frame::build_with_version(
6728 target.negotiated_ver,
6729 FrameType::Push,
6730 control_flags(),
6731 0,
6732 0,
6733 0,
6734 body.clone(),
6735 ) {
6736 Ok(frame) => frame,
6737 Err(err) => {
6738 warn!(
6739 route_channel = target.channel,
6740 error = %err,
6741 "failed to build route lifecycle control PUSH frame"
6742 );
6743 continue;
6744 }
6745 };
6746 if let Err(err) = target.sink.try_send(frame) {
6747 if target.close_on_delivery_failure() {
6748 warn!(
6749 target_connection_id = target.connection_id.get(),
6750 route_channel = target.channel,
6751 error = %err,
6752 "route lifecycle control PUSH was not delivered to client; closing target connection"
6753 );
6754 let _ = forwarding.escalate_client_delivery_failure(
6755 target.connection_id,
6756 target.channel,
6757 target.epoch,
6758 CloseReason::new(
6759 "route_lifecycle_push_delivery_failed",
6760 format!(
6761 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
6762 target.channel
6763 ),
6764 ),
6765 crate::forwarding::UndeliveredFrame {
6766 module_id: target.module_id.as_deref(),
6767 sink: &target.sink,
6768 },
6769 );
6770 }
6771 }
6772 }
6773}
6774
6775#[cfg(test)]
6776mod tests {
6777 #[cfg(unix)]
6778 #[tokio::test]
6779 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
6780 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
6781 let path = dir.join("subc.jsonc");
6782 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
6783 "program":"/bin/sleep","args":["60"],"protocol":"none",
6784 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
6785 }}}).to_string()).unwrap();
6786 let mut configured = crate::daemon_config::load(&path)
6787 .unwrap()
6788 .unwrap()
6789 .modules
6790 .pop()
6791 .unwrap();
6792 let registry = std::sync::Arc::new(crate::Registry::default());
6793 let handle = crate::SupervisorHandle::new();
6794 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
6795 .with_handle(handle.clone());
6796 let module = supervisor
6797 .supervise_configured_with_health(
6798 configured.module_spec(),
6799 true,
6800 configured.health.clone(),
6801 None,
6802 configured.restart,
6803 )
6804 .unwrap();
6805 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
6806 let before = module.status().unwrap().pid;
6807 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
6808 configured.health.cadence = std::time::Duration::from_secs(3600);
6809 let health_only = handler
6810 .reconcile_supervised_modules(&supervisor, vec![configured.clone()], false)
6811 .await
6812 .unwrap();
6813 assert!(
6814 health_only.changed_pending_reload.is_empty(),
6815 "health policy is already applied live"
6816 );
6817 assert_eq!(module.status().unwrap().pid, before);
6818 assert_eq!(
6819 module.configuration().unwrap().1.http,
6820 configured.health.http
6821 );
6822 configured.args = vec!["61".into()];
6823 let launch = handler
6824 .reconcile_supervised_modules(&supervisor, vec![configured], false)
6825 .await
6826 .unwrap();
6827 assert_eq!(launch.changed_pending_reload, ["stock"]);
6828 assert_eq!(
6829 module.status().unwrap().pid,
6830 before,
6831 "a launch edit is stored until reload"
6832 );
6833 module.drain().await.unwrap();
6834 }
6835 use cortexkit_test_support::ScratchDir;
6836 use std::{
6837 collections::BTreeMap,
6838 fmt,
6839 path::PathBuf,
6840 sync::{Arc, Mutex},
6841 time::Duration,
6842 };
6843
6844 use serde_json::{json, Value};
6845 use subc_protocol::{
6846 manifest::{
6847 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
6848 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
6849 },
6850 session::HealthStatus,
6851 FrameType,
6852 };
6853
6854 use super::*;
6855 use crate::{
6856 forwarding::{DataRoute, DataRouteState},
6857 registry::ChannelState,
6858 router::FrameSink,
6859 stderr_tail::DEFAULT_MAX_LINE_BYTES,
6860 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
6861 RouteCtx, Router,
6862 };
6863 use tokio::{
6864 sync::mpsc,
6865 time::{sleep, Instant},
6866 };
6867 use tracing::{
6868 field::{Field, Visit},
6869 Event, Subscriber,
6870 };
6871 use tracing_subscriber::{layer::Context, prelude::*, Layer};
6872
6873 fn fake_aft_stub_path() -> PathBuf {
6892 let mut path = std::env::current_exe().expect("current_exe available in tests");
6893 path.pop(); path.pop(); path.push(if cfg!(windows) {
6896 "fake-aft-stub.exe"
6897 } else {
6898 "fake-aft-stub"
6899 });
6900 assert!(
6901 path.exists(),
6902 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
6903 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
6904 path.display()
6905 );
6906 path
6907 }
6908
6909 fn client_retries(code: &str) -> bool {
6914 subc_protocol::error_codes::is_retryable_route_open(code)
6915 }
6916
6917 #[test]
6928 fn retryability_of_forwarding_codes_matches_the_failure() {
6929 let transient = [
6932 ForwardingError::NoModuleConnection,
6933 ForwardingError::ModuleReloading {
6934 module_id: "m".into(),
6935 },
6936 ForwardingError::StaleModuleEndpoint,
6937 ForwardingError::UnknownReservation {
6938 client_channel: 1,
6939 module_channel: 1,
6940 },
6941 ForwardingError::ConnectionClosing {
6942 connection_id: ConnectionId::new(1),
6943 },
6944 ForwardingError::ClientEgressClosed {
6945 connection_id: ConnectionId::new(1),
6946 },
6947 ForwardingError::ModuleEgressUnavailable {
6948 connection_id: ConnectionId::new(1),
6949 },
6950 ];
6951 for err in transient {
6952 let code = forwarding_error_code(&err);
6953 assert!(
6954 client_retries(code),
6955 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
6956 );
6957 }
6958
6959 let permanent = [
6964 ForwardingError::ConnectionRoleConflict {
6965 connection_id: ConnectionId::new(1),
6966 },
6967 ForwardingError::ClientRouteChannelExhausted {
6968 connection_id: ConnectionId::new(1),
6969 },
6970 ForwardingError::ModuleRouteChannelExhausted {
6971 endpoint: ModuleEndpointId {
6972 connection_id: ConnectionId::new(1),
6973 generation: 1,
6974 },
6975 },
6976 ForwardingError::RelayCorrelationExhausted,
6977 ForwardingError::RouteOpenBuild("x".into()),
6978 ForwardingError::Poisoned,
6979 ];
6980 for err in permanent {
6981 let code = forwarding_error_code(&err);
6982 assert!(
6983 !client_retries(code),
6984 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
6985 );
6986 }
6987 }
6988
6989 #[tokio::test]
7017 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
7018 let handler = ControlHandler::default();
7019 let frame =
7020 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
7021
7022 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
7026 assert_eq!(
7027 stamped,
7028 Principal::Direct,
7029 "a caller that proved nothing must not be stamped as a supervised module"
7030 );
7031
7032 let forged = handler
7037 .route_open_principal(
7038 &frame,
7039 Some(ConsumerIdentity {
7040 module_id: "aft".to_string(),
7041 launch_nonce: "not-a-real-nonce".to_string(),
7042 }),
7043 )
7044 .unwrap();
7045 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
7046 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
7047 }
7048
7049 #[test]
7066 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
7067 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
7068 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
7069 let ClientControlRequest::RouteOpen {
7070 consumer_identity, ..
7071 } = parsed
7072 else {
7073 panic!("route.open body must parse as RouteOpen");
7074 };
7075 assert_eq!(
7076 consumer_identity,
7077 Some(ConsumerIdentity {
7078 module_id: "aft".to_string(),
7079 launch_nonce: "n".to_string(),
7080 }),
7081 "the wire field name must reach the value route_open_principal reads"
7082 );
7083 }
7084
7085 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
7086 ModuleManifest::builder(module_id, "0.1.0")
7087 .protocol_ver(protocol_ver)
7088 .provides(vec![ProviderRole::ToolProvider {
7089 tools: vec![Tool {
7090 name: "read".to_string(),
7091 description: None,
7092 execution_mode: ExecutionMode::Pure,
7093 schema: json!({"type": "object"}),
7094 }],
7095 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
7096 concurrency: Concurrency::ModuleManaged,
7097 emits_push: true,
7098 sub_supervises: true,
7099 }])
7100 .build()
7101 }
7102
7103 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
7104 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
7105 }
7106
7107 fn hello_frame_with_control_ops(
7108 module_id: &str,
7109 protocol_ver: u8,
7110 corr: u64,
7111 control_ops: Option<Vec<String>>,
7112 ) -> Frame {
7113 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
7114 }
7115
7116 fn hello_frame_with_nonce(
7117 module_id: &str,
7118 protocol_ver: u8,
7119 corr: u64,
7120 launch_nonce: Option<&str>,
7121 ) -> Frame {
7122 hello_frame_full(
7123 module_id,
7124 protocol_ver,
7125 corr,
7126 None,
7127 launch_nonce.map(ToOwned::to_owned),
7128 )
7129 }
7130
7131 fn hello_frame_full(
7132 module_id: &str,
7133 protocol_ver: u8,
7134 corr: u64,
7135 control_ops: Option<Vec<String>>,
7136 launch_nonce: Option<String>,
7137 ) -> Frame {
7138 let body = serde_json::to_vec(&ModuleHelloBody {
7139 manifest: manifest(module_id, protocol_ver),
7140 protocol_ver,
7141 control_ops,
7142 launch_nonce,
7143 })
7144 .unwrap();
7145 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7146 }
7147
7148 fn non_routable_hello_frame_with_control_ops(
7149 module_id: &str,
7150 corr: u64,
7151 control_ops: Option<Vec<String>>,
7152 ) -> Frame {
7153 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
7154 manifest.provides.clear();
7155 let body = serde_json::to_vec(&ModuleHelloBody {
7156 manifest,
7157 protocol_ver: PROTOCOL_VERSION,
7158 control_ops,
7159 launch_nonce: None,
7160 })
7161 .unwrap();
7162 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7163 }
7164
7165 fn capability_grammar_hello_frame(
7166 capabilities: Value,
7167 runtime_computed: Option<Value>,
7168 corr: u64,
7169 ) -> Frame {
7170 let mut body = serde_json::to_value(ModuleHelloBody {
7171 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
7172 protocol_ver: PROTOCOL_VERSION,
7173 control_ops: None,
7174 launch_nonce: None,
7175 })
7176 .expect("HELLO body serializes");
7177 body["manifest"]["capabilities"] = capabilities;
7178 if let Some(runtime_computed) = runtime_computed {
7179 body["runtime_computed"] = runtime_computed;
7180 }
7181 Frame::build(
7182 FrameType::Hello,
7183 control_flags(),
7184 0,
7185 0,
7186 corr,
7187 serde_json::to_vec(&body).expect("HELLO body reserializes"),
7188 )
7189 .expect("HELLO frame builds")
7190 }
7191
7192 fn channel_request(channel: u16, corr: u64) -> Frame {
7193 Frame::build(
7194 FrameType::Request,
7195 Flags::new(true, Priority::Interactive, false),
7196 channel,
7197 0,
7198 corr,
7199 b"opaque".to_vec(),
7200 )
7201 .unwrap()
7202 }
7203
7204 fn route_ctx(
7205 connection_id: ConnectionId,
7206 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
7207 let (tx, rx) = mpsc::channel(8);
7208 (
7209 RouteCtx {
7210 connection_id,
7211 egress: FrameSink::new(tx),
7212 },
7213 rx,
7214 )
7215 }
7216
7217 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
7218 serde_json::from_slice(&frame.body).unwrap()
7219 }
7220
7221 async fn hello_via_sink(
7226 handler: &ControlHandler,
7227 ctx: &RouteCtx,
7228 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7229 hello: Frame,
7230 ) -> Frame {
7231 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
7232 assert!(
7233 replies.is_empty(),
7234 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
7235 );
7236 let ack = rx
7237 .try_recv()
7238 .expect("HELLO_ACK is queued on the module sink")
7239 .frame;
7240 assert_eq!(ack.header.ty, FrameType::HelloAck);
7241 ack
7242 }
7243
7244 fn parse_error(frame: &Frame) -> Value {
7245 serde_json::from_slice(&frame.body).unwrap()
7246 }
7247
7248 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
7249 serde_json::from_slice(&frame.body).unwrap()
7250 }
7251
7252 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
7253 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
7254 route_channel,
7255 route_epoch: 0,
7256 kind,
7257 })
7258 .unwrap();
7259 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7260 }
7261
7262 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
7263 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
7264 module_id: module_id.to_string(),
7265 })
7266 .unwrap();
7267 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7268 }
7269
7270 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
7271 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
7272 }
7273
7274 fn route_open_frame_with_consumer_capabilities(
7275 corr: u64,
7276 module_id: &str,
7277 project_root: ScratchDir,
7278 consumer_capabilities: Option<Vec<String>>,
7279 ) -> Frame {
7280 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7281 target: RouteTarget::ToolProvider {
7282 module_id: module_id.to_string(),
7283 },
7284 identity: BindIdentity::new(
7285 project_root.path().to_path_buf(),
7286 "unit".to_string(),
7287 "session".to_string(),
7288 ),
7289 consumer_identity: None,
7290 consumer_capabilities,
7291 role_versions: None,
7292 admission_facts: None,
7293 scope: None,
7294 })
7295 .unwrap();
7296 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7297 }
7298
7299 fn route_open_frame_with_role_versions(
7300 corr: u64,
7301 module_id: &str,
7302 project_root: ScratchDir,
7303 role_versions: Option<BTreeMap<String, String>>,
7304 ) -> Frame {
7305 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7306 target: RouteTarget::ToolProvider {
7307 module_id: module_id.to_string(),
7308 },
7309 identity: BindIdentity::new(
7310 project_root.path().to_path_buf(),
7311 "unit".to_string(),
7312 format!("session-{corr}"),
7313 ),
7314 consumer_identity: None,
7315 consumer_capabilities: None,
7316 role_versions,
7317 admission_facts: None,
7318 scope: None,
7319 })
7320 .unwrap();
7321 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7322 }
7323
7324 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
7325 entries
7326 .iter()
7327 .map(|(role, version)| (role.to_string(), version.to_string()))
7328 .collect()
7329 }
7330
7331 fn route_open_frame_with_admission_facts(
7332 corr: u64,
7333 module_id: &str,
7334 project_root: ScratchDir,
7335 consumer_identity: Option<subc_control::ConsumerIdentity>,
7336 facts: Option<Value>,
7337 ) -> Frame {
7338 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7339 target: RouteTarget::ToolProvider {
7340 module_id: module_id.to_string(),
7341 },
7342 identity: BindIdentity::new(
7343 project_root.path().to_path_buf(),
7344 "unit".to_string(),
7345 format!("session-{corr}"),
7346 ),
7347 consumer_identity,
7348 consumer_capabilities: None,
7349 role_versions: None,
7350 admission_facts: facts,
7351 scope: None,
7352 })
7353 .unwrap();
7354 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7355 }
7356
7357 #[derive(Clone, Default)]
7358 struct EventCapture {
7359 events: Arc<Mutex<Vec<CapturedEvent>>>,
7360 }
7361
7362 #[derive(Clone, Debug)]
7363 struct CapturedEvent {
7364 target: String,
7365 level: tracing::Level,
7366 fields: BTreeMap<String, String>,
7367 }
7368
7369 impl EventCapture {
7370 fn events(&self) -> Vec<CapturedEvent> {
7371 self.events.lock().unwrap().clone()
7372 }
7373 }
7374
7375 impl<S> Layer<S> for EventCapture
7376 where
7377 S: Subscriber,
7378 {
7379 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
7380 let mut visitor = EventFieldVisitor::default();
7381 event.record(&mut visitor);
7382 self.events.lock().unwrap().push(CapturedEvent {
7383 target: event.metadata().target().to_string(),
7384 level: *event.metadata().level(),
7385 fields: visitor.fields,
7386 });
7387 }
7388 }
7389
7390 #[derive(Default)]
7391 struct EventFieldVisitor {
7392 fields: BTreeMap<String, String>,
7393 }
7394
7395 impl Visit for EventFieldVisitor {
7396 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
7397 self.fields
7398 .insert(field.name().to_string(), format!("{value:?}"));
7399 }
7400 }
7401
7402 fn health_response(corr: u64, status: HealthStatus) -> Frame {
7403 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
7404 status,
7405 detail: Some("warming".to_string()),
7406 metrics: Some(json!({"queue_depth": 3})),
7407 })
7408 .unwrap();
7409 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7410 }
7411
7412 fn route_bind_ack(corr: u64) -> Frame {
7413 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
7414 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7415 }
7416
7417 fn unique_project_root(label: &str) -> ScratchDir {
7418 ScratchDir::new(label)
7419 }
7420
7421 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
7422 match parse_route_poll(frame) {
7423 ClientControlResponse::RoutePoll {
7424 status: None,
7425 live: Some(live),
7426 ..
7427 } => assert_eq!(live, expected_live),
7428 other => panic!("unexpected route.poll response: {other:?}"),
7429 }
7430 }
7431
7432 fn bind_liveness_route(
7433 registry: &Registry,
7434 forwarding: &ForwardingTable,
7435 module_id: &str,
7436 ) -> (RouteCtx, u16, u32) {
7437 let module_connection = ConnectionId::new(101);
7438 let client_connection = ConnectionId::new(202);
7439 let registration = registry
7440 .register_with_control_ops(
7441 manifest(module_id, PROTOCOL_VERSION),
7442 PROTOCOL_VERSION,
7443 module_connection,
7444 module_baseline_control_ops(),
7445 )
7446 .unwrap();
7447 let (module_tx, _module_rx) = mpsc::channel(8);
7448 let endpoint = forwarding
7449 .register_module_connection(
7450 module_connection,
7451 module_id.to_string(),
7452 PROTOCOL_VERSION,
7453 manifest_concurrency(®istration.manifest),
7454 FrameSink::new(module_tx),
7455 )
7456 .unwrap();
7457 let (client_ctx, _client_rx) = route_ctx(client_connection);
7458 let pending = forwarding
7459 .begin_route_bind_relay_for_test(
7460 client_connection,
7461 client_ctx.egress.clone(),
7462 1,
7463 module_id,
7464 )
7465 .unwrap();
7466 assert_eq!(pending.endpoint, endpoint);
7467 let route_channel = pending.client_channel;
7468 let route_epoch = pending.client_epoch;
7469 forwarding
7470 .complete_pending_relay(
7471 module_connection,
7472 pending.corr,
7473 RouteBindRelayOutcome::Accepted,
7474 )
7475 .unwrap();
7476 (client_ctx, route_channel, route_epoch)
7477 }
7478
7479 struct FakeProcessLiveness {
7480 live: Option<bool>,
7481 }
7482
7483 impl ModuleProcessLiveness for FakeProcessLiveness {
7484 fn process_live(&self, _module_id: &str) -> Option<bool> {
7485 self.live
7486 }
7487 }
7488
7489 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7490 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
7491 {
7492 let registry = Arc::new(Registry::default());
7493 let supervisor_handle = SupervisorHandle::new();
7494 let supervisor = Supervisor::new_for_test(
7495 Arc::clone(®istry),
7496 RestartPolicy::new(1, Duration::from_millis(10)),
7497 )
7498 .with_handle(supervisor_handle.clone());
7499 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
7500 let module = supervisor
7501 .spawn(ModuleSpec {
7502 module_id: "stderr-tail-wire".to_string(),
7503 program: fake_aft_stub_path(),
7504 args: Vec::new(),
7505 env: vec![
7506 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
7507 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
7508 ],
7509 reserved: false,
7510 reserved_prefixes: Vec::new(),
7511 protocol: ModuleProtocol::Subc,
7512 overlap: Default::default(),
7513 })
7514 .unwrap();
7515
7516 let deadline = Instant::now() + Duration::from_secs(5);
7517 loop {
7518 let tail = module.stderr_tail(None, None);
7519 if tail
7520 .entries
7521 .iter()
7522 .any(|entry| matches!(entry, TailEntry::ProcessStart))
7523 && tail.entries.iter().any(|entry| {
7524 matches!(
7525 entry,
7526 TailEntry::Line {
7527 truncated: true,
7528 ..
7529 }
7530 )
7531 })
7532 {
7533 break;
7534 }
7535 assert!(
7536 Instant::now() < deadline,
7537 "module did not produce a truncated line and restart boundary: {tail:?}"
7538 );
7539 sleep(Duration::from_millis(10)).await;
7540 }
7541
7542 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
7543 let request = ClientControlRequest::SupervisorStderrTail {
7544 module_id: "stderr-tail-wire".to_string(),
7545 max_lines: None,
7546 max_bytes: None,
7547 };
7548 let frame = Frame::build(
7549 FrameType::Request,
7550 control_flags(),
7551 0,
7552 0,
7553 1,
7554 serde_json::to_vec(&request).unwrap(),
7555 )
7556 .unwrap();
7557 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7558 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7559 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
7560 serde_json::from_slice(&responses[0].body).unwrap()
7561 else {
7562 panic!("expected supervisor.stderr_tail response");
7563 };
7564
7565 assert!(
7566 tail.entries
7567 .iter()
7568 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
7569 "the control response lost the restart boundary"
7570 );
7571 let Some(StderrTailEntry::Line {
7572 text,
7573 truncated,
7574 at_ms,
7575 }) = tail.entries.iter().find(|entry| {
7576 matches!(
7577 entry,
7578 StderrTailEntry::Line {
7579 truncated: true,
7580 ..
7581 }
7582 )
7583 })
7584 else {
7585 panic!("the control response lost the truncated line");
7586 };
7587 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
7588 assert!(*truncated);
7589 assert!(
7590 at_ms.is_some(),
7591 "the control response lost the line's capture time"
7592 );
7593 }
7594
7595 #[tokio::test(flavor = "current_thread")]
7600 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
7601 let dir = ScratchDir::new("terminals-off-worker");
7602 let journal_path = dir.join("terminals.jsonl");
7603 let registry = Arc::new(Registry::default());
7604 let supervisor_handle = SupervisorHandle::new();
7605 let supervisor =
7606 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7607 .with_handle(supervisor_handle.clone())
7608 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
7609 let module = supervisor
7610 .spawn(ModuleSpec {
7611 module_id: "terminal-off-worker".to_string(),
7612 program: fake_aft_stub_path(),
7613 args: Vec::new(),
7614 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7615 reserved: false,
7616 reserved_prefixes: Vec::new(),
7617 protocol: ModuleProtocol::Subc,
7618 overlap: Default::default(),
7619 })
7620 .unwrap();
7621 let deadline = Instant::now() + Duration::from_secs(5);
7622 while module.terminal_history().entries.len() != 2 {
7623 assert!(Instant::now() < deadline, "module did not record two exits");
7624 sleep(Duration::from_millis(10)).await;
7625 }
7626
7627 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
7628 let handler =
7629 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
7630 let frame = Frame::build(
7631 FrameType::Request,
7632 control_flags(),
7633 0,
7634 0,
7635 1,
7636 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
7637 module_id: "terminal-off-worker".to_string(),
7638 })
7639 .unwrap(),
7640 )
7641 .unwrap();
7642 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7643 let spawned_at = std::time::Instant::now();
7644 let read = tokio::spawn({
7645 let handler = Arc::clone(&handler);
7646 async move { handler.handle_control_frame(&ctx, frame).await }
7647 });
7648 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
7651 .await
7652 .unwrap()
7653 .expect("the history read reached its pause");
7654 let elapsed = spawned_at.elapsed();
7655 assert!(
7656 elapsed < Duration::from_secs(2) && !read.is_finished(),
7657 "this task could not run while the history read was paused \
7658 (resumed after {elapsed:?}, read finished: {})",
7659 read.is_finished()
7660 );
7661
7662 drop(release);
7663 let responses = read.await.unwrap().unwrap();
7664 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7665 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
7666 panic!("expected supervisor.terminals response");
7667 };
7668 assert_eq!(terminals.entries.len(), 2);
7669 assert_eq!(terminals.journal_skipped_lines, 0);
7670 assert_eq!(terminals.journal_read_errors, 0);
7671 }
7672
7673 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7674 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
7675 let registry = Arc::new(Registry::default());
7676 let supervisor_handle = SupervisorHandle::new();
7677 let supervisor =
7678 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7679 .with_handle(supervisor_handle.clone());
7680 let module = supervisor
7681 .spawn(ModuleSpec {
7682 module_id: "terminal-golden".to_string(),
7683 program: fake_aft_stub_path(),
7684 args: Vec::new(),
7685 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7686 reserved: false,
7687 reserved_prefixes: Vec::new(),
7688 protocol: ModuleProtocol::Subc,
7689 overlap: Default::default(),
7690 })
7691 .unwrap();
7692
7693 let deadline = Instant::now() + Duration::from_secs(5);
7694 while module.terminal_history().entries.len() != 2 {
7695 assert!(
7696 Instant::now() < deadline,
7697 "module did not retain two terminal exits: {:?}",
7698 module.terminal_history()
7699 );
7700 sleep(Duration::from_millis(10)).await;
7701 }
7702
7703 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
7704 let request = ClientControlRequest::SupervisorTerminals {
7705 module_id: "terminal-golden".to_string(),
7706 };
7707 let frame = Frame::build(
7708 FrameType::Request,
7709 control_flags(),
7710 0,
7711 0,
7712 1,
7713 serde_json::to_vec(&request).unwrap(),
7714 )
7715 .unwrap();
7716 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7717 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7718 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7719 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
7720 panic!("expected supervisor.terminals response");
7721 };
7722 assert_eq!(terminals.entries.len(), 2);
7723 assert_eq!(terminals.dropped, 0);
7724
7725 let mut rendered = serde_json::to_value(response).unwrap();
7726 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
7729 for (index, entry) in rendered["entries"]
7730 .as_array_mut()
7731 .expect("terminal response entries array")
7732 .iter_mut()
7733 .enumerate()
7734 {
7735 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
7736 }
7737
7738 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7739 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
7740 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
7741 if std::env::var_os("UPDATE_GOLDEN").is_some() {
7742 std::fs::write(&golden_path, &serialized).unwrap();
7743 }
7744 let expected: Value =
7745 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
7746 assert_eq!(rendered, expected);
7747 }
7748
7749 #[test]
7750 fn hello_registers_manifest_and_returns_ack() {
7751 let registry = Arc::new(Registry::default());
7752 let handler = ControlHandler::new(Arc::clone(®istry));
7753 let conn = ConnectionId::new(1);
7754
7755 let responses = handler
7756 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
7757 .unwrap();
7758
7759 assert_eq!(responses.len(), 1);
7760 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7761 assert_eq!(responses[0].header.channel, 0);
7762 assert_eq!(responses[0].header.corr, 7);
7763 let ack = parse_ack(&responses[0]);
7764 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
7765 assert!(ack
7766 .subc_capabilities
7767 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
7768 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
7769 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
7770 assert!(ack
7771 .subc_ops
7772 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
7773 assert!(ack
7774 .subc_ops
7775 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
7776
7777 let registration = registry.get_module("aft").unwrap().unwrap();
7778 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
7779 assert_eq!(registration.state, ChannelState::Active);
7780 assert_eq!(registration.connection_id, conn);
7781 assert_eq!(registration.control_ops, module_baseline_control_ops());
7782 }
7783
7784 #[test]
7785 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
7786 let invalid_identifiers = [
7787 ("case_change", "credentials-Provider/v1"),
7788 ("leading_zero", "credentials-provider/v01"),
7789 ("trailing_hyphen", "credentials-provider-/v1"),
7790 ("consecutive_hyphens", "credentials--provider/v1"),
7791 ("uppercase", "Credentials-provider/v1"),
7792 ("missing_v", "credentials-provider/1"),
7793 ("whitespace", "credentials provider/v1"),
7794 ("zero_version", "credentials-provider/v0"),
7795 ("out_of_range_version", "credentials-provider/v4294967296"),
7796 (
7797 "overlength_name",
7798 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
7799 ),
7800 ];
7801 let mut cases = invalid_identifiers
7802 .into_iter()
7803 .map(|(name, identifier)| {
7804 (
7805 format!("identifier_{name}"),
7806 "capabilities.provides[0]".to_string(),
7807 identifier.to_string(),
7808 json!({ "provides": [identifier] }),
7809 None,
7810 )
7811 })
7812 .collect::<Vec<_>>();
7813 cases.extend([
7814 (
7815 "unknown_need".to_string(),
7816 "capabilities.requires[0].need".to_string(),
7817 "deferred".to_string(),
7818 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
7819 None,
7820 ),
7821 (
7822 "duplicate_provides".to_string(),
7823 "capabilities.provides[1]".to_string(),
7824 "credentials-provider/v1".to_string(),
7825 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
7826 None,
7827 ),
7828 (
7829 "duplicate_must_never_reach".to_string(),
7830 "capabilities.must_never_reach[1]".to_string(),
7831 "credentials-provider/v1".to_string(),
7832 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
7833 None,
7834 ),
7835 (
7836 "duplicate_requires_same_need".to_string(),
7837 "capabilities.requires[1]".to_string(),
7838 "credentials-provider/v1".to_string(),
7839 json!({ "requires": [
7840 { "capability": "credentials-provider/v1", "need": "required" },
7841 { "capability": "credentials-provider/v1", "need": "required" }
7842 ] }),
7843 None,
7844 ),
7845 (
7846 "duplicate_requires_conflicting_need".to_string(),
7847 "capabilities.requires[1]".to_string(),
7848 "credentials-provider/v1".to_string(),
7849 json!({ "requires": [
7850 { "capability": "credentials-provider/v1", "need": "required" },
7851 { "capability": "credentials-provider/v1", "need": "optional" }
7852 ] }),
7853 None,
7854 ),
7855 (
7856 "capabilities_root_pointer".to_string(),
7857 "runtime_computed[0]".to_string(),
7858 "/capabilities".to_string(),
7859 json!({}),
7860 Some(json!(["/capabilities"])),
7861 ),
7862 (
7863 "capabilities_descendant_pointer".to_string(),
7864 "runtime_computed[0]".to_string(),
7865 "/capabilities/provides".to_string(),
7866 json!({}),
7867 Some(json!(["/capabilities/provides"])),
7868 ),
7869 (
7870 "malformed_pointer_without_leading_slash".to_string(),
7871 "runtime_computed[0]".to_string(),
7872 "capabilities".to_string(),
7873 json!({}),
7874 Some(json!(["capabilities"])),
7875 ),
7876 (
7877 "malformed_pointer_escape".to_string(),
7878 "runtime_computed[0]".to_string(),
7879 "/roles/~2/tools".to_string(),
7880 json!({}),
7881 Some(json!(["/roles/~2/tools"])),
7882 ),
7883 (
7884 "unknown_capabilities_field".to_string(),
7885 "capabilities.future".to_string(),
7886 "<array>".to_string(),
7887 json!({ "future": [] }),
7888 None,
7889 ),
7890 ]);
7891
7892 for (index, (name, field, value, capabilities, runtime_computed)) in
7893 cases.into_iter().enumerate()
7894 {
7895 let registry = Arc::new(Registry::default());
7896 let handler = ControlHandler::new(Arc::clone(®istry));
7897 let response = handler
7898 .handle_control(
7899 ConnectionId::new((index + 1) as u64),
7900 capability_grammar_hello_frame(
7901 capabilities,
7902 runtime_computed,
7903 index as u64 + 1,
7904 ),
7905 )
7906 .expect("invalid HELLO returns a refusal");
7907
7908 assert_eq!(response.len(), 1, "{name} must emit one refusal");
7909 let error = parse_error(&response[0]);
7910 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
7911 let message = error["message"]
7912 .as_str()
7913 .expect("error message is a string");
7914 assert!(
7915 message.contains(&field),
7916 "{name}: field missing from {message}"
7917 );
7918 assert!(
7919 message.contains(&value),
7920 "{name}: value missing from {message}"
7921 );
7922 assert_eq!(
7923 registry
7924 .active_registration_count()
7925 .expect("registry reads"),
7926 0,
7927 "{name}: refused HELLO must not create a catalog entry"
7928 );
7929 }
7930 }
7931
7932 #[test]
7933 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
7934 let registry = Arc::new(Registry::default());
7935 let handler = ControlHandler::new(Arc::clone(®istry));
7936 let capabilities = json!({
7937 "provides": ["credentials-provider/v1"],
7938 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
7939 "must_never_reach": ["federation-transport/v1"]
7940 });
7941 let response = handler
7942 .handle_control(
7943 ConnectionId::new(99),
7944 capability_grammar_hello_frame(
7945 capabilities.clone(),
7946 Some(json!(["/roles/0/tools"])),
7947 99,
7948 ),
7949 )
7950 .expect("valid HELLO registers");
7951 assert_eq!(response[0].header.ty, FrameType::HelloAck);
7952
7953 let request = Frame::build(
7954 FrameType::Request,
7955 control_flags(),
7956 0,
7957 0,
7958 100,
7959 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7960 .expect("catalog request serializes"),
7961 )
7962 .expect("catalog request frame builds");
7963 let response = handler
7964 .handle_catalog_list(request, None)
7965 .expect("catalog list succeeds");
7966 let ClientControlResponse::CatalogList { modules, .. } =
7967 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
7968 else {
7969 panic!("catalog request must return catalog.list");
7970 };
7971 assert_eq!(modules.len(), 1);
7972 assert_eq!(
7973 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
7974 capabilities
7975 );
7976 }
7977
7978 #[test]
7979 fn catalog_list_mirrors_management_operation_description() {
7980 let registry = Arc::new(Registry::default());
7981 let handler = ControlHandler::new(Arc::clone(®istry));
7982 let description = "List managed records and return their identifiers and metadata.";
7983 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
7984 manifest.provides = vec![ProviderRole::ManagementSurface {
7985 operations: vec![ManagementOperation {
7986 name: "records.list".to_string(),
7987 kind: ManagementOperationKind::Query,
7988 description: Some(description.to_string()),
7989 }],
7990 config_schema: json!({"type": "object"}),
7991 observability: vec![ObservabilitySurface {
7992 name: "records.stats".to_string(),
7993 kind: ObservabilityKind::Snapshot,
7994 }],
7995 identity_scope: vec![IdentityScope::Project],
7996 concurrency: Concurrency::ModuleManaged,
7997 }];
7998 registry
7999 .register_with_control_ops(
8000 manifest,
8001 PROTOCOL_VERSION,
8002 ConnectionId::new(99),
8003 Vec::new(),
8004 )
8005 .expect("described management manifest registers");
8006
8007 let request = Frame::build(
8008 FrameType::Request,
8009 control_flags(),
8010 0,
8011 0,
8012 100,
8013 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8014 .expect("catalog request serializes"),
8015 )
8016 .expect("catalog request frame builds");
8017 let response = handler
8018 .handle_catalog_list(request, None)
8019 .expect("catalog list succeeds");
8020 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8021 assert_eq!(
8022 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
8023 "catalog.list must preserve the declared operation description verbatim"
8024 );
8025 }
8026
8027 #[test]
8028 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
8029 let registry = Arc::new(Registry::default());
8030 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8031 [("vault".to_string(), true), ("squatter".to_string(), true)],
8032 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8033 );
8034 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
8035 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8036 provides: vec!["credentials-provider/v1".to_string()],
8037 requires: Vec::new(),
8038 must_never_reach: Vec::new(),
8039 });
8040 let frame = Frame::build(
8041 FrameType::Hello,
8042 control_flags(),
8043 0,
8044 0,
8045 77,
8046 serde_json::to_vec(&ModuleHelloBody {
8047 manifest: squatter,
8048 protocol_ver: PROTOCOL_VERSION,
8049 control_ops: None,
8050 launch_nonce: None,
8051 })
8052 .expect("HELLO serializes"),
8053 )
8054 .expect("HELLO frame builds");
8055 let response = handler
8056 .handle_control(ConnectionId::new(77), frame)
8057 .expect("reserved claim receives a typed refusal");
8058 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8059 assert_eq!(
8060 registry
8061 .active_registration_count()
8062 .expect("registry reads"),
8063 0,
8064 "a reserved capability refusal must not leave a catalog entry"
8065 );
8066 }
8067
8068 #[test]
8069 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
8070 for settlement in ["timeout", "inconclusive", "drop"] {
8071 let breakers = RouteBindBreakers::default();
8072 let RouteBindAdmission::Admitted {
8073 guard: mut old,
8074 probe: false,
8075 } = breakers.admit("prov")
8076 else {
8077 panic!("ordinary relay admitted")
8078 };
8079 let RouteBindAdmission::Admitted {
8080 guard: mut opener, ..
8081 } = breakers.admit("prov")
8082 else {
8083 panic!("second relay admitted")
8084 };
8085 assert!(
8086 !opener
8087 .record_timeout(1, Duration::ZERO)
8088 .unwrap()
8089 .reopened_after_probe
8090 );
8091 let RouteBindAdmission::Admitted {
8092 guard: mut probe,
8093 probe: true,
8094 } = breakers.admit("prov")
8095 else {
8096 panic!("one cooldown probe admitted")
8097 };
8098 match settlement {
8099 "timeout" => assert!(
8100 !old.record_timeout(1, Duration::ZERO)
8101 .unwrap()
8102 .reopened_after_probe
8103 ),
8104 "inconclusive" => old.record_inconclusive(),
8105 "drop" => drop(old),
8106 _ => unreachable!(),
8107 }
8108 assert!(
8109 matches!(
8110 breakers.admit("prov"),
8111 RouteBindAdmission::Refused {
8112 probe_in_flight: true,
8113 ..
8114 }
8115 ),
8116 "{settlement} of a pre-open relay cannot release the real probe"
8117 );
8118 assert!(
8119 probe
8120 .record_timeout(1, Duration::ZERO)
8121 .unwrap()
8122 .reopened_after_probe
8123 );
8124 assert!(matches!(
8125 breakers.admit("prov"),
8126 RouteBindAdmission::Admitted { probe: true, .. }
8127 ));
8128 }
8129 let breakers = RouteBindBreakers::default();
8130 let admit = || match breakers.admit("prov") {
8131 RouteBindAdmission::Admitted { guard, .. } => guard,
8132 _ => panic!("relay admitted"),
8133 };
8134 admit().record_timeout(1, Duration::ZERO);
8135 let mut old_probe = admit();
8136 breakers.reset_for_new_module_connection("prov");
8137 admit().record_timeout(1, Duration::ZERO);
8138 let _new_probe = admit();
8139 old_probe.record_inconclusive();
8140 assert!(matches!(
8141 breakers.admit("prov"),
8142 RouteBindAdmission::Refused {
8143 probe_in_flight: true,
8144 ..
8145 }
8146 ));
8147 }
8148
8149 #[tokio::test]
8150 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
8151 for candidate in [false, true] {
8152 let registry = Arc::new(Registry::default());
8153 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8154 [("vault".to_string(), true), ("squatter".to_string(), true)],
8155 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8156 );
8157 let conn = ConnectionId::new(77);
8158 let (ctx, mut rx) = route_ctx(conn);
8159 let initial = capability_manifest("squatter", &[], &[]);
8160 if candidate {
8161 registry
8162 .register_candidate_with_control_ops(
8163 initial.clone(),
8164 PROTOCOL_VERSION,
8165 conn,
8166 module_baseline_control_ops(),
8167 )
8168 .unwrap();
8169 handler
8170 .forwarding
8171 .register_candidate_module_connection(
8172 conn,
8173 "squatter".to_string(),
8174 PROTOCOL_VERSION,
8175 manifest_concurrency(&initial),
8176 ctx.egress.clone(),
8177 )
8178 .unwrap();
8179 } else {
8180 hello_via_sink(
8181 &handler,
8182 &ctx,
8183 &mut rx,
8184 hello_frame_with_manifest(initial.clone(), 1),
8185 )
8186 .await;
8187 }
8188 let response = handler
8189 .handle_control_frame(
8190 &ctx,
8191 catalog_update_with_capabilities_frame(
8192 2,
8193 capability_manifest("squatter", &["credentials-provider/v1"], &[])
8194 .capabilities
8195 .unwrap(),
8196 ),
8197 )
8198 .await
8199 .unwrap();
8200 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8201 assert_eq!(
8202 registry
8203 .get_module_by_connection(conn)
8204 .unwrap()
8205 .unwrap()
8206 .manifest,
8207 initial
8208 );
8209 }
8210 }
8211
8212 #[test]
8213 fn server_describe_surfaces_required_capability_verdict_fields() {
8214 let registry = Arc::new(Registry::default());
8215 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8216 [
8217 ("consumer".to_string(), true),
8218 ("provider".to_string(), false),
8219 ],
8220 BTreeMap::new(),
8221 );
8222 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
8223 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8224 provides: Vec::new(),
8225 requires: vec![subc_protocol::manifest::CapabilityRequirement {
8226 capability: "credentials-provider/v1".to_string(),
8227 need: subc_protocol::manifest::CapabilityNeed::Required,
8228 }],
8229 must_never_reach: Vec::new(),
8230 });
8231 let hello = Frame::build(
8232 FrameType::Hello,
8233 control_flags(),
8234 0,
8235 0,
8236 78,
8237 serde_json::to_vec(&ModuleHelloBody {
8238 manifest: consumer,
8239 protocol_ver: PROTOCOL_VERSION,
8240 control_ops: None,
8241 launch_nonce: None,
8242 })
8243 .expect("HELLO serializes"),
8244 )
8245 .expect("HELLO frame builds");
8246 handler
8247 .handle_control(ConnectionId::new(78), hello)
8248 .expect("consumer registers");
8249 let describe = Frame::build(
8250 FrameType::Request,
8251 control_flags(),
8252 0,
8253 0,
8254 79,
8255 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
8256 .expect("request serializes"),
8257 )
8258 .expect("describe frame builds");
8259 let response = handler
8260 .handle_server_describe(describe)
8261 .expect("server.describe succeeds");
8262 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
8263 let requirement = &rendered["capability_requirements"][0];
8264 assert_eq!(requirement["consumer"], "consumer");
8265 assert_eq!(requirement["verdict"], "never_provided");
8266 assert_eq!(requirement["episode_seq"], 1);
8267 assert_eq!(requirement["config_satisfiable"], false);
8268 assert_eq!(requirement["runtime_available"], false);
8269 assert!(requirement["detail"]
8270 .as_str()
8271 .expect("detail string")
8272 .contains("credentials-provider/v1"));
8273 }
8274
8275 #[test]
8276 fn catalog_list_omits_capabilities_for_legacy_manifest() {
8277 let registry = Arc::new(Registry::default());
8278 let handler = ControlHandler::new(Arc::clone(®istry));
8279 let hello = handler
8280 .handle_control(
8281 ConnectionId::new(101),
8282 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
8283 )
8284 .expect("legacy HELLO registers");
8285 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
8286
8287 let request = Frame::build(
8288 FrameType::Request,
8289 control_flags(),
8290 0,
8291 0,
8292 102,
8293 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8294 .expect("catalog request serializes"),
8295 )
8296 .expect("catalog request frame builds");
8297 let response = handler
8298 .handle_catalog_list(request, None)
8299 .expect("catalog list succeeds");
8300 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8301 assert!(
8302 body["modules"][0].get("capabilities").is_none(),
8303 "legacy manifest must retain an absent capabilities field on catalog.list"
8304 );
8305 }
8306
8307 #[test]
8308 fn hello_ack_omits_storage_when_no_storage_config() {
8309 let registry = Arc::new(Registry::default());
8310 let handler = ControlHandler::new(Arc::clone(®istry));
8311 let responses = handler
8312 .handle_control(
8313 ConnectionId::new(1),
8314 hello_frame("aft", PROTOCOL_VERSION, 7),
8315 )
8316 .unwrap();
8317 let ack = parse_ack(&responses[0]);
8318 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
8319 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
8320 }
8321
8322 #[tokio::test]
8323 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
8324 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
8325 let registry = Arc::new(Registry::default());
8326 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
8327 let responses = handler
8328 .handle_control(
8329 ConnectionId::new(1),
8330 hello_frame("aft", PROTOCOL_VERSION, 7),
8331 )
8332 .unwrap();
8333 let ack = parse_ack(&responses[0]);
8334 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
8335
8336 let described = handler
8337 .handle_control_frame(
8338 &route_ctx(ConnectionId::new(2)).0,
8339 Frame::build(
8340 FrameType::Request,
8341 control_flags(),
8342 0,
8343 0,
8344 9,
8345 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8346 )
8347 .unwrap(),
8348 )
8349 .await
8350 .unwrap();
8351 let ClientControlResponse::ServerDescribe { machine_id, .. } =
8352 serde_json::from_slice(&described[0].body).unwrap()
8353 else {
8354 panic!("server.describe answered with another shape");
8355 };
8356 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
8357 }
8358
8359 #[test]
8360 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
8361 let registry = Arc::new(Registry::default());
8364 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
8365 crate::daemon_config::StorageConfig::Sqlite {
8366 data_home: std::path::PathBuf::from("/data"),
8367 },
8368 ));
8369
8370 let responses = handler
8371 .handle_control(
8372 ConnectionId::new(1),
8373 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
8374 )
8375 .unwrap();
8376 let ack = parse_ack(&responses[0]);
8377 assert_eq!(
8378 ack.storage,
8379 Some(serde_json::json!({
8380 "module_id": "alfonso-routing",
8381 "storage_namespace": "default",
8382 "isolation": { "kind": "module" },
8383 "backend": {
8384 "backend": "sqlite",
8385 "path": "/data/cortexkit/alfonso-routing/store.db"
8386 }
8387 })),
8388 "the delivered descriptor is the module's own sqlite store path"
8389 );
8390 }
8391
8392 #[test]
8393 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
8394 let registry = Arc::new(Registry::default());
8395 let handler = ControlHandler::new(Arc::clone(®istry));
8396 let conn = ConnectionId::new(1);
8397 let responses = handler
8398 .handle_control(
8399 conn,
8400 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8401 )
8402 .unwrap();
8403 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8404 let registration = registry.get_module("aft").unwrap().unwrap();
8405 assert_eq!(registration.control_ops, module_baseline_control_ops());
8406
8407 let frame =
8408 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
8409 assert!(handler
8410 .guard_module_control_op(&frame, "aft", "route.bind")
8411 .unwrap()
8412 .is_none());
8413 let error = handler
8414 .guard_module_control_op(&frame, "aft", "test.synthetic")
8415 .unwrap()
8416 .expect("synthetic ungranted op should be rejected");
8417 assert_eq!(error.header.ty, FrameType::Error);
8418 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
8419 }
8420
8421 #[test]
8422 fn hello_control_ops_some_adds_optional_grants() {
8423 let registry = Arc::new(Registry::default());
8424 let handler = ControlHandler::new(Arc::clone(®istry));
8425 handler
8426 .handle_control(
8427 ConnectionId::new(1),
8428 hello_frame_with_control_ops(
8429 "aft",
8430 PROTOCOL_VERSION,
8431 7,
8432 Some(vec![
8433 "future.synthetic".to_string(),
8434 "route.bind".to_string(),
8435 ]),
8436 ),
8437 )
8438 .unwrap();
8439 let registration = registry.get_module("aft").unwrap().unwrap();
8440 assert_eq!(
8441 registration.control_ops,
8442 vec![
8443 "route.bind".to_string(),
8444 "route.status".to_string(),
8445 "future.synthetic".to_string(),
8446 ]
8447 );
8448 let frame =
8449 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
8450 assert!(handler
8451 .guard_module_control_op(&frame, "aft", "future.synthetic")
8452 .unwrap()
8453 .is_none());
8454 }
8455
8456 #[tokio::test]
8457 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
8458 let registry = Arc::new(Registry::default());
8459 let forwarding = Arc::new(ForwardingTable::default());
8460 let handler =
8461 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8462 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
8463 hello_via_sink(
8464 &handler,
8465 &module_ctx,
8466 &mut module_rx,
8467 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8468 )
8469 .await;
8470
8471 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
8472 let responses = handler
8473 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
8474 .await
8475 .unwrap();
8476 assert_eq!(responses.len(), 1);
8477 assert_eq!(responses[0].header.ty, FrameType::Error);
8478 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
8479 assert!(module_rx.try_recv().is_err());
8480 }
8481
8482 #[tokio::test]
8483 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
8484 let registry = Arc::new(Registry::default());
8485 let forwarding = Arc::new(ForwardingTable::default());
8486 let handler =
8487 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8488 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
8489 hello_via_sink(
8490 &handler,
8491 &module_ctx,
8492 &mut module_rx,
8493 hello_frame_with_control_ops(
8494 "aft",
8495 PROTOCOL_VERSION,
8496 7,
8497 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8498 ),
8499 )
8500 .await;
8501
8502 let project_root = unique_project_root("demux");
8503 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
8504 let route_handler = handler.clone();
8505 let route_task = tokio::spawn(async move {
8506 route_handler
8507 .handle_control_frame(
8508 &route_client_ctx,
8509 route_open_frame(100, "aft", project_root),
8510 )
8511 .await
8512 .unwrap()
8513 });
8514 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8515 .await
8516 .unwrap()
8517 .unwrap();
8518 assert!(matches!(
8519 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
8520 ModuleControlRequest::RouteBind { .. }
8521 ));
8522
8523 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
8524 let health_handler = handler.clone();
8525 let health_task = tokio::spawn(async move {
8526 health_handler
8527 .handle_control_frame(
8528 &health_client_ctx,
8529 supervisor_health_probe_frame(101, "aft"),
8530 )
8531 .await
8532 .unwrap()
8533 });
8534 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8535 .await
8536 .unwrap()
8537 .unwrap();
8538 assert_eq!(
8539 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8540 ModuleControlRequest::HealthCheck {}
8541 );
8542
8543 handler
8544 .handle_control_frame(
8545 &module_ctx,
8546 health_response(health_frame.header.corr, HealthStatus::Degraded),
8547 )
8548 .await
8549 .unwrap();
8550 let health_response = health_task.await.unwrap();
8551 assert_eq!(health_response.len(), 1);
8552 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8553 ClientControlResponse::SupervisorHealthProbe {
8554 module_id,
8555 status,
8556 detail,
8557 metrics,
8558 } => {
8559 assert_eq!(module_id, "aft");
8560 assert_eq!(status, HealthStatus::Degraded);
8561 assert_eq!(detail.as_deref(), Some("warming"));
8562 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
8563 }
8564 other => panic!("unexpected health response: {other:?}"),
8565 }
8566
8567 handler
8568 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8569 .await
8570 .unwrap();
8571 let route_response = route_task.await.unwrap();
8572 assert!(route_response.is_empty());
8573 let published = route_client_rx.recv().await.unwrap();
8574 assert!(matches!(
8575 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8576 ClientControlResponse::RouteOpen { .. }
8577 ));
8578 }
8579
8580 async fn relay_route_open(
8585 handler: &ControlHandler,
8586 client_connection: ConnectionId,
8587 client_egress: &FrameSink,
8588 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8589 corr: u64,
8590 module_id: &str,
8591 project_root_label: &str,
8592 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
8593 let ctx = RouteCtx {
8594 connection_id: client_connection,
8595 egress: client_egress.clone(),
8596 };
8597 let handler = handler.clone();
8598 let project_root = unique_project_root(project_root_label);
8599 let module_id = module_id.to_string();
8600 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
8601 let task = tokio::spawn(async move {
8602 let _guard = tracing::dispatcher::set_default(&dispatch);
8603 handler
8604 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
8605 .await
8606 .unwrap()
8607 });
8608 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
8609 .await
8610 .expect("module receives the relayed route.bind")
8611 .expect("module egress is open");
8612 (task, bind.frame)
8613 }
8614
8615 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
8616 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
8617 ModuleControlRequest::RouteBind {
8618 route_channel,
8619 epoch,
8620 ..
8621 } => (route_channel, epoch),
8622 other => panic!("expected a route.bind request, got {other:?}"),
8623 }
8624 }
8625
8626 fn published_route(frame: &Frame) -> (u16, u32) {
8627 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
8628 ClientControlResponse::RouteOpen {
8629 route_channel,
8630 route_epoch,
8631 } => (route_channel, route_epoch),
8632 other => panic!("expected a route.open response, got {other:?}"),
8633 }
8634 }
8635
8636 #[tokio::test]
8656 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
8657 let registry = Arc::new(Registry::default());
8658 let forwarding = Arc::new(ForwardingTable::default());
8659 let handler =
8660 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8661
8662 let module_connection = ConnectionId::new(30);
8663 let (module_ctx, mut module_rx) = route_ctx(module_connection);
8664 hello_via_sink(
8665 &handler,
8666 &module_ctx,
8667 &mut module_rx,
8668 hello_frame("aft", PROTOCOL_VERSION, 7),
8669 )
8670 .await;
8671
8672 let dying_client = ConnectionId::new(31);
8673 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
8674
8675 let (first_task, first_bind) = relay_route_open(
8678 &handler,
8679 dying_client,
8680 &dying_ctx.egress,
8681 &mut module_rx,
8682 100,
8683 "aft",
8684 "closing-first",
8685 )
8686 .await;
8687 handler
8688 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
8689 .await
8690 .unwrap();
8691 assert!(first_task.await.unwrap().is_empty());
8692 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
8693
8694 let (second_task, second_bind) = relay_route_open(
8696 &handler,
8697 dying_client,
8698 &dying_ctx.egress,
8699 &mut module_rx,
8700 101,
8701 "aft",
8702 "closing-second",
8703 )
8704 .await;
8705 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
8706
8707 assert!(forwarding
8710 .escalate_client_delivery_failure(
8711 dying_client,
8712 first_channel,
8713 first_epoch,
8714 CloseReason::new(
8715 "module_to_client_delivery_failed",
8716 "client egress refused a module frame",
8717 ),
8718 crate::forwarding::UndeliveredFrame {
8719 module_id: None,
8720 sink: &dying_ctx.egress,
8721 },
8722 )
8723 .unwrap());
8724 assert!(!dying_ctx.egress.is_closed());
8725
8726 let ack = handler
8728 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
8729 .await;
8730 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
8731 if module_loop_error.is_some() {
8732 handler.cleanup_connection(module_connection).unwrap();
8736 }
8737 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8741 .await
8742 .ok()
8743 .flatten();
8744
8745 assert!(
8747 registry
8748 .get_module_by_connection(module_connection)
8749 .unwrap()
8750 .is_some(),
8751 "one client's closing connection ended the shared module connection: \
8752 {module_loop_error:?}"
8753 );
8754 let cotenant = ConnectionId::new(32);
8756 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
8757 let (cotenant_task, cotenant_bind) = relay_route_open(
8758 &handler,
8759 cotenant,
8760 &cotenant_ctx.egress,
8761 &mut module_rx,
8762 102,
8763 "aft",
8764 "closing-cotenant",
8765 )
8766 .await;
8767 handler
8768 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
8769 .await
8770 .unwrap();
8771 assert!(cotenant_task.await.unwrap().is_empty());
8772 let (cotenant_channel, cotenant_epoch) =
8773 published_route(&cotenant_rx.recv().await.unwrap());
8774 assert!(matches!(
8775 forwarding
8776 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
8777 .unwrap(),
8778 DataRoute::Client(DataRouteState::Bound(_))
8779 ));
8780
8781 let goodbye = post_ack_module_frame
8784 .expect("module receives a GOODBYE for the abandoned route channel");
8785 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8786 assert_eq!(goodbye.header.channel, abandoned_channel);
8787 assert_eq!(goodbye.header.epoch, abandoned_epoch);
8788
8789 assert!(dying_rx.try_recv().is_err());
8793 let second_response = second_task.await.unwrap();
8794 assert_eq!(second_response.len(), 1);
8795 assert_eq!(
8796 parse_error(&second_response[0])["code"],
8797 "target_unavailable"
8798 );
8799 }
8800
8801 async fn route_open_held_after_bind_send() -> (
8805 ControlHandler,
8806 Arc<tokio::sync::Barrier>,
8807 RouteCtx,
8808 mpsc::Receiver<crate::router::OutboundFrame>,
8809 ) {
8810 let hold = Arc::new(tokio::sync::Barrier::new(2));
8811 let handler = ControlHandler::with_forwarding(
8812 Arc::new(Registry::default()),
8813 Arc::new(ForwardingTable::default()),
8814 )
8815 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
8816 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
8817 hello_via_sink(
8818 &handler,
8819 &module_ctx,
8820 &mut module_rx,
8821 hello_frame("aft", PROTOCOL_VERSION, 7),
8822 )
8823 .await;
8824 (handler, hold, module_ctx, module_rx)
8825 }
8826
8827 #[tokio::test]
8834 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
8835 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
8836 let client = ConnectionId::new(41);
8837 let (client_ctx, mut client_rx) = route_ctx(client);
8838
8839 let (open_task, bind) = relay_route_open(
8840 &handler,
8841 client,
8842 &client_ctx.egress,
8843 &mut module_rx,
8844 200,
8845 "aft",
8846 "acked-before-mark",
8847 )
8848 .await;
8849 let (module_channel, module_epoch) = route_bind_channel(&bind);
8850 hold.wait().await;
8852
8853 assert!(handler
8855 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
8856 .await
8857 .unwrap()
8858 .is_empty());
8859 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
8860
8861 hold.wait().await;
8862 assert!(
8863 open_task.await.unwrap().is_empty(),
8864 "route.open answers accepted"
8865 );
8866
8867 match module_rx.try_recv() {
8870 Err(mpsc::error::TryRecvError::Empty) => {}
8871 Ok(outbound) => panic!(
8872 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
8873 outbound.frame.header.ty,
8874 outbound.frame.header.channel,
8875 outbound.frame.header.epoch
8876 ),
8877 Err(err) => panic!("module egress closed: {err}"),
8878 }
8879
8880 let router = Router::with_control_handler(Arc::new(handler.clone()));
8883 let request = Frame::build(
8884 FrameType::Request,
8885 Flags::new(true, Priority::Interactive, false),
8886 client_channel,
8887 client_epoch,
8888 201,
8889 b"first request".to_vec(),
8890 )
8891 .unwrap();
8892 router
8893 .route_for_connection(&client_ctx, request)
8894 .await
8895 .unwrap();
8896 let delivered = module_rx.try_recv().unwrap().frame;
8897 assert_eq!(delivered.header.ty, FrameType::Request);
8898 assert_eq!(
8899 (delivered.header.channel, delivered.header.epoch),
8900 (module_channel, module_epoch)
8901 );
8902 assert_eq!(delivered.body, b"first request");
8903 }
8904
8905 #[tokio::test]
8911 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
8912 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
8913 let client = ConnectionId::new(42);
8914 let (client_ctx, _client_rx) = route_ctx(client);
8915
8916 let (open_task, bind) = relay_route_open(
8917 &handler,
8918 client,
8919 &client_ctx.egress,
8920 &mut module_rx,
8921 300,
8922 "aft",
8923 "released-before-mark",
8924 )
8925 .await;
8926 let (module_channel, module_epoch) = route_bind_channel(&bind);
8927 hold.wait().await;
8928
8929 handler.cleanup_connection(client).unwrap();
8930 assert!(
8931 module_rx.try_recv().is_err(),
8932 "client teardown must not GOODBYE a bind route.open is still queueing"
8933 );
8934
8935 hold.wait().await;
8936 let response = open_task.await.unwrap();
8937 assert_eq!(response.len(), 1);
8938 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8939
8940 let goodbye = module_rx
8941 .try_recv()
8942 .expect("module is told to drop the binding the daemon abandoned")
8943 .frame;
8944 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8945 assert_eq!(
8946 (goodbye.header.channel, goodbye.header.epoch),
8947 (module_channel, module_epoch)
8948 );
8949 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
8950 }
8951
8952 #[tokio::test]
8957 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
8958 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
8959 let client = ConnectionId::new(43);
8960 let (client_ctx, client_rx) = route_ctx(client);
8961
8962 let (open_task, bind) = relay_route_open(
8963 &handler,
8964 client,
8965 &client_ctx.egress,
8966 &mut module_rx,
8967 400,
8968 "aft",
8969 "gone-client-before-mark",
8970 )
8971 .await;
8972 let (module_channel, module_epoch) = route_bind_channel(&bind);
8973 hold.wait().await;
8974
8975 drop(client_rx);
8976 assert!(client_ctx.egress.is_closed());
8977 handler
8978 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
8979 .await
8980 .unwrap();
8981
8982 hold.wait().await;
8983 let response = open_task.await.unwrap();
8984 assert_eq!(response.len(), 1);
8985 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8986
8987 let goodbye = module_rx
8988 .try_recv()
8989 .expect("module is told to drop the binding it acked")
8990 .frame;
8991 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8992 assert_eq!(
8993 (goodbye.header.channel, goodbye.header.epoch),
8994 (module_channel, module_epoch)
8995 );
8996 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
8997 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
8998 }
8999
9000 #[test]
9007 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
9008 let handler = ControlHandler::default();
9009 let module_connection = ConnectionId::new(30);
9010 let client_connection = ConnectionId::new(31);
9011
9012 handler
9013 .refuse_to_end_module_connection_for_a_client(
9014 module_connection,
9015 77,
9016 ForwardingError::ConnectionClosing {
9017 connection_id: client_connection,
9018 },
9019 )
9020 .expect("a closing client must never end the module connection");
9021
9022 assert!(matches!(
9023 handler.refuse_to_end_module_connection_for_a_client(
9024 module_connection,
9025 78,
9026 ForwardingError::ConnectionClosing {
9027 connection_id: module_connection,
9028 },
9029 ),
9030 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
9031 connection_id
9032 })) if connection_id == module_connection
9033 ));
9034 assert!(matches!(
9035 handler.refuse_to_end_module_connection_for_a_client(
9036 module_connection,
9037 79,
9038 ForwardingError::Poisoned,
9039 ),
9040 Err(RouterError::Forwarding(ForwardingError::Poisoned))
9041 ));
9042 assert!(matches!(
9043 handler.refuse_to_end_module_connection_for_a_client(
9044 module_connection,
9045 80,
9046 ForwardingError::StaleModuleEndpoint,
9047 ),
9048 Err(RouterError::Forwarding(
9049 ForwardingError::StaleModuleEndpoint
9050 ))
9051 ));
9052 }
9053
9054 #[tokio::test]
9067 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
9068 let registry = Arc::new(Registry::default());
9069 let forwarding = Arc::new(ForwardingTable::default());
9070 let supervisor = SupervisorHandle::new();
9071 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9072 let handler =
9073 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9074 .with_supervisor(supervisor);
9075
9076 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9077 hello_via_sink(
9078 &handler,
9079 &target_ctx,
9080 &mut target_rx,
9081 hello_frame("target", PROTOCOL_VERSION, 1),
9082 )
9083 .await;
9084
9085 let wrong_nonce = handler
9089 .handle_control_frame(
9090 &route_ctx(ConnectionId::new(91)).0,
9091 route_open_frame_with_admission_facts(
9092 20,
9093 "target",
9094 unique_project_root("admission-facts"),
9095 Some(subc_control::ConsumerIdentity {
9096 module_id: "fed".to_string(),
9097 launch_nonce: "not-the-real-nonce".to_string(),
9098 }),
9099 None,
9100 ),
9101 )
9102 .await
9103 .unwrap();
9104 assert_eq!(
9105 parse_error(&wrong_nonce[0])["code"],
9106 "bad_consumer_identity",
9107 "a mismatched launch nonce must be refused, not stamped Reserved"
9108 );
9109
9110 let never_spawned = handler
9114 .handle_control_frame(
9115 &route_ctx(ConnectionId::new(92)).0,
9116 route_open_frame_with_admission_facts(
9117 21,
9118 "target",
9119 unique_project_root("admission-facts"),
9120 Some(subc_control::ConsumerIdentity {
9121 module_id: "never-spawned".to_string(),
9122 launch_nonce: "any-nonce".to_string(),
9123 }),
9124 None,
9125 ),
9126 )
9127 .await
9128 .unwrap();
9129 assert_eq!(
9130 parse_error(&never_spawned[0])["code"],
9131 "bad_consumer_identity",
9132 "an unspawned module_id must be refused rather than accepted for lack of a record"
9133 );
9134 }
9135
9136 #[tokio::test]
9159 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
9160 let registry = Arc::new(Registry::default());
9161 let forwarding = Arc::new(ForwardingTable::default());
9162 let supervisor = SupervisorHandle::new();
9163 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9164 let handler =
9165 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9166 .with_supervisor(supervisor);
9167
9168 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
9169 hello_via_sink(
9170 &handler,
9171 &target_ctx,
9172 &mut target_rx,
9173 hello_frame("target", PROTOCOL_VERSION, 1),
9174 )
9175 .await;
9176
9177 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
9178 let route_handler = handler.clone();
9179 let route_task = tokio::spawn(async move {
9180 route_handler
9181 .handle_control_frame(
9182 &client_ctx,
9183 route_open_frame_with_admission_facts(
9184 30,
9185 "target",
9186 unique_project_root("admission-facts"),
9187 Some(subc_control::ConsumerIdentity {
9188 module_id: "fed".to_string(),
9189 launch_nonce: "fed-nonce".to_string(),
9190 }),
9191 None,
9192 ),
9193 )
9194 .await
9195 .unwrap()
9196 });
9197
9198 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9205 .await
9206 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
9207 .expect("module control channel closed before route.bind");
9208 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9209 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
9210 panic!("expected route.bind")
9211 };
9212 assert_eq!(
9213 principal,
9214 Some(Principal::Reserved {
9215 module_id: "fed".to_string()
9216 }),
9217 "a correctly attested consumer must be stamped Reserved for its own id"
9218 );
9219
9220 handler
9221 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9222 .await
9223 .unwrap();
9224 assert!(route_task.await.unwrap().is_empty());
9225 assert!(
9226 matches!(
9227 serde_json::from_slice::<ClientControlResponse>(
9228 &client_rx.recv().await.unwrap().body
9229 )
9230 .unwrap(),
9231 ClientControlResponse::RouteOpen { .. }
9232 ),
9233 "the route must actually open, not merely avoid an error"
9234 );
9235 }
9236
9237 #[tokio::test(start_paused = true)]
9238 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
9239 let registry = Arc::new(Registry::default());
9240 let forwarding = Arc::new(ForwardingTable::default());
9241 let supervisor = SupervisorHandle::new();
9242 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9243 let handler =
9244 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9245 .with_supervisor(supervisor);
9246
9247 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
9248 hello_via_sink(
9249 &handler,
9250 &target_ctx,
9251 &mut target_rx,
9252 hello_frame("target", PROTOCOL_VERSION, 1),
9253 )
9254 .await;
9255
9256 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
9257 let direct_handler = handler.clone();
9258 let direct_open = tokio::spawn(async move {
9259 direct_handler
9260 .handle_control_frame(
9261 &direct_ctx,
9262 route_open_frame(2, "target", unique_project_root("route-census-direct")),
9263 )
9264 .await
9265 .unwrap()
9266 });
9267 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9268 .await
9269 .expect("no direct route.bind within 5s")
9270 .expect("target control channel closed before direct route.bind");
9271 handler
9272 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
9273 .await
9274 .unwrap();
9275 assert!(direct_open.await.unwrap().is_empty());
9276 let _ = direct_rx.recv().await.unwrap();
9277
9278 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
9279 let reserved_handler = handler.clone();
9280 let reserved_open = tokio::spawn(async move {
9281 reserved_handler
9282 .handle_control_frame(
9283 &reserved_ctx,
9284 route_open_frame_with_admission_facts(
9285 3,
9286 "target",
9287 unique_project_root("admission-facts"),
9288 Some(ConsumerIdentity {
9289 module_id: "fed".to_string(),
9290 launch_nonce: "fed-nonce".to_string(),
9291 }),
9292 None,
9293 ),
9294 )
9295 .await
9296 .unwrap()
9297 });
9298 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9299 .await
9300 .expect("no reserved route.bind within 5s")
9301 .expect("target control channel closed before reserved route.bind");
9302 handler
9303 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
9304 .await
9305 .unwrap();
9306 assert!(reserved_open.await.unwrap().is_empty());
9307 let _ = reserved_rx.recv().await.unwrap();
9308
9309 forwarding
9310 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
9311 .unwrap();
9312 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
9313 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
9314 module_id: Some("target".to_string()),
9315 })
9316 .unwrap();
9317 let census_frame =
9318 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
9319 let response = handler
9320 .handle_control_frame(&census_ctx, census_frame)
9321 .await
9322 .unwrap()
9323 .pop()
9324 .unwrap();
9325 let actual: Value = serde_json::from_slice(&response.body).unwrap();
9326 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
9327 assert!(matches!(
9328 decoded,
9329 ClientControlResponse::SupervisorRoutes { .. }
9330 ));
9331 let routes = actual["modules"][0]["routes"].as_array().unwrap();
9332 assert_eq!(routes.len(), 2);
9333 assert!(routes.iter().all(|route| route["draining"] == true));
9334 assert!(
9337 routes.iter().all(|route| route["drain_reason"] == "reload"),
9338 "draining routes must name the drain's reason: {routes:?}"
9339 );
9340 assert!(routes.iter().any(|route| {
9341 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
9342 }));
9343 assert!(routes.iter().any(|route| {
9344 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
9345 }));
9346
9347 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9348 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
9349 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9350 std::fs::write(
9351 &golden_path,
9352 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
9353 )
9354 .unwrap();
9355 }
9356 let expected: Value =
9357 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
9358 assert_eq!(actual, expected);
9359 }
9360
9361 async fn query_live_roots(
9362 handler: &ControlHandler,
9363 module_ctx: &RouteCtx,
9364 ) -> ModuleControlResponseToModule {
9365 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
9366 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
9367 let response = handler
9368 .handle_control_frame(module_ctx, frame)
9369 .await
9370 .unwrap()
9371 .pop()
9372 .unwrap();
9373 serde_json::from_slice(&response.body).unwrap()
9374 }
9375
9376 #[tokio::test(start_paused = true)]
9377 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
9378 let registry = Arc::new(Registry::default());
9379 let forwarding = Arc::new(ForwardingTable::default());
9380 let handler = ControlHandler::with_forwarding(registry, forwarding);
9381 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
9382 hello_via_sink(
9383 &handler,
9384 &target_ctx,
9385 &mut target_rx,
9386 hello_frame("target", PROTOCOL_VERSION, 1),
9387 )
9388 .await;
9389 let root = unique_project_root("live-roots-known");
9390 let path = ProjectRootId::from_path_allowing_missing(root.path())
9391 .unwrap()
9392 .as_path()
9393 .to_path_buf();
9394 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
9395 let open_handler = handler.clone();
9396 let opened = tokio::spawn(async move {
9397 open_handler
9398 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
9399 .await
9400 .unwrap()
9401 });
9402 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9403 .await
9404 .unwrap()
9405 .unwrap();
9406 handler
9407 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
9408 .await
9409 .unwrap();
9410 assert!(opened.await.unwrap().is_empty());
9411 let _ = client_rx.recv().await.unwrap();
9412
9413 let root = unique_project_root("live-roots-pending");
9414 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
9415 .unwrap()
9416 .as_path()
9417 .to_path_buf();
9418 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
9419 let open_handler = handler.clone();
9420 let pending = tokio::spawn(async move {
9421 open_handler
9422 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
9423 .await
9424 .unwrap()
9425 });
9426 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9427 .await
9428 .unwrap()
9429 .unwrap();
9430 let actual = query_live_roots(&handler, &target_ctx).await;
9431 let ModuleControlResponseToModule::LiveRoots {
9432 roots,
9433 unknown_root_bindings,
9434 total_bindings,
9435 } = actual
9436 else {
9437 panic!("expected live roots")
9438 };
9439 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
9440 assert_eq!(unknown_root_bindings, 0);
9441 assert_eq!(
9442 roots.len(),
9443 2,
9444 "root-known arm must retain each canonical root"
9445 );
9446 assert_eq!(
9447 total_bindings,
9448 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9449 );
9450 let counts = roots
9451 .iter()
9452 .map(|root| (root.project_root.clone(), root.bound, root.pending))
9453 .collect::<Vec<_>>();
9454 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
9455 expected.sort_by(|a, b| a.0.cmp(&b.0));
9456 assert_eq!(
9457 counts, expected,
9458 "roots must sort by path and count pending separately"
9459 );
9460 handler
9461 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
9462 .await
9463 .unwrap();
9464 assert!(pending.await.unwrap().is_empty());
9465 }
9466
9467 #[tokio::test(start_paused = true)]
9468 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
9469 let forwarding = Arc::new(ForwardingTable::default());
9470 let handler =
9471 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
9472 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
9473 hello_via_sink(
9474 &handler,
9475 &target_ctx,
9476 &mut target_rx,
9477 hello_frame("target", PROTOCOL_VERSION, 1),
9478 )
9479 .await;
9480 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
9481 let pending = forwarding
9482 .begin_route_bind_relay_for_test(
9483 client_ctx.connection_id,
9484 client_ctx.egress.clone(),
9485 2,
9486 "target",
9487 )
9488 .unwrap();
9489 forwarding
9490 .complete_pending_relay(
9491 target_ctx.connection_id,
9492 pending.corr,
9493 RouteBindRelayOutcome::Accepted,
9494 )
9495 .unwrap();
9496 let actual = query_live_roots(&handler, &target_ctx).await;
9497 let ModuleControlResponseToModule::LiveRoots {
9498 roots,
9499 unknown_root_bindings,
9500 total_bindings,
9501 } = actual
9502 else {
9503 panic!("expected live roots")
9504 };
9505 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
9506 assert_eq!(
9507 unknown_root_bindings, 1,
9508 "unknown-root arm must not read as no bindings"
9509 );
9510 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
9511 assert_eq!(
9512 total_bindings,
9513 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9514 );
9515 }
9516
9517 #[tokio::test(start_paused = true)]
9525 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
9526 let forwarding = Arc::new(ForwardingTable::default());
9527 let handler =
9528 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
9529 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
9530 let replies = handler
9531 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
9532 .await
9533 .unwrap();
9534 let queued_by_hello = module_rx.len();
9535
9536 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
9537 let open_handler = handler.clone();
9538 let open = tokio::spawn(async move {
9539 open_handler
9540 .handle_control_frame(
9541 &client_ctx,
9542 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
9543 )
9544 .await
9545 .unwrap()
9546 });
9547 let mut spins = 0;
9549 while module_rx.len() == queued_by_hello {
9550 spins += 1;
9551 assert!(spins < 10_000, "route.open never queued a route.bind");
9552 tokio::task::yield_now().await;
9553 }
9554
9555 for reply in replies {
9557 module_ctx.egress.send(reply).await.unwrap();
9558 }
9559
9560 let first = module_rx.recv().await.unwrap().frame;
9561 assert_eq!(
9562 first.header.ty,
9563 FrameType::HelloAck,
9564 "the first frame a registering module reads must be its HELLO_ACK"
9565 );
9566 assert_eq!(first.header.corr, 7);
9567 let second = module_rx.recv().await.unwrap().frame;
9568 assert_eq!(second.header.ty, FrameType::Request);
9569 assert!(
9570 matches!(
9571 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
9572 ModuleControlRequest::RouteBind { .. }
9573 ),
9574 "the route.bind follows the ack"
9575 );
9576 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
9577
9578 handler
9579 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
9580 .await
9581 .unwrap();
9582 assert!(open.await.unwrap().is_empty());
9583 let _ = client_rx.recv().await.unwrap();
9584 }
9585
9586 #[tokio::test(start_paused = true)]
9587 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
9588 let handler = ControlHandler::with_forwarding(
9589 Arc::new(Registry::default()),
9590 Arc::new(ForwardingTable::default()),
9591 );
9592 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
9593 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
9594 hello_via_sink(
9595 &handler,
9596 &first_ctx,
9597 &mut first_rx,
9598 hello_frame("first", PROTOCOL_VERSION, 1),
9599 )
9600 .await;
9601 hello_via_sink(
9602 &handler,
9603 &second_ctx,
9604 &mut second_rx,
9605 hello_frame("second", PROTOCOL_VERSION, 2),
9606 )
9607 .await;
9608 let root = unique_project_root("second-only");
9609 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
9610 let cloned = handler.clone();
9611 let open = tokio::spawn(async move {
9612 cloned
9613 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
9614 .await
9615 .unwrap()
9616 });
9617 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
9618 .await
9619 .unwrap()
9620 .unwrap();
9621 let first = query_live_roots(&handler, &first_ctx).await;
9622 let second = query_live_roots(&handler, &second_ctx).await;
9623 assert!(
9624 matches!(
9625 first,
9626 ModuleControlResponseToModule::LiveRoots {
9627 total_bindings: 0,
9628 ..
9629 }
9630 ),
9631 "cross-module scope must not expose another module's roots"
9632 );
9633 assert!(
9634 matches!(
9635 second,
9636 ModuleControlResponseToModule::LiveRoots {
9637 total_bindings: 1,
9638 ..
9639 }
9640 ),
9641 "second module must see its pending route"
9642 );
9643 handler
9644 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
9645 .await
9646 .unwrap();
9647 assert!(open.await.unwrap().is_empty());
9648 }
9649
9650 #[tokio::test(start_paused = true)]
9651 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
9652 let handler = ControlHandler::with_forwarding(
9653 Arc::new(Registry::default()),
9654 Arc::new(ForwardingTable::default()),
9655 );
9656 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
9657 hello_via_sink(
9658 &handler,
9659 &target_ctx,
9660 &mut target_rx,
9661 hello_frame("target", PROTOCOL_VERSION, 1),
9662 )
9663 .await;
9664 let actual = query_live_roots(&handler, &target_ctx).await;
9665 let ModuleControlResponseToModule::LiveRoots {
9666 roots,
9667 unknown_root_bindings,
9668 total_bindings,
9669 } = actual
9670 else {
9671 panic!("expected live roots")
9672 };
9673 assert!(roots.is_empty());
9674 assert_eq!(unknown_root_bindings, 0);
9675 assert_eq!(total_bindings, 0);
9676 assert_eq!(
9677 total_bindings,
9678 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9679 );
9680 }
9681
9682 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
9689 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9690 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
9691 let text = std::fs::read_to_string(&path)
9692 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
9693 let vectors: Vec<(String, Value)> = text
9694 .lines()
9695 .filter(|line| !line.trim().is_empty())
9696 .map(|line| {
9697 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
9698 let id = entry["corpus_id"]
9699 .as_str()
9700 .expect("every vector carries a corpus_id")
9701 .to_string();
9702 (id, entry["package"].clone())
9703 })
9704 .collect();
9705 assert_eq!(
9709 vectors.len(),
9710 3,
9711 "vendored fed corpus changed size; re-sync from subc-federation"
9712 );
9713
9714 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
9729 let richest = vectors
9730 .iter()
9731 .filter_map(|(_, package)| package.as_object())
9732 .map(|object| {
9733 object
9734 .keys()
9735 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
9736 .count()
9737 })
9738 .max()
9739 .unwrap_or(0);
9740 assert!(
9741 richest >= 2,
9742 "vendored corpus no longer carries a package with unmodelled fields, \
9743 so the relay test can no longer distinguish a verbatim relay from a lossy one"
9744 );
9745
9746 vectors
9747 }
9748
9749 #[tokio::test]
9757 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
9758 for (corpus_id, package) in fed_admission_facts_vectors() {
9759 let registry = Arc::new(Registry::default());
9760 let forwarding = Arc::new(ForwardingTable::default());
9761 let supervisor = SupervisorHandle::new();
9762 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9763 let handler =
9764 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9765 .with_supervisor(supervisor)
9766 .with_admission_facts_config(
9767 Some("fed".to_string()),
9768 Some(vec!["target".to_string()]),
9769 );
9770
9771 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9772 hello_via_sink(
9773 &handler,
9774 &target_ctx,
9775 &mut target_rx,
9776 hello_frame("target", PROTOCOL_VERSION, 1),
9777 )
9778 .await;
9779
9780 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
9781 let route_handler = handler.clone();
9782 let expected = package.clone();
9783 let route_task = tokio::spawn(async move {
9784 route_handler
9785 .handle_control_frame(
9786 &client_ctx,
9787 route_open_frame_with_admission_facts(
9788 20,
9789 "target",
9790 unique_project_root("admission-facts"),
9791 Some(subc_control::ConsumerIdentity {
9792 module_id: "fed".to_string(),
9793 launch_nonce: "fed-nonce".to_string(),
9794 }),
9795 Some(package),
9796 ),
9797 )
9798 .await
9799 .unwrap()
9800 });
9801
9802 let bind_frame = target_rx.recv().await.unwrap();
9803 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9804 let ModuleControlRequest::RouteBind {
9805 admission_facts, ..
9806 } = bind
9807 else {
9808 panic!("{corpus_id}: expected route.bind")
9809 };
9810 assert_eq!(
9811 admission_facts,
9812 Some(expected),
9813 "{corpus_id}: relay must not add, drop or reshape any field"
9814 );
9815
9816 handler
9817 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9818 .await
9819 .unwrap();
9820 route_task.await.unwrap();
9821 }
9822 }
9823
9824 #[tokio::test]
9825 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
9826 let registry = Arc::new(Registry::default());
9827 let forwarding = Arc::new(ForwardingTable::default());
9828 let supervisor = SupervisorHandle::new();
9829 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9830 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
9831 let handler =
9832 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9833 .with_supervisor(supervisor)
9834 .with_admission_facts_config(
9835 Some("fed".to_string()),
9836 Some(vec!["target".to_string()]),
9837 );
9838
9839 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
9840 hello_via_sink(
9841 &handler,
9842 &target_ctx,
9843 &mut target_rx,
9844 hello_frame("target", PROTOCOL_VERSION, 1),
9845 )
9846 .await;
9847 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
9848 hello_via_sink(
9849 &handler,
9850 &other_ctx,
9851 &mut other_rx,
9852 hello_frame("other", PROTOCOL_VERSION, 2),
9853 )
9854 .await;
9855
9856 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
9857 let expected_facts = facts.clone();
9858 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
9859 let route_handler = handler.clone();
9860 let route_task = tokio::spawn(async move {
9861 route_handler
9862 .handle_control_frame(
9863 &client_ctx,
9864 route_open_frame_with_admission_facts(
9865 10,
9866 "target",
9867 unique_project_root("admission-facts"),
9868 Some(subc_control::ConsumerIdentity {
9869 module_id: "fed".to_string(),
9870 launch_nonce: "fed-nonce".to_string(),
9871 }),
9872 Some(facts.clone()),
9873 ),
9874 )
9875 .await
9876 .unwrap()
9877 });
9878 let bind_frame = target_rx.recv().await.unwrap();
9879 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9880 let ModuleControlRequest::RouteBind {
9881 admission_facts, ..
9882 } = bind
9883 else {
9884 panic!("expected route.bind")
9885 };
9886 assert_eq!(admission_facts, Some(expected_facts));
9887 handler
9888 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9889 .await
9890 .unwrap();
9891 assert!(route_task.await.unwrap().is_empty());
9892 assert!(matches!(
9893 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
9894 .unwrap(),
9895 ClientControlResponse::RouteOpen { .. }
9896 ));
9897
9898 let direct = handler
9899 .handle_control_frame(
9900 &route_ctx(ConnectionId::new(73)).0,
9901 route_open_frame_with_admission_facts(
9902 11,
9903 "target",
9904 unique_project_root("admission-facts"),
9905 None,
9906 Some(json!({"x": 1})),
9907 ),
9908 )
9909 .await
9910 .unwrap();
9911 assert_eq!(
9912 parse_error(&direct[0])["code"],
9913 "admission_facts_not_permitted"
9914 );
9915
9916 let different_reserved = handler
9917 .handle_control_frame(
9918 &route_ctx(ConnectionId::new(77)).0,
9919 route_open_frame_with_admission_facts(
9920 15,
9921 "target",
9922 unique_project_root("admission-facts"),
9923 Some(subc_control::ConsumerIdentity {
9924 module_id: "other".to_string(),
9925 launch_nonce: "other-nonce".to_string(),
9926 }),
9927 Some(json!({"x": 1})),
9928 ),
9929 )
9930 .await
9931 .unwrap();
9932 assert_eq!(
9933 parse_error(&different_reserved[0])["code"],
9934 "admission_facts_not_permitted"
9935 );
9936
9937 let other_target = handler
9938 .handle_control_frame(
9939 &route_ctx(ConnectionId::new(74)).0,
9940 route_open_frame_with_admission_facts(
9941 12,
9942 "other",
9943 unique_project_root("admission-facts"),
9944 Some(subc_control::ConsumerIdentity {
9945 module_id: "fed".to_string(),
9946 launch_nonce: "fed-nonce".to_string(),
9947 }),
9948 Some(json!({"x": 1})),
9949 ),
9950 )
9951 .await
9952 .unwrap();
9953 assert_eq!(
9954 parse_error(&other_target[0])["code"],
9955 "admission_facts_target_not_allowed"
9956 );
9957
9958 let nonexistent = handler
9959 .handle_control_frame(
9960 &route_ctx(ConnectionId::new(75)).0,
9961 route_open_frame_with_admission_facts(
9962 13,
9963 "missing",
9964 unique_project_root("admission-facts"),
9965 None,
9966 Some(json!({"x": 1})),
9967 ),
9968 )
9969 .await
9970 .unwrap();
9971 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
9972
9973 let described = handler
9974 .handle_control_frame(
9975 &route_ctx(ConnectionId::new(76)).0,
9976 Frame::build(
9977 FrameType::Request,
9978 control_flags(),
9979 0,
9980 0,
9981 14,
9982 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9983 )
9984 .unwrap(),
9985 )
9986 .await
9987 .unwrap();
9988 let ClientControlResponse::ServerDescribe { capabilities, .. } =
9989 serde_json::from_slice(&described[0].body).unwrap()
9990 else {
9991 panic!("expected server.describe response")
9992 };
9993 assert!(capabilities
9994 .iter()
9995 .any(|cap| cap == "admission_facts_relay_v1"));
9996 }
9997
9998 #[tokio::test]
9999 async fn admission_facts_without_configured_carrier_are_rejected() {
10000 let registry = Arc::new(Registry::default());
10001 let forwarding = Arc::new(ForwardingTable::default());
10002 let handler = ControlHandler::with_forwarding(registry, forwarding);
10003 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
10004 hello_via_sink(
10005 &handler,
10006 &target_ctx,
10007 &mut target_rx,
10008 hello_frame("target", PROTOCOL_VERSION, 1),
10009 )
10010 .await;
10011
10012 let responses = handler
10013 .handle_control_frame(
10014 &route_ctx(ConnectionId::new(79)).0,
10015 route_open_frame_with_admission_facts(
10016 16,
10017 "target",
10018 unique_project_root("admission-facts"),
10019 None,
10020 Some(json!({"x": 1})),
10021 ),
10022 )
10023 .await
10024 .unwrap();
10025 assert_eq!(
10026 parse_error(&responses[0])["code"],
10027 "admission_facts_not_permitted"
10028 );
10029 }
10030
10031 #[tokio::test]
10032 async fn route_open_relays_consumer_capabilities_verbatim() {
10033 let registry = Arc::new(Registry::default());
10034 let forwarding = Arc::new(ForwardingTable::default());
10035 let handler =
10036 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10037 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
10038 hello_via_sink(
10039 &handler,
10040 &module_ctx,
10041 &mut module_rx,
10042 hello_frame("aft", PROTOCOL_VERSION, 7),
10043 )
10044 .await;
10045
10046 let expected = vec!["elicitation".to_string(), "roots".to_string()];
10047 let expected_for_request = expected.clone();
10048 let project_root = unique_project_root("consumer-capabilities-present");
10049 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
10050 let route_handler = handler.clone();
10051 let route_task = tokio::spawn(async move {
10052 route_handler
10053 .handle_control_frame(
10054 &client_ctx,
10055 route_open_frame_with_consumer_capabilities(
10056 401,
10057 "aft",
10058 project_root,
10059 Some(expected_for_request),
10060 ),
10061 )
10062 .await
10063 .unwrap()
10064 });
10065 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10066 .await
10067 .unwrap()
10068 .unwrap();
10069 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10070 let ModuleControlRequest::RouteBind {
10071 consumer_capabilities,
10072 ..
10073 } = bind
10074 else {
10075 panic!("expected route.bind request, got {bind:?}");
10076 };
10077 assert_eq!(consumer_capabilities, Some(expected.clone()));
10078
10079 handler
10080 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10081 .await
10082 .unwrap();
10083 let route_response = route_task.await.unwrap();
10084 assert!(route_response.is_empty());
10085 let published = client_rx.recv().await.unwrap();
10086 assert!(matches!(
10087 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10088 ClientControlResponse::RouteOpen { .. }
10089 ));
10090 }
10091
10092 #[tokio::test]
10093 async fn route_open_without_consumer_capabilities_relays_none() {
10094 let registry = Arc::new(Registry::default());
10095 let forwarding = Arc::new(ForwardingTable::default());
10096 let handler =
10097 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10098 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
10099 hello_via_sink(
10100 &handler,
10101 &module_ctx,
10102 &mut module_rx,
10103 hello_frame("aft", PROTOCOL_VERSION, 7),
10104 )
10105 .await;
10106
10107 let project_root = unique_project_root("consumer-capabilities-absent");
10108 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
10109 let route_handler = handler.clone();
10110 let route_task = tokio::spawn(async move {
10111 route_handler
10112 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
10113 .await
10114 .unwrap()
10115 });
10116 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10117 .await
10118 .unwrap()
10119 .unwrap();
10120 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10121 let ModuleControlRequest::RouteBind {
10122 consumer_capabilities,
10123 ..
10124 } = bind
10125 else {
10126 panic!("expected route.bind request, got {bind:?}");
10127 };
10128 assert_eq!(consumer_capabilities, None);
10129
10130 handler
10131 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10132 .await
10133 .unwrap();
10134 let route_response = route_task.await.unwrap();
10135 assert!(route_response.is_empty());
10136 let published = client_rx.recv().await.unwrap();
10137 assert!(matches!(
10138 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10139 ClientControlResponse::RouteOpen { .. }
10140 ));
10141 }
10142
10143 async fn bind_role_versions_for(
10147 sent: Option<BTreeMap<String, String>>,
10148 connection: u64,
10149 ) -> Option<BTreeMap<String, String>> {
10150 let registry = Arc::new(Registry::default());
10151 let forwarding = Arc::new(ForwardingTable::default());
10152 let handler =
10153 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10154 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
10155 hello_via_sink(
10156 &handler,
10157 &module_ctx,
10158 &mut module_rx,
10159 hello_frame("aft", PROTOCOL_VERSION, 7),
10160 )
10161 .await;
10162 let project_root = unique_project_root("role-versions");
10163 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
10164 let route_handler = handler.clone();
10165 let route_task = tokio::spawn(async move {
10166 route_handler
10167 .handle_control_frame(
10168 &client_ctx,
10169 route_open_frame_with_role_versions(403, "aft", project_root, sent),
10170 )
10171 .await
10172 .unwrap()
10173 });
10174 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10175 .await
10176 .expect("a well-formed route.open reaches the module as a bind")
10177 .unwrap();
10178 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10179 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
10180 panic!("expected route.bind request, got {bind:?}");
10181 };
10182 handler
10183 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10184 .await
10185 .unwrap();
10186 assert!(route_task.await.unwrap().is_empty());
10187 let published = client_rx.recv().await.unwrap();
10188 assert!(matches!(
10189 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10190 ClientControlResponse::RouteOpen { .. }
10191 ));
10192 role_versions
10193 }
10194
10195 #[tokio::test]
10196 async fn route_open_relays_role_versions_verbatim() {
10197 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
10198 assert_eq!(
10199 bind_role_versions_for(Some(sent.clone()), 141).await,
10200 Some(sent)
10201 );
10202 }
10203
10204 #[tokio::test]
10207 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
10208 assert_eq!(bind_role_versions_for(None, 143).await, None);
10209 assert_eq!(
10210 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
10211 None
10212 );
10213 }
10214
10215 #[tokio::test]
10216 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
10217 let registry = Arc::new(Registry::default());
10218 let forwarding = Arc::new(ForwardingTable::default());
10219 let handler =
10220 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10221 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
10222 hello_via_sink(
10223 &handler,
10224 &module_ctx,
10225 &mut module_rx,
10226 hello_frame("aft", PROTOCOL_VERSION, 7),
10227 )
10228 .await;
10229
10230 let nine: BTreeMap<String, String> = (0..9)
10231 .map(|index| (format!("role-{index}"), "v1".to_string()))
10232 .collect();
10233 for (label, malformed) in [
10234 (
10235 "invalid role name",
10236 role_versions(&[("Tool_Provider", "v1")]),
10237 ),
10238 ("invalid version", role_versions(&[("tool-provider", "v0")])),
10239 ("nine entries", nine),
10240 ] {
10241 let responses = tokio::time::timeout(
10244 Duration::from_secs(1),
10245 handler.handle_control_frame(
10246 &route_ctx(ConnectionId::new(148)).0,
10247 route_open_frame_with_role_versions(
10248 404,
10249 "aft",
10250 unique_project_root("role-versions-malformed"),
10251 Some(malformed),
10252 ),
10253 ),
10254 )
10255 .await
10256 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
10257 .unwrap();
10258 assert_eq!(responses.len(), 1, "{label}");
10259 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
10260 let error = parse_error(&responses[0]);
10261 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
10262 assert_eq!(
10263 error["detail"]["field"], "role_versions",
10264 "{label}: {error}"
10265 );
10266 assert!(
10267 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
10268 "{label}: a malformed declaration is terminal"
10269 );
10270 assert!(
10271 module_rx.try_recv().is_err(),
10272 "{label}: the module must never see a bind"
10273 );
10274 }
10275 }
10276
10277 #[tokio::test]
10281 async fn route_role_versions_capability_is_advertised() {
10282 let handler = ControlHandler::new(Arc::new(Registry::default()));
10283 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
10284 let ack = hello_via_sink(
10285 &handler,
10286 &ctx,
10287 &mut rx,
10288 hello_frame("m", PROTOCOL_VERSION, 1),
10289 )
10290 .await;
10291 let ack = parse_ack(&ack);
10292 assert!(
10293 ack.subc_capabilities
10294 .iter()
10295 .any(|c| c == "route-role-versions/v1"),
10296 "{:?}",
10297 ack.subc_capabilities
10298 );
10299
10300 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
10301 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
10302 let reply = handler
10303 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
10304 .await
10305 .unwrap()
10306 .pop()
10307 .unwrap();
10308 let ClientControlResponse::ServerDescribe { capabilities, .. } =
10309 serde_json::from_slice(&reply.body).unwrap()
10310 else {
10311 panic!("not a server.describe reply");
10312 };
10313 assert!(
10314 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
10315 "{capabilities:?}"
10316 );
10317 }
10318
10319 #[tokio::test]
10323 async fn catalog_events_capability_is_advertised() {
10324 let handler = ControlHandler::new(Arc::new(Registry::default()));
10325 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
10326 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
10327 let reply = handler
10328 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
10329 .await
10330 .unwrap()
10331 .pop()
10332 .unwrap();
10333 let ClientControlResponse::ServerDescribe { capabilities, .. } =
10334 serde_json::from_slice(&reply.body).unwrap()
10335 else {
10336 panic!("not a server.describe reply");
10337 };
10338 assert!(
10339 capabilities
10340 .iter()
10341 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
10342 "{capabilities:?}"
10343 );
10344 }
10345
10346 #[tokio::test]
10347 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
10348 let registry = Arc::new(Registry::default());
10349 let forwarding = Arc::new(ForwardingTable::default());
10350 let handler =
10351 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10352 .with_health_probe_timeout(Duration::from_secs(5));
10353 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
10354 hello_via_sink(
10355 &handler,
10356 &module_ctx,
10357 &mut module_rx,
10358 non_routable_hello_frame_with_control_ops(
10359 "mcp",
10360 300,
10361 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10362 ),
10363 )
10364 .await;
10365 assert!(registry
10366 .get_module("mcp")
10367 .unwrap()
10368 .unwrap()
10369 .manifest
10370 .provides
10371 .is_empty());
10372
10373 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
10374 let route_response = handler
10375 .handle_control_frame(
10376 &route_client_ctx,
10377 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
10378 )
10379 .await
10380 .unwrap();
10381 assert_eq!(route_response[0].header.ty, FrameType::Error);
10382 assert_eq!(
10383 parse_error(&route_response[0])["code"],
10384 "target_unavailable"
10385 );
10386 assert!(parse_error(&route_response[0])["message"]
10387 .as_str()
10388 .unwrap()
10389 .contains("does not provide the requested target"));
10390 assert!(module_rx.try_recv().is_err());
10391
10392 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
10393 let health_handler = handler.clone();
10394 let health_task = tokio::spawn(async move {
10395 health_handler
10396 .handle_control_frame(
10397 &health_client_ctx,
10398 supervisor_health_probe_frame(302, "mcp"),
10399 )
10400 .await
10401 .unwrap()
10402 });
10403 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10404 .await
10405 .unwrap()
10406 .unwrap();
10407 assert_eq!(
10408 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10409 ModuleControlRequest::HealthCheck {}
10410 );
10411 handler
10412 .handle_control_frame(
10413 &module_ctx,
10414 health_response(health_frame.header.corr, HealthStatus::Ok),
10415 )
10416 .await
10417 .unwrap();
10418 let health_response = health_task.await.unwrap();
10419 assert_eq!(health_response[0].header.ty, FrameType::Response);
10420 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10421 ClientControlResponse::SupervisorHealthProbe {
10422 module_id, status, ..
10423 } => {
10424 assert_eq!(module_id, "mcp");
10425 assert_eq!(status, HealthStatus::Ok);
10426 }
10427 other => panic!("unexpected health response: {other:?}"),
10428 }
10429
10430 forwarding
10435 .cleanup_connection(module_ctx.connection_id)
10436 .unwrap();
10437 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
10438 let cleanup_response = tokio::time::timeout(
10439 Duration::from_millis(200),
10440 handler.handle_control_frame(
10441 &cleanup_probe_ctx,
10442 supervisor_health_probe_frame(303, "mcp"),
10443 ),
10444 )
10445 .await
10446 .expect("probe should fail immediately when the control lane is gone")
10447 .unwrap();
10448 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
10449 assert_eq!(
10450 parse_error(&cleanup_response[0])["code"],
10451 "target_unavailable"
10452 );
10453 assert!(parse_error(&cleanup_response[0])["message"]
10454 .as_str()
10455 .unwrap()
10456 .contains("no module connection"));
10457
10458 handler
10459 .cleanup_connection(module_ctx.connection_id)
10460 .unwrap();
10461 }
10462
10463 #[tokio::test]
10464 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
10465 let registry = Arc::new(Registry::default());
10466 let supervisor_handle = SupervisorHandle::new();
10467 let supervisor =
10468 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
10469 .with_handle(supervisor_handle.clone())
10470 .with_connection_file_path(
10471 std::env::temp_dir()
10472 .join(format!("subc-route-open-warming-{}", std::process::id())),
10473 );
10474 let module = supervisor
10475 .supervise_configured(
10476 ModuleSpec {
10477 module_id: "warming".to_string(),
10478 program: fake_aft_stub_path(),
10479 args: Vec::new(),
10480 env: Vec::new(),
10481 reserved: false,
10482 reserved_prefixes: Vec::new(),
10483 protocol: ModuleProtocol::Subc,
10484 overlap: Default::default(),
10485 },
10486 true,
10487 )
10488 .unwrap();
10489 assert_eq!(module.state().unwrap(), ModuleState::Running);
10490
10491 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10492 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
10493 let response = handler
10494 .handle_control_frame(
10495 &ctx,
10496 route_open_frame(304, "warming", unique_project_root("warming")),
10497 )
10498 .await
10499 .unwrap();
10500 module.stop().await.unwrap();
10501
10502 assert_eq!(response[0].header.ty, FrameType::Error);
10503 let error = parse_error(&response[0]);
10504 assert_eq!(error["code"], "module_warming");
10505 assert!(error["message"]
10506 .as_str()
10507 .unwrap()
10508 .contains("state=running, enabled=true, live=false"));
10509 }
10510
10511 #[test]
10512 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
10513 let handler = ControlHandler::new(Arc::new(Registry::default()));
10514 let capture = EventCapture::default();
10515 let _subscriber =
10516 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10517 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
10518 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
10519 let pending = (0..limit).collect::<Vec<_>>();
10520 let response = handler
10521 .route_open_capacity_refusal(
10522 &ctx,
10523 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
10524 "busy",
10525 pending.len(),
10526 limit,
10527 )
10528 .unwrap();
10529 assert_eq!(parse_error(&response)["code"], "target_unavailable");
10530 let event = capture
10531 .events()
10532 .into_iter()
10533 .find(|event| {
10534 event.target == "control"
10535 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
10536 })
10537 .expect("connection admission refusal event");
10538 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
10539 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
10540 }
10541
10542 #[test]
10543 fn route_open_target_cap_logs_admission_reason_and_capacity() {
10544 let handler = ControlHandler::new(Arc::new(Registry::default()));
10545 let capture = EventCapture::default();
10546 let _subscriber =
10547 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10548 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
10549 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
10550 let guards = (0..limit)
10551 .map(|_| {
10552 handler
10553 .route_bind_concurrency
10554 .try_admit("busy", limit)
10555 .unwrap()
10556 })
10557 .collect::<Vec<_>>();
10558 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
10559 Err(in_flight) => in_flight,
10560 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
10561 };
10562 let response = handler
10563 .route_open_target_capacity_refusal(
10564 &ctx,
10565 &route_open_frame(397, "busy", unique_project_root("target-cap")),
10566 "busy",
10567 in_flight,
10568 )
10569 .unwrap();
10570 assert_eq!(parse_error(&response)["code"], "target_unavailable");
10571 let event = capture
10572 .events()
10573 .into_iter()
10574 .find(|event| {
10575 event.target == "control"
10576 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
10577 })
10578 .expect("target admission refusal event");
10579 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
10580 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
10581 drop(guards);
10582 }
10583
10584 #[tokio::test(flavor = "current_thread")]
10585 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
10586 let handler = ControlHandler::new(Arc::new(Registry::default()));
10587 let capture = EventCapture::default();
10588 let _subscriber =
10589 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10590 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
10591 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
10592
10593 let _ = handler
10594 .handle_client_control_request(
10595 &ctx,
10596 frame,
10597 ClientControlRequest::SupervisorSetEnabled {
10598 module_id: "broca".to_string(),
10599 enabled: false,
10600 },
10601 )
10602 .await
10603 .unwrap();
10604
10605 let events = capture
10606 .events()
10607 .into_iter()
10608 .filter(|event| {
10609 event.target == "control"
10610 && event.fields.get("message").map(String::as_str)
10611 == Some("supervisor request received")
10612 })
10613 .collect::<Vec<_>>();
10614 assert_eq!(events.len(), 1, "one supervisor request log line");
10615 let event = &events[0];
10616 assert_eq!(event.level, tracing::Level::INFO);
10617 assert_eq!(
10618 event.fields.get("op"),
10619 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
10620 );
10621 assert_eq!(
10622 event.fields.get("module_id"),
10623 Some(&"\"broca\"".to_string())
10624 );
10625 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
10626 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
10627 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
10628 }
10629
10630 #[tokio::test(flavor = "current_thread")]
10631 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
10632 let registry = Arc::new(Registry::default());
10633 let module_connection = ConnectionId::new(303);
10634 registry
10635 .register_with_control_ops(
10636 manifest("aft", PROTOCOL_VERSION),
10637 PROTOCOL_VERSION,
10638 module_connection,
10639 module_baseline_control_ops(),
10640 )
10641 .unwrap();
10642 let handler = ControlHandler::new(registry);
10643 let capture = EventCapture::default();
10644 let _subscriber =
10645 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10646 let (ctx, _rx) = route_ctx(module_connection);
10647 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
10648
10649 let _ = handler
10650 .handle_client_control_request(
10651 &ctx,
10652 frame,
10653 ClientControlRequest::SupervisorSetEnabled {
10654 module_id: "broca".to_string(),
10655 enabled: false,
10656 },
10657 )
10658 .await
10659 .unwrap();
10660
10661 let callers = capture
10662 .events()
10663 .into_iter()
10664 .filter(|event| {
10665 event.fields.get("message").map(String::as_str)
10666 == Some("supervisor request received")
10667 })
10668 .map(|event| event.fields.get("caller").cloned())
10669 .collect::<Vec<_>>();
10670 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
10671 }
10672
10673 #[tokio::test(flavor = "current_thread")]
10674 async fn supervisor_list_does_not_log_request_received() {
10675 let handler = ControlHandler::new(Arc::new(Registry::default()));
10676 let capture = EventCapture::default();
10677 let _subscriber =
10678 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10679 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
10680 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
10681
10682 handler
10683 .handle_client_control_request(&ctx, frame, ClientControlRequest::SupervisorList {})
10684 .await
10685 .unwrap();
10686
10687 assert!(capture.events().into_iter().all(|event| {
10688 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
10689 }));
10690 }
10691
10692 #[tokio::test(flavor = "current_thread")]
10693 async fn supervisor_rescan_preview_does_not_log_request_received() {
10694 let handler = ControlHandler::new(Arc::new(Registry::default()));
10695 let capture = EventCapture::default();
10696 let _subscriber =
10697 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10698 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
10699 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
10700
10701 handler
10702 .handle_client_control_request(
10703 &ctx,
10704 frame,
10705 ClientControlRequest::SupervisorRescan { preview: true },
10706 )
10707 .await
10708 .unwrap();
10709
10710 assert!(capture.events().into_iter().all(|event| {
10711 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
10712 }));
10713 }
10714
10715 #[tokio::test]
10720 async fn route_open_refusal_names_the_check_that_refused() {
10721 let handler = ControlHandler::new(Arc::new(Registry::default()));
10722 let capture = EventCapture::default();
10723 let _subscriber =
10724 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10725 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
10726 let response = handler
10727 .handle_control_frame(
10728 &ctx,
10729 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
10730 )
10731 .await
10732 .unwrap();
10733
10734 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10735 let event = capture
10736 .events()
10737 .into_iter()
10738 .find(|event| {
10739 event.target == "control"
10740 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
10741 })
10742 .expect("route.open refusal event");
10743 assert_eq!(
10744 event.fields.get("reason"),
10745 Some(&"\"not_registered\"".to_string())
10746 );
10747 }
10748
10749 #[tokio::test]
10750 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
10751 let registry = Arc::new(Registry::default());
10752 let supervisor_handle = SupervisorHandle::new();
10753 let supervisor =
10754 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
10755 .with_handle(supervisor_handle.clone())
10756 .with_connection_file_path(std::env::temp_dir().join(format!(
10757 "subc-route-open-refusal-info-{}",
10758 std::process::id()
10759 )));
10760 let module = supervisor
10761 .supervise_configured(
10762 ModuleSpec {
10763 module_id: "warming".to_string(),
10764 program: fake_aft_stub_path(),
10765 args: Vec::new(),
10766 env: Vec::new(),
10767 reserved: false,
10768 reserved_prefixes: Vec::new(),
10769 protocol: ModuleProtocol::Subc,
10770 overlap: Default::default(),
10771 },
10772 true,
10773 )
10774 .unwrap();
10775 assert_eq!(module.state().unwrap(), ModuleState::Running);
10776
10777 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10778 assert!(handler
10779 .counters()
10780 .snapshot()
10781 .get("route_open_refused_by_code")
10782 .is_none());
10783 let capture = EventCapture::default();
10784 let _subscriber =
10785 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10786 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
10787 let response = handler
10788 .handle_control_frame(
10789 &ctx,
10790 route_open_frame(394, "warming", unique_project_root("refusal-info")),
10791 )
10792 .await
10793 .unwrap();
10794 module.stop().await.unwrap();
10795
10796 assert_eq!(parse_error(&response[0])["code"], "module_warming");
10797 let event = capture
10798 .events()
10799 .into_iter()
10800 .find(|event| {
10801 event.target == "control"
10802 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
10803 })
10804 .expect("route.open refusal event");
10805 assert_eq!(
10806 event.fields.get("module_id"),
10807 Some(&"\"warming\"".to_string())
10808 );
10809 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
10810 assert_eq!(
10811 event.fields.get("reason"),
10812 Some(&"\"supervised_not_registered\"".to_string())
10813 );
10814 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
10815 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
10816 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
10817 assert_eq!(
10818 handler.counters().snapshot()["route_open_refused_by_code"],
10819 json!({ "module_warming": 1 })
10820 );
10821 }
10822
10823 const OUTAGE_START: &str = "route.open refusing module: not serving";
10824 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
10825
10826 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
10827 capture
10828 .events()
10829 .into_iter()
10830 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
10831 .collect()
10832 }
10833
10834 fn supervise_stub(
10835 registry: &Arc<Registry>,
10836 module_id: &str,
10837 enabled: bool,
10838 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
10839 let supervisor_handle = SupervisorHandle::new();
10840 let supervisor =
10841 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
10842 .with_handle(supervisor_handle.clone())
10843 .with_connection_file_path(std::env::temp_dir().join(format!(
10844 "subc-route-outage-{module_id}-{}",
10845 std::process::id()
10846 )));
10847 let module = supervisor
10848 .supervise_configured(
10849 ModuleSpec {
10850 module_id: module_id.to_string(),
10851 program: fake_aft_stub_path(),
10852 args: Vec::new(),
10853 env: Vec::new(),
10854 reserved: false,
10855 reserved_prefixes: Vec::new(),
10856 protocol: ModuleProtocol::Subc,
10857 overlap: Default::default(),
10858 },
10859 enabled,
10860 )
10861 .unwrap();
10862 (supervisor_handle, module)
10863 }
10864
10865 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
10866 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
10867 module_id: module_id.to_string(),
10868 drain_timeout_ms: Some(50),
10869 })
10870 .unwrap();
10871 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
10872 }
10873
10874 #[test]
10878 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
10879 let registry = Arc::new(Registry::default());
10880 let forwarding = Arc::new(ForwardingTable::default());
10881 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10882 let second = ControlHandler::with_forwarding(registry, forwarding);
10883 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
10884 }
10885
10886 #[tokio::test(flavor = "current_thread")]
10890 async fn route_open_unknown_module_refusals_add_no_outage_state() {
10891 let handler = ControlHandler::new(Arc::new(Registry::default()));
10892 let capture = EventCapture::default();
10893 let _subscriber =
10894 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10895 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
10896 for corr in 0..8 {
10897 let response = handler
10898 .handle_control_frame(
10899 &ctx,
10900 route_open_frame(
10901 380 + corr,
10902 &format!("nobody-{corr}"),
10903 unique_project_root("outage-unknown"),
10904 ),
10905 )
10906 .await
10907 .unwrap();
10908 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10909 }
10910
10911 assert_eq!(handler.route_outages.tracked_module_count(), 0);
10912 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
10913 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
10914 }
10915
10916 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10921 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
10922 let registry = Arc::new(Registry::default());
10923 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
10924 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10925 let capture = EventCapture::default();
10926 let _subscriber =
10927 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10928 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
10929 handler.route_outages.record_accepted("outage-restart");
10932
10933 let response = handler
10934 .handle_control_frame(
10935 &ctx,
10936 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
10937 )
10938 .await
10939 .unwrap();
10940 assert_eq!(response[0].header.ty, FrameType::Error);
10941 let starts = outage_lines(&capture, OUTAGE_START);
10942 assert_eq!(starts.len(), 1, "{starts:?}");
10943 assert_eq!(starts[0].level, tracing::Level::WARN);
10944 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
10945 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
10946 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
10947 handler.route_outages.record_accepted("outage-restart");
10948 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
10949
10950 let restart = handler
10951 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
10952 .await
10953 .unwrap();
10954 assert_eq!(
10955 restart[0].header.ty,
10956 FrameType::Response,
10957 "{:?}",
10958 parse_error(&restart[0])
10959 );
10960 handler
10961 .handle_control_frame(
10962 &ctx,
10963 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
10964 )
10965 .await
10966 .unwrap();
10967 module.stop().await.unwrap();
10968
10969 let starts = outage_lines(&capture, OUTAGE_START);
10970 assert_eq!(starts.len(), 2, "{starts:?}");
10971 assert_eq!(starts[1].level, tracing::Level::INFO);
10972 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
10973 }
10974
10975 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10979 async fn failed_operator_restart_leaves_no_operator_mark() {
10980 let registry = Arc::new(Registry::default());
10981 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
10982 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10983 let capture = EventCapture::default();
10984 let _subscriber =
10985 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10986 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
10987 handler.route_outages.record_accepted("outage-disabled");
10988
10989 let restart = handler
10990 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
10991 .await
10992 .unwrap();
10993 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
10994 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
10995
10996 handler
10997 .handle_control_frame(
10998 &ctx,
10999 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
11000 )
11001 .await
11002 .unwrap();
11003 let starts = outage_lines(&capture, OUTAGE_START);
11004 assert_eq!(starts.len(), 1, "{starts:?}");
11005 assert_eq!(starts[0].level, tracing::Level::WARN);
11006 }
11007
11008 #[tokio::test(flavor = "current_thread")]
11009 async fn route_open_unknown_module_escapes_target_module_id() {
11010 let handler = ControlHandler::new(Arc::new(Registry::default()));
11011 let capture = EventCapture::default();
11012 let _subscriber =
11013 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11014 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
11015 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
11016 let response = handler
11017 .handle_control_frame(
11018 &ctx,
11019 route_open_frame(
11020 395,
11021 hostile_module_id,
11022 unique_project_root("hostile-target-module-id"),
11023 ),
11024 )
11025 .await
11026 .unwrap();
11027
11028 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11029 let event = capture
11030 .events()
11031 .into_iter()
11032 .find(|event| {
11033 event.target == "control"
11034 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
11035 })
11036 .expect("route.open unknown-module refusal event");
11037 let logged = event.fields.get("module_id").expect("module_id field");
11038 assert!(!logged.bytes().any(|byte| byte < 0x20));
11039 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
11040 }
11041
11042 #[tokio::test(flavor = "current_thread")]
11043 async fn route_open_module_rejection_uses_daemon_counter_key() {
11044 let registry = Arc::new(Registry::default());
11045 let forwarding = Arc::new(ForwardingTable::default());
11046 let handler =
11047 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11048 let module_connection = ConnectionId::new(95);
11049 let (module_ctx, mut module_rx) = route_ctx(module_connection);
11050 hello_via_sink(
11051 &handler,
11052 &module_ctx,
11053 &mut module_rx,
11054 hello_frame("aft", PROTOCOL_VERSION, 395),
11055 )
11056 .await;
11057
11058 let client_connection = ConnectionId::new(96);
11059 let (client_ctx, _client_rx) = route_ctx(client_connection);
11060 let capture = EventCapture::default();
11061 let _subscriber =
11062 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11063 let (route_task, bind) = relay_route_open(
11064 &handler,
11065 client_connection,
11066 &client_ctx.egress,
11067 &mut module_rx,
11068 396,
11069 "aft",
11070 "hostile-module-code",
11071 )
11072 .await;
11073 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
11074 let rejection = Frame::build(
11075 FrameType::Error,
11076 control_flags(),
11077 0,
11078 0,
11079 bind.header.corr,
11080 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
11081 )
11082 .unwrap();
11083 handler
11084 .handle_control_frame(&module_ctx, rejection)
11085 .await
11086 .unwrap();
11087
11088 let response = route_task.await.unwrap();
11089 assert_eq!(parse_error(&response[0])["code"], hostile_code);
11090 let counters = handler.counters().snapshot();
11091 assert_eq!(
11092 counters["route_open_refused_by_code"],
11093 json!({ "module_rejected": 1 })
11094 );
11095 assert!(counters["route_open_refused_by_code"]
11096 .get(hostile_code)
11097 .is_none());
11098
11099 let event = capture
11100 .events()
11101 .into_iter()
11102 .find(|event| {
11103 event.target == "control"
11104 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
11105 })
11106 .expect("route.open module-rejection refusal event");
11107 let logged = event.fields.get("module_code").expect("module_code field");
11108 assert!(!logged.bytes().any(|byte| byte < 0x20));
11109 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
11110 }
11111
11112 #[tokio::test]
11113 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
11114 let registry = Arc::new(Registry::default());
11115 let supervisor_handle = SupervisorHandle::new();
11116 let missing_program = std::env::temp_dir().join(format!(
11117 "subc-route-open-missing-program-{}",
11118 std::process::id()
11119 ));
11120 let supervisor =
11121 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11122 .with_handle(supervisor_handle.clone());
11123 let module = supervisor
11124 .supervise_configured(
11125 ModuleSpec {
11126 module_id: "failed".to_string(),
11127 program: missing_program,
11128 args: Vec::new(),
11129 env: Vec::new(),
11130 reserved: false,
11131 reserved_prefixes: Vec::new(),
11132 protocol: ModuleProtocol::Subc,
11133 overlap: Default::default(),
11134 },
11135 true,
11136 )
11137 .unwrap();
11138 assert_eq!(module.state().unwrap(), ModuleState::Failed);
11139
11140 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11141 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
11142 let response = handler
11143 .handle_control_frame(
11144 &ctx,
11145 route_open_frame(305, "failed", unique_project_root("failed")),
11146 )
11147 .await
11148 .unwrap();
11149
11150 assert_eq!(response[0].header.ty, FrameType::Error);
11151 let error = parse_error(&response[0]);
11152 assert_eq!(error["code"], "target_unavailable");
11153 assert!(error["message"]
11154 .as_str()
11155 .unwrap()
11156 .contains("state=failed, enabled=true, live=false"));
11157 }
11158
11159 #[tokio::test]
11160 async fn route_open_role_mismatch_remains_target_unavailable() {
11161 let registry = Arc::new(Registry::default());
11162 let handler = ControlHandler::new(Arc::clone(®istry));
11163 handler
11164 .handle_control(
11165 ConnectionId::new(41),
11166 non_routable_hello_frame_with_control_ops("health-only", 306, None),
11167 )
11168 .unwrap();
11169
11170 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
11171 let response = handler
11172 .handle_control_frame(
11173 &ctx,
11174 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
11175 )
11176 .await
11177 .unwrap();
11178
11179 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11180 assert!(parse_error(&response[0])["message"]
11181 .as_str()
11182 .unwrap()
11183 .contains("does not provide the requested target"));
11184 }
11185
11186 #[tokio::test]
11187 async fn route_open_inactive_registration_remains_target_unavailable() {
11188 let registry = Arc::new(Registry::default());
11189 let handler = ControlHandler::new(Arc::clone(®istry));
11190 handler
11191 .handle_control(
11192 ConnectionId::new(43),
11193 hello_frame("inactive", PROTOCOL_VERSION, 308),
11194 )
11195 .unwrap();
11196 assert!(registry
11197 .set_module_state_for_test("inactive", ChannelState::Closed)
11198 .unwrap());
11199
11200 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
11201 let response = handler
11202 .handle_control_frame(
11203 &ctx,
11204 route_open_frame(309, "inactive", unique_project_root("inactive")),
11205 )
11206 .await
11207 .unwrap();
11208
11209 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11210 assert!(parse_error(&response[0])["message"]
11211 .as_str()
11212 .unwrap()
11213 .contains("is not active"));
11214 }
11215
11216 #[tokio::test]
11217 async fn late_health_reply_is_recorded_through_the_module_response_path() {
11218 let registry = Arc::new(Registry::default());
11219 let forwarding = Arc::new(ForwardingTable::default());
11220 let supervisor_handle = SupervisorHandle::new();
11221 let supervisor =
11222 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
11223 .with_forwarding(Arc::clone(&forwarding))
11224 .with_handle(supervisor_handle.clone());
11225 let module = supervisor
11226 .supervise_configured(
11227 crate::ModuleSpec {
11228 module_id: "late-health-response".to_string(),
11229 program: PathBuf::from("disabled-module"),
11230 args: Vec::new(),
11231 env: Vec::new(),
11232 reserved: false,
11233 reserved_prefixes: Vec::new(),
11234 protocol: ModuleProtocol::Subc,
11235 overlap: Default::default(),
11236 },
11237 false,
11238 )
11239 .unwrap();
11240 let handler =
11241 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11242 .with_supervisor(supervisor_handle);
11243 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
11244 handler
11245 .handle_control_frame(
11246 &module_ctx,
11247 hello_frame_with_control_ops(
11248 "late-health-response",
11249 PROTOCOL_VERSION,
11250 7,
11251 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11252 ),
11253 )
11254 .await
11255 .unwrap();
11256 let probe_started_at = Instant::now() - Duration::from_millis(80);
11257 let pending = forwarding
11258 .begin_health_probe_rpc_for(
11259 "late-health-response",
11260 MODULE_CONTROL_OP_HEALTH_CHECK,
11261 probe_started_at,
11262 Instant::now() - Duration::from_millis(1),
11263 )
11264 .unwrap();
11265 assert!(forwarding
11266 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
11267 .unwrap());
11268
11269 let responses = handler
11270 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
11271 .await
11272 .unwrap();
11273
11274 assert!(responses.is_empty());
11275 let health = module.status().unwrap().health;
11276 assert_eq!(health.late_answer_count, 1);
11277 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
11278 }
11279
11280 #[tokio::test]
11281 async fn health_probe_timeout_and_module_death_are_typed() {
11282 let registry = Arc::new(Registry::default());
11283 let forwarding = Arc::new(ForwardingTable::default());
11284 let handler =
11285 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11286 .with_health_probe_timeout(Duration::from_millis(50));
11287 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11288 hello_via_sink(
11289 &handler,
11290 &module_ctx,
11291 &mut module_rx,
11292 hello_frame_with_control_ops(
11293 "aft",
11294 PROTOCOL_VERSION,
11295 7,
11296 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11297 ),
11298 )
11299 .await;
11300
11301 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11302 let responses = handler
11303 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
11304 .await
11305 .unwrap();
11306 assert_eq!(responses[0].header.ty, FrameType::Error);
11307 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
11308 let _ = module_rx.try_recv();
11309
11310 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
11311 let health_handler = handler.clone();
11312 let death_task = tokio::spawn(async move {
11313 health_handler
11314 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
11315 .await
11316 .unwrap()
11317 });
11318 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11319 .await
11320 .unwrap()
11321 .unwrap();
11322 handler
11323 .cleanup_connection(module_ctx.connection_id)
11324 .unwrap();
11325 let responses = death_task.await.unwrap();
11326 assert_eq!(responses[0].header.ty, FrameType::Error);
11327 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
11328 }
11329
11330 #[test]
11331 fn hello_requires_exact_protocol_version() {
11332 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
11333 let registry = Arc::new(Registry::default());
11334 let handler = ControlHandler::new(Arc::clone(®istry));
11335 let responses = handler
11336 .handle_control(
11337 ConnectionId::new(connection),
11338 hello_frame("aft", offered, 9),
11339 )
11340 .unwrap();
11341
11342 assert_eq!(responses.len(), 1);
11343 assert_eq!(responses[0].header.ty, FrameType::Error);
11344 let error = parse_error(&responses[0]);
11345 assert_eq!(error["code"], "version_unsupported");
11346 assert!(registry.get_module("aft").unwrap().is_none());
11347 assert_eq!(registry.active_registration_count().unwrap(), 0);
11348 }
11349 }
11350
11351 #[test]
11352 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
11353 let registry = Arc::new(Registry::default());
11354 let forwarding = Arc::new(ForwardingTable::default());
11355 let handler =
11356 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11357 let module_connection = ConnectionId::new(301);
11358 let registration = registry
11359 .register_with_control_ops(
11360 manifest("aft-push", PROTOCOL_VERSION),
11361 PROTOCOL_VERSION,
11362 module_connection,
11363 module_baseline_control_ops(),
11364 )
11365 .unwrap();
11366 let (module_tx, _module_rx) = mpsc::channel(8);
11367 let endpoint = forwarding
11368 .register_module_connection(
11369 module_connection,
11370 "aft-push".to_string(),
11371 PROTOCOL_VERSION,
11372 manifest_concurrency(®istration.manifest),
11373 FrameSink::new(module_tx),
11374 )
11375 .unwrap();
11376
11377 let unknown = Frame::build(
11379 FrameType::Push,
11380 control_flags(),
11381 0,
11382 0,
11383 5,
11384 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
11385 )
11386 .unwrap();
11387 let out = handler.handle_status_update(endpoint, unknown).unwrap();
11388 assert!(
11389 out.is_empty(),
11390 "unknown push op must be ignored, got {out:?}"
11391 );
11392
11393 let malformed = Frame::build(
11395 FrameType::Push,
11396 control_flags(),
11397 0,
11398 0,
11399 6,
11400 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
11401 )
11402 .unwrap();
11403 let out = handler.handle_status_update(endpoint, malformed).unwrap();
11404 assert_eq!(out.len(), 1);
11405 assert_eq!(out[0].header.ty, FrameType::Error);
11406 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
11407 }
11408
11409 #[test]
11410 fn hello_rejected_when_connection_already_owns_client_routes() {
11411 let registry = Arc::new(Registry::default());
11412 let forwarding = Arc::new(ForwardingTable::default());
11413 let handler =
11414 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11415 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
11417 let client_connection = ConnectionId::new(202);
11418
11419 let responses = handler
11422 .handle_control(
11423 client_connection,
11424 hello_frame("aft-second", PROTOCOL_VERSION, 9),
11425 )
11426 .unwrap();
11427 assert_eq!(responses[0].header.ty, FrameType::Error);
11428 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
11429 assert!(registry.get_module("aft-second").unwrap().is_none());
11430 }
11431
11432 #[tokio::test]
11433 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
11434 let registry = Arc::new(Registry::default());
11435 let forwarding = Arc::new(ForwardingTable::default());
11436 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
11437 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
11438 hello_via_sink(
11439 &handler,
11440 &module_ctx,
11441 &mut module_rx,
11442 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
11443 )
11444 .await;
11445 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
11446 let pending = forwarding
11447 .begin_route_bind_relay_for_test(
11448 client_ctx.connection_id,
11449 client_ctx.egress.clone(),
11450 2,
11451 "alpha",
11452 )
11453 .unwrap();
11454 forwarding
11455 .complete_pending_relay(
11456 module_ctx.connection_id,
11457 pending.corr,
11458 RouteBindRelayOutcome::Accepted,
11459 )
11460 .unwrap();
11461 client_rx.try_recv().unwrap();
11462 for module_id in ["beta", "alpha"] {
11463 let replies = handler
11464 .handle_control_frame(
11465 &module_ctx,
11466 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
11467 )
11468 .await
11469 .unwrap();
11470 assert_eq!(replies.len(), 1, "second HELLO must be refused");
11471 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
11472 }
11473 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
11474 assert!(registry.get_module("beta").unwrap().is_none());
11475 assert!(matches!(
11476 forwarding
11477 .lookup_data_route(
11478 client_ctx.connection_id,
11479 pending.client_channel,
11480 pending.client_epoch,
11481 )
11482 .unwrap(),
11483 DataRoute::Client(DataRouteState::Bound(_))
11484 ));
11485 assert!(handler
11486 .hello_launch_nonces
11487 .lock()
11488 .unwrap()
11489 .presented(module_ctx.connection_id, Some("alpha-nonce")));
11490 assert!(module_rx.try_recv().is_err());
11491 }
11492
11493 #[test]
11494 fn reserved_module_hello_requires_matching_launch_nonce() {
11495 let registry = Arc::new(Registry::default());
11496 let supervisor = SupervisorHandle::new();
11497 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
11500 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11501
11502 let no_nonce = handler
11504 .handle_control(
11505 ConnectionId::new(1),
11506 hello_frame("vault", PROTOCOL_VERSION, 1),
11507 )
11508 .unwrap();
11509 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
11510 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
11511 assert!(registry.get_module("vault").unwrap().is_none());
11512
11513 let wrong = handler
11515 .handle_control(
11516 ConnectionId::new(2),
11517 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
11518 )
11519 .unwrap();
11520 assert_eq!(wrong[0].header.ty, FrameType::Error);
11521 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
11522 assert!(registry.get_module("vault").unwrap().is_none());
11523
11524 let ok = handler
11526 .handle_control(
11527 ConnectionId::new(3),
11528 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
11529 )
11530 .unwrap();
11531 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
11532 assert!(registry.get_module("vault").unwrap().is_some());
11533 }
11534
11535 #[test]
11536 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
11537 let registry = Arc::new(Registry::default());
11538 let supervisor = SupervisorHandle::new();
11539 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
11540 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
11541 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11542
11543 let squat = handler
11544 .handle_control(
11545 ConnectionId::new(1),
11546 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
11547 )
11548 .unwrap();
11549 assert_eq!(squat[0].header.ty, FrameType::Error);
11550 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
11551 assert!(parse_error(&squat[0])["message"]
11552 .as_str()
11553 .unwrap()
11554 .contains("fed:"));
11555
11556 let accepted_peer = handler
11557 .handle_control(
11558 ConnectionId::new(2),
11559 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
11560 )
11561 .unwrap();
11562 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
11563
11564 let accepted_short = handler
11565 .handle_control(
11566 ConnectionId::new(3),
11567 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
11568 )
11569 .unwrap();
11570 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
11571
11572 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
11573 let response = handler
11574 .handle_control(
11575 ConnectionId::new(conn),
11576 hello_frame(module_id, PROTOCOL_VERSION, conn),
11577 )
11578 .unwrap();
11579 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
11580 }
11581 }
11582
11583 #[test]
11584 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
11585 let registry = Arc::new(Registry::default());
11586 let supervisor = SupervisorHandle::new();
11587 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
11588 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
11589 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
11590 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11591
11592 let owner_nonce = handler
11593 .handle_control(
11594 ConnectionId::new(1),
11595 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
11596 )
11597 .unwrap();
11598 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
11599 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
11600 assert!(registry.get_module("fed:special").unwrap().is_none());
11601
11602 let exact_nonce = handler
11603 .handle_control(
11604 ConnectionId::new(2),
11605 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
11606 )
11607 .unwrap();
11608 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
11609 assert!(registry.get_module("fed:special").unwrap().is_some());
11610 }
11611
11612 #[test]
11613 fn non_reserved_module_ignores_launch_nonce() {
11614 let registry = Arc::new(Registry::default());
11615 let handler = ControlHandler::new(Arc::clone(®istry));
11618 let no_nonce = handler
11619 .handle_control(
11620 ConnectionId::new(1),
11621 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
11622 )
11623 .unwrap();
11624 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
11625 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
11626
11627 let echoed_nonce = handler
11628 .handle_control(
11629 ConnectionId::new(2),
11630 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
11631 )
11632 .unwrap();
11633 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
11634 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
11635 }
11636
11637 #[test]
11638 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
11639 let handler = ControlHandler::default();
11640 let conn = ConnectionId::new(1);
11641 let malformed = Frame::build(
11642 FrameType::Hello,
11643 control_flags(),
11644 0,
11645 0,
11646 3,
11647 b"{not json".to_vec(),
11648 )
11649 .unwrap();
11650
11651 let error = handler.handle_control(conn, malformed).unwrap();
11652 assert_eq!(error[0].header.ty, FrameType::Error);
11653 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
11654
11655 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
11656 let pong = handler.handle_control(conn, ping).unwrap();
11657 assert_eq!(pong[0].header.ty, FrameType::Pong);
11658 assert_eq!(pong[0].header.corr, 4);
11659 }
11660
11661 #[test]
11662 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
11663 let registry = Arc::new(Registry::default());
11664 let handler = ControlHandler::new(Arc::clone(®istry));
11665
11666 handler
11667 .handle_control(
11668 ConnectionId::new(1),
11669 hello_frame("aft", PROTOCOL_VERSION, 1),
11670 )
11671 .unwrap();
11672 let duplicate = handler
11673 .handle_control(
11674 ConnectionId::new(2),
11675 hello_frame("aft", PROTOCOL_VERSION, 2),
11676 )
11677 .unwrap();
11678
11679 assert_eq!(duplicate[0].header.ty, FrameType::Error);
11680 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11681 let registration = registry.get_module("aft").unwrap().unwrap();
11682 assert_eq!(registration.connection_id, ConnectionId::new(1));
11683 }
11684
11685 #[test]
11686 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
11687 let registry = Arc::new(Registry::default());
11688 let forwarding = Arc::new(ForwardingTable::default());
11689 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
11690 let handler =
11691 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11692 .with_process_liveness(process_liveness);
11693 let (ctx, route_channel, route_epoch) =
11694 bind_liveness_route(®istry, &forwarding, "aft-dead");
11695 let responses = handler
11696 .handle_route_poll(
11697 &ctx,
11698 route_poll_frame(41, PollKind::Liveness, route_channel),
11699 route_channel,
11700 route_epoch,
11701 PollKind::Liveness,
11702 )
11703 .unwrap();
11704
11705 assert_eq!(responses.len(), 1);
11706 assert_eq!(responses[0].header.ty, FrameType::Response);
11707 assert_route_poll_liveness(&responses[0], false);
11708 }
11709
11710 #[test]
11711 fn liveness_poll_without_process_source_uses_bound_route() {
11712 let registry = Arc::new(Registry::default());
11713 let forwarding = Arc::new(ForwardingTable::default());
11714 let handler =
11715 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11716 let (ctx, route_channel, route_epoch) =
11717 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
11718 let responses = handler
11719 .handle_route_poll(
11720 &ctx,
11721 route_poll_frame(42, PollKind::Liveness, route_channel),
11722 route_channel,
11723 route_epoch,
11724 PollKind::Liveness,
11725 )
11726 .unwrap();
11727
11728 assert_route_poll_liveness(&responses[0], true);
11729 }
11730
11731 #[test]
11732 fn liveness_poll_untracked_process_source_uses_bound_route() {
11733 let registry = Arc::new(Registry::default());
11734 let forwarding = Arc::new(ForwardingTable::default());
11735 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
11736 let handler =
11737 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11738 .with_process_liveness(process_liveness);
11739 let (ctx, route_channel, route_epoch) =
11740 bind_liveness_route(®istry, &forwarding, "aft-untracked");
11741 let responses = handler
11742 .handle_route_poll(
11743 &ctx,
11744 route_poll_frame(43, PollKind::Liveness, route_channel),
11745 route_channel,
11746 route_epoch,
11747 PollKind::Liveness,
11748 )
11749 .unwrap();
11750
11751 assert_route_poll_liveness(&responses[0], true);
11752 }
11753
11754 #[tokio::test]
11755 async fn unknown_op_returns_unknown_control_op() {
11756 let handler = ControlHandler::default();
11757 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
11758 let request = Frame::build(
11759 FrameType::Request,
11760 control_flags(),
11761 0,
11762 0,
11763 55,
11764 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
11765 )
11766 .unwrap();
11767
11768 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11769
11770 assert_eq!(response.len(), 1);
11771 assert_eq!(response[0].header.ty, FrameType::Error);
11772 assert_eq!(response[0].header.corr, 55);
11773 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
11774 }
11775
11776 #[tokio::test]
11777 async fn supervisor_provenance_rejects_unknown_exact_module() {
11778 let handler = ControlHandler::default();
11779 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
11780 let request = Frame::build(
11781 FrameType::Request,
11782 control_flags(),
11783 0,
11784 0,
11785 57,
11786 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
11787 )
11788 .unwrap();
11789
11790 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11791
11792 assert_eq!(response.len(), 1);
11793 assert_eq!(response[0].header.ty, FrameType::Error);
11794 assert_eq!(response[0].header.corr, 57);
11795 let error = parse_error(&response[0]);
11796 assert_eq!(error["code"], "unknown_module");
11797 assert_eq!(error["message"], "module_id 'missing' is not supervised");
11798 }
11799
11800 #[test]
11801 fn provenance_probe_override_keeps_handler_tests_deterministic() {
11802 let expected = subc_control::RunningImageAgreement::Unavailable {
11803 reason: subc_control::RunningImageUnavailableReason::HashFailed,
11804 };
11805 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
11806 assert_eq!(handler.provenance_probe_override, Some(expected));
11807 }
11808
11809 #[test]
11810 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
11811 let verdict = reload_verdict(
11812 std::path::Path::new("/bin/new"),
11813 Some(std::path::Path::new("/bin/old")),
11814 subc_control::RunningImageAgreement::Unavailable {
11815 reason: subc_control::RunningImageUnavailableReason::HashFailed,
11816 },
11817 );
11818 assert!(matches!(
11819 verdict.path,
11820 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
11821 if configured == std::path::Path::new("/bin/new")
11822 && spawned_from == std::path::Path::new("/bin/old")
11823 ));
11824 }
11825
11826 #[cfg(windows)]
11827 #[test]
11828 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
11829 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
11830 let configured = dir.join("module.exe");
11831 std::fs::write(&configured, b"executable path fixture").unwrap();
11832 let spawned = std::fs::canonicalize(&configured).unwrap();
11833 assert_ne!(
11834 configured, spawned,
11835 "fixture must exercise different path spellings"
11836 );
11837 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
11838 let lower_case = configured.to_string_lossy().to_lowercase();
11839 for spelling in [
11840 &configured,
11841 Path::new(&forward_slashes),
11842 Path::new(&lower_case),
11843 ] {
11844 let verdict = reload_verdict(
11845 spelling,
11846 Some(&spawned),
11847 subc_control::RunningImageAgreement::Unavailable {
11848 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
11849 },
11850 );
11851 assert_eq!(
11852 verdict.path,
11853 ReloadPathAgreement::Match,
11854 "configured {spelling:?}, spawned {spawned:?}"
11855 );
11856 }
11857 }
11858
11859 #[cfg(windows)]
11860 #[test]
11861 fn windows_reload_path_keeps_different_existing_program_pending() {
11862 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
11863 let old = dir.join("old.exe");
11864 let new = dir.join("new.exe");
11865 std::fs::write(&old, b"same executable content").unwrap();
11866 std::fs::copy(&old, &new).unwrap();
11867 let spawned = std::fs::canonicalize(&old).unwrap();
11868 let image = subc_control::RunningImageAgreement::Unavailable {
11869 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
11870 };
11871 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
11872 assert!(
11873 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
11874 if configured == new && spawned_from == spawned)
11875 );
11876 let missing = dir.join("missing.exe");
11877 assert!(matches!(
11878 reload_verdict(&missing, Some(&spawned), image).path,
11879 ReloadPathAgreement::Mismatch { .. }
11880 ));
11881 }
11882
11883 #[test]
11884 fn reload_verdict_detects_replaced_image_at_same_path() {
11885 let image = subc_control::RunningImageAgreement::Mismatch {
11886 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
11887 digest: "old".into(),
11888 },
11889 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
11890 digest: "new".into(),
11891 },
11892 };
11893 let verdict = reload_verdict(
11894 std::path::Path::new("/bin/same"),
11895 Some(std::path::Path::new("/bin/same")),
11896 image.clone(),
11897 );
11898 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11899 assert_eq!(verdict.image, image);
11900 }
11901
11902 #[test]
11903 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
11904 let image = subc_control::RunningImageAgreement::Unavailable {
11905 reason: subc_control::RunningImageUnavailableReason::NotRunning,
11906 };
11907 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
11908 assert_eq!(
11909 verdict.path,
11910 subc_control::ReloadPathAgreement::Unavailable {
11911 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
11912 }
11913 );
11914 assert_eq!(verdict.image, image);
11915
11916 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
11917 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
11918 };
11919 let verdict = reload_verdict(
11920 std::path::Path::new("/bin/same"),
11921 Some(std::path::Path::new("/bin/same")),
11922 unconfirmed.clone(),
11923 );
11924 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11925 assert_eq!(verdict.image, unconfirmed);
11926 }
11927
11928 #[test]
11929 fn reload_verdict_preserves_each_image_unavailability_reason() {
11930 use subc_control::RunningImageUnavailableReason as Reason;
11931
11932 for reason in [
11933 Reason::NotRunning,
11934 Reason::UnsupportedPlatform,
11935 Reason::RunningExecutableUnreadable,
11936 Reason::SpawnedPathUnreadable,
11937 Reason::HashFailed,
11938 Reason::ProcessIdentityUnconfirmed,
11939 Reason::Unknown("future_probe_reason".to_string()),
11940 ] {
11941 let image = subc_control::RunningImageAgreement::Unavailable {
11942 reason: reason.clone(),
11943 };
11944 let verdict = reload_verdict(
11945 std::path::Path::new("/bin/same"),
11946 Some(std::path::Path::new("/bin/same")),
11947 image.clone(),
11948 );
11949 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11950 assert_eq!(verdict.image, image, "{reason:?}");
11951 }
11952 }
11953
11954 #[tokio::test]
11955 async fn malformed_control_bodies_return_invalid_control_body() {
11956 let handler = ControlHandler::default();
11957 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
11958
11959 for (corr, body) in [
11960 (56, br#"{"route_channel":1}"#.as_slice()),
11961 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
11962 (
11963 58,
11964 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
11965 ),
11966 ] {
11967 let request = Frame::build(
11968 FrameType::Request,
11969 control_flags(),
11970 0,
11971 0,
11972 corr,
11973 body.to_vec(),
11974 )
11975 .unwrap();
11976 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11977
11978 assert_eq!(response.len(), 1);
11979 assert_eq!(response[0].header.ty, FrameType::Error);
11980 assert_eq!(response[0].header.corr, corr);
11981 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
11982 }
11983 }
11984
11985 #[tokio::test]
11986 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
11987 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
11988 let registry = Arc::new(Registry::default());
11989 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
11990 let router = Router::with_control_handler(Arc::clone(&control));
11991 let connection = router.begin_connection();
11992 let (ctx, mut rx) = route_ctx(connection.id());
11993
11994 router
11995 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
11996 .await
11997 .unwrap();
11998 let response = rx.recv().await.unwrap();
11999 let ack = parse_ack(&response);
12000 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
12001 let channel = 1;
12002
12003 let goodbye =
12004 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
12005 router.route_for_connection(&ctx, goodbye).await.unwrap();
12006 assert!(rx.try_recv().is_err());
12007 assert!(registry.get_module("aft").unwrap().is_none());
12008
12009 router
12010 .route_for_connection(&ctx, channel_request(channel, 13))
12011 .await
12012 .unwrap();
12013 let error_frame = rx.recv().await.unwrap();
12014 assert_eq!(error_frame.header.ty, FrameType::Error);
12015 assert_eq!(error_frame.header.channel, channel);
12016 let captured = crate::router::test_log::captured_logs(&logs);
12017 assert_eq!(
12018 captured
12019 .lines()
12020 .filter(|line| {
12021 line.contains("module_id=aft")
12022 && line.contains("reason=explicit_goodbye")
12023 && line.contains("module registration ended")
12024 })
12025 .count(),
12026 1,
12027 "unexpected GOODBYE registry log: {captured}"
12028 );
12029 }
12030
12031 #[tokio::test]
12032 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
12033 let registry = Arc::new(Registry::default());
12034 let handler = ControlHandler::new(registry).with_capability_config(
12035 [("prov".to_string(), true), ("cons".to_string(), true)],
12036 BTreeMap::new(),
12037 );
12038 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
12039 register_capability_manifest(
12040 &handler,
12041 &provider_ctx,
12042 &mut provider_rx,
12043 capability_manifest("prov", &["thing/v1"], &[]),
12044 1,
12045 )
12046 .await;
12047 let mut consumer = capability_manifest("cons", &[], &[]);
12048 consumer.capabilities.as_mut().unwrap().requires.push(
12049 subc_protocol::manifest::CapabilityRequirement {
12050 capability: "thing/v1".to_string(),
12051 need: subc_protocol::manifest::CapabilityNeed::Required,
12052 },
12053 );
12054 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
12055 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
12056 assert_eq!(
12057 handler.capability_evaluator.verdict("cons", "thing/v1"),
12058 Some(CapabilityVerdict::Provided)
12059 );
12060 let (mut client_rx, _) = open_route_for_capability_test(
12061 &handler,
12062 &provider_ctx,
12063 &mut provider_rx,
12064 703,
12065 3,
12066 "prov",
12067 None,
12068 )
12069 .await;
12070 let goodbye =
12071 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
12072 handler
12073 .handle_control_frame(&provider_ctx, goodbye)
12074 .await
12075 .unwrap();
12076 assert_eq!(
12077 handler.capability_evaluator.verdict("cons", "thing/v1"),
12078 Some(CapabilityVerdict::NeverProvided)
12079 );
12080 let closed = client_rx
12081 .try_recv()
12082 .expect("GOODBYE pushes route.closed before route GOODBYE");
12083 assert!(
12084 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
12085 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
12086 );
12087 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
12088 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
12089 }
12090
12091 struct RouteClosedFixture {
12092 handler: ControlHandler,
12093 #[cfg(unix)]
12094 supervisor: Supervisor,
12095 module_ctx: RouteCtx,
12096 #[cfg(unix)]
12097 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12098 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12099 route: Arc<crate::forwarding::RouteBinding>,
12100 }
12101
12102 async fn route_closed_fixture() -> RouteClosedFixture {
12103 let registry = Arc::new(Registry::default());
12104 let handler = ControlHandler::new(Arc::clone(®istry));
12105 #[cfg(unix)]
12106 let supervisor = Supervisor::new(registry, RestartPolicy::default())
12107 .with_forwarding(Arc::clone(&handler.forwarding));
12108 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
12109 register_capability_manifest(
12110 &handler,
12111 &module_ctx,
12112 &mut module_rx,
12113 capability_manifest("prov", &[], &[]),
12114 1,
12115 )
12116 .await;
12117 let (client_rx, bind) = open_route_for_capability_test(
12118 &handler,
12119 &module_ctx,
12120 &mut module_rx,
12121 702,
12122 2,
12123 "prov",
12124 None,
12125 )
12126 .await;
12127 let ModuleControlRequest::RouteBind {
12128 route_channel: channel,
12129 epoch: route_epoch,
12130 ..
12131 } = bind
12132 else {
12133 panic!("expected route.bind");
12134 };
12135 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
12136 handler
12137 .forwarding
12138 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
12139 .unwrap()
12140 else {
12141 panic!("the acknowledged route must be bound");
12142 };
12143 RouteClosedFixture {
12144 handler,
12145 #[cfg(unix)]
12146 supervisor,
12147 module_ctx,
12148 #[cfg(unix)]
12149 module_rx,
12150 client_rx,
12151 route,
12152 }
12153 }
12154
12155 impl RouteClosedFixture {
12156 #[cfg(unix)]
12157 async fn cut(&mut self) {
12158 let notices = async {
12159 let mut draining = self.module_rx.recv().await.unwrap();
12160 assert!(matches!(
12161 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
12162 &draining.body
12163 )
12164 .unwrap(),
12165 subc_protocol::session::ModuleControlCommand::Draining {
12166 reason: RouteCloseReason::Restart,
12167 ..
12168 }
12169 ));
12170 draining.flushed.take().unwrap().send(()).unwrap();
12171 let mut closing = self.client_rx.recv().await.unwrap();
12172 assert!(matches!(
12173 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
12174 ClientControlPush::RouteClosing {
12175 reason: RouteCloseReason::Restart,
12176 ..
12177 }
12178 ));
12179 closing.flushed.take().unwrap().send(()).unwrap();
12180 };
12181 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
12182 result.unwrap();
12183 }
12184
12185 fn disconnect(&mut self) -> ClientControlPush {
12186 self.handler
12187 .cleanup_connection(self.module_ctx.connection_id)
12188 .unwrap();
12189 let closed = self.client_rx.try_recv().unwrap();
12190 assert_eq!(closed.header.ty, FrameType::Push);
12191 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
12192 assert!(matches!(
12193 &push,
12194 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
12195 if channels == &[self.route.client_channel]
12196 ));
12197 assert_eq!(
12198 self.client_rx.try_recv().unwrap().header.ty,
12199 FrameType::Goodbye
12200 );
12201 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
12202 push
12203 }
12204 }
12205
12206 #[cfg(unix)]
12207 #[tokio::test(start_paused = true)]
12208 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
12209 let mut fixture = route_closed_fixture().await;
12210 fixture.cut().await;
12211 assert!(matches!(
12212 fixture.disconnect(),
12213 ClientControlPush::RouteClosed {
12214 reason: RouteCloseReason::Restart,
12215 drained: true,
12216 ..
12217 }
12218 ));
12219 }
12220
12221 #[cfg(unix)]
12222 #[tokio::test(start_paused = true)]
12223 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
12224 let mut fixture = route_closed_fixture().await;
12225 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
12226 let started = tokio::time::Instant::now();
12227 fixture.cut().await;
12228 assert!(started.elapsed() >= Duration::from_secs(2));
12229 assert_eq!(fixture.route.flow.in_flight(), 1);
12230 assert!(matches!(
12231 fixture.disconnect(),
12232 ClientControlPush::RouteClosed {
12233 reason: RouteCloseReason::Restart,
12234 drained: false,
12235 ..
12236 }
12237 ));
12238 }
12239
12240 #[tokio::test]
12241 async fn module_connection_crash_reports_crash_and_not_drained() {
12242 let mut fixture = route_closed_fixture().await;
12243 assert!(matches!(
12247 fixture.disconnect(),
12248 ClientControlPush::RouteClosed {
12249 reason: RouteCloseReason::Crash,
12250 drained: false,
12251 ..
12252 }
12253 ));
12254 }
12255
12256 #[tokio::test]
12257 async fn dropping_router_connection_releases_registration() {
12258 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12259 let registry = Arc::new(Registry::default());
12260 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
12261 let router = Router::with_control_handler(Arc::clone(&control));
12262 let connection = router.begin_connection();
12263 let connection_id = connection.id();
12264 let (ctx, mut rx) = route_ctx(connection_id);
12265
12266 router
12267 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
12268 .await
12269 .unwrap();
12270 let response = rx.recv().await.unwrap();
12271 let ack = parse_ack(&response);
12272 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
12273 assert!(registry.get_module("aft").unwrap().is_some());
12274
12275 drop(connection);
12276
12277 assert!(registry.get_module("aft").unwrap().is_none());
12278 assert_eq!(registry.active_registration_count().unwrap(), 0);
12279
12280 control
12281 .cleanup_connection(ConnectionId::new(u64::MAX))
12282 .unwrap();
12283 let captured = crate::router::test_log::captured_logs(&logs);
12284 println!("captured registry lifecycle logs:\n{captured}");
12285 let events: Vec<_> = captured
12286 .lines()
12287 .filter(|line| {
12288 line.contains("module registered module_id=aft ")
12289 || line.contains("module registration ended")
12290 })
12291 .collect();
12292 assert_eq!(
12293 events.len(),
12294 2,
12295 "unexpected registry lifecycle logs: {captured}"
12296 );
12297 assert!(events[0].contains("module registered module_id=aft "));
12298 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
12299 assert!(events[1].contains(&format!(
12300 "module_id=aft connection_id={}",
12301 connection_id.get()
12302 )));
12303 assert!(events[1].contains("reason=connection_closed"));
12304 assert!(events[1].contains("module registration ended"));
12305 assert_eq!(
12306 captured
12307 .lines()
12308 .filter(|line| line.contains("module registered module_id=aft "))
12309 .count(),
12310 1,
12311 "legacy registration admission line must appear once: {captured}"
12312 );
12313 }
12314
12315 #[tokio::test]
12316 async fn hello_registration_keeps_legacy_module_registered_line_once() {
12317 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12318 let registry = Arc::new(Registry::default());
12319 let control = ControlHandler::new(Arc::clone(®istry));
12320 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
12321 hello_via_sink(
12322 &control,
12323 &ctx,
12324 &mut rx,
12325 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
12326 )
12327 .await;
12328
12329 let captured = crate::router::test_log::captured_logs(&logs);
12330 let admissions: Vec<_> = captured
12331 .lines()
12332 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
12333 .collect();
12334 assert_eq!(
12335 admissions.len(),
12336 1,
12337 "legacy admission line must remain exactly once: {captured}"
12338 );
12339 assert!(admissions[0].contains("routable_provider=true"));
12340 assert!(admissions[0].contains("connection_id=777"));
12341 }
12342
12343 fn capability_manifest(
12344 module_id: &str,
12345 provides: &[&str],
12346 must_never_reach: &[&str],
12347 ) -> ModuleManifest {
12348 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
12349 manifest.capabilities = Some(CapabilityDeclarations {
12350 provides: provides
12351 .iter()
12352 .map(|capability| (*capability).to_string())
12353 .collect(),
12354 requires: Vec::new(),
12355 must_never_reach: must_never_reach
12356 .iter()
12357 .map(|capability| (*capability).to_string())
12358 .collect(),
12359 });
12360 manifest
12361 }
12362
12363 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
12364 Frame::build(
12365 FrameType::Hello,
12366 control_flags(),
12367 0,
12368 0,
12369 corr,
12370 serde_json::to_vec(&ModuleHelloBody {
12371 protocol_ver: manifest.protocol_ver,
12372 manifest,
12373 control_ops: None,
12374 launch_nonce: None,
12375 })
12376 .expect("capability test HELLO serializes"),
12377 )
12378 .expect("capability test HELLO frame builds")
12379 }
12380
12381 fn catalog_update_with_capabilities_frame(
12382 corr: u64,
12383 capabilities: CapabilityDeclarations,
12384 ) -> Frame {
12385 Frame::build(
12386 FrameType::Request,
12387 control_flags(),
12388 0,
12389 0,
12390 corr,
12391 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
12392 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
12393 capabilities: Some(capabilities),
12394 ready: None,
12395 })
12396 .expect("capability catalog.update serializes"),
12397 )
12398 .expect("capability catalog.update frame builds")
12399 }
12400
12401 async fn register_capability_manifest(
12402 handler: &ControlHandler,
12403 ctx: &RouteCtx,
12404 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
12405 manifest: ModuleManifest,
12406 corr: u64,
12407 ) {
12408 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
12409 }
12410
12411 async fn open_route_for_capability_test(
12412 handler: &ControlHandler,
12413 target_ctx: &RouteCtx,
12414 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
12415 client_connection_id: u64,
12416 corr: u64,
12417 target_module_id: &str,
12418 consumer_identity: Option<ConsumerIdentity>,
12419 ) -> (
12420 mpsc::Receiver<crate::router::OutboundFrame>,
12421 ModuleControlRequest,
12422 ) {
12423 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
12424 let route_handler = handler.clone();
12425 let target_module_id = target_module_id.to_string();
12426 let route_task = tokio::spawn(async move {
12427 route_handler
12428 .handle_control_frame(
12429 &client_ctx,
12430 route_open_frame_with_admission_facts(
12431 corr,
12432 &target_module_id,
12433 unique_project_root("admission-facts"),
12434 consumer_identity,
12435 None,
12436 ),
12437 )
12438 .await
12439 .expect("capability test route.open succeeds")
12440 });
12441 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
12442 .await
12443 .expect("capability test route.open must reach route.bind")
12444 .expect("target control receiver stays open");
12445 let bind_request: ModuleControlRequest =
12446 serde_json::from_slice(&bind.body).expect("route.bind decodes");
12447 handler
12448 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
12449 .await
12450 .expect("capability test route.bind ACK succeeds");
12451 assert!(route_task.await.expect("route.open task joins").is_empty());
12452 let opened = client_rx
12453 .recv()
12454 .await
12455 .expect("successful route.open publishes a response");
12456 assert!(matches!(
12457 serde_json::from_slice::<ClientControlResponse>(&opened.body),
12458 Ok(ClientControlResponse::RouteOpen { .. })
12459 ));
12460 (client_rx, bind_request)
12461 }
12462
12463 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
12464 assert_eq!(frame.header.ty, FrameType::Push);
12465 assert_eq!(frame.header.channel, 0);
12466 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
12467 .expect("route.closed control push decodes");
12468 let ClientControlPush::RouteClosed { channels, .. } = &push else {
12469 panic!("expected route.closed");
12470 };
12471 assert_eq!(channels.len(), 1, "exactly one violating route closed");
12472 let channels = channels.clone();
12473 assert_eq!(
12474 push,
12475 ClientControlPush::RouteClosed {
12476 module_id: target_module_id.to_string(),
12477 channels,
12478 reason: RouteCloseReason::CapabilityDenied,
12479 drained: false,
12480 abandoned: 0,
12481 excluded_subscriptions: 0,
12482 terminal: Some(false),
12483 }
12484 );
12485 }
12486
12487 #[tokio::test]
12488 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
12489 let registry = Arc::new(Registry::default());
12490 let forwarding = Arc::new(ForwardingTable::default());
12491 let supervisor = SupervisorHandle::new();
12492 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12493 let handler =
12494 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12495 .with_supervisor(supervisor);
12496 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
12497 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
12498 register_capability_manifest(
12499 &handler,
12500 &target_ctx,
12501 &mut target_rx,
12502 capability_manifest("target", &["credentials-provider/v1"], &[]),
12503 1,
12504 )
12505 .await;
12506 register_capability_manifest(
12507 &handler,
12508 &opener_ctx,
12509 &mut opener_rx,
12510 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12511 2,
12512 )
12513 .await;
12514
12515 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
12516 let replies = handler
12517 .handle_control_frame(
12518 &client_ctx,
12519 route_open_frame_with_admission_facts(
12520 3,
12521 "target",
12522 unique_project_root("admission-facts"),
12523 Some(ConsumerIdentity {
12524 module_id: "opener".to_string(),
12525 launch_nonce: "opener-nonce".to_string(),
12526 }),
12527 None,
12528 ),
12529 )
12530 .await
12531 .expect("denied route.open returns a typed frame");
12532 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
12533 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12534 assert!(
12535 target_rx.try_recv().is_err(),
12536 "forbidden route.open must not relay route.bind"
12537 );
12538 }
12539
12540 #[tokio::test]
12541 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
12542 let registry = Arc::new(Registry::default());
12543 let forwarding = Arc::new(ForwardingTable::default());
12544 let supervisor = SupervisorHandle::new();
12545 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12546 let handler =
12547 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12548 .with_supervisor(supervisor);
12549 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
12550 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
12551 register_capability_manifest(
12552 &handler,
12553 &target_ctx,
12554 &mut target_rx,
12555 capability_manifest("target", &["credentials-provider/v1"], &[]),
12556 1,
12557 )
12558 .await;
12559 register_capability_manifest(
12560 &handler,
12561 &old_opener_ctx,
12562 &mut old_opener_rx,
12563 capability_manifest("opener", &[], &[]),
12564 2,
12565 )
12566 .await;
12567 let (mut client_rx, _) = open_route_for_capability_test(
12568 &handler,
12569 &target_ctx,
12570 &mut target_rx,
12571 712,
12572 3,
12573 "target",
12574 Some(ConsumerIdentity {
12575 module_id: "opener".to_string(),
12576 launch_nonce: "opener-nonce".to_string(),
12577 }),
12578 )
12579 .await;
12580 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12581
12582 handler
12583 .cleanup_connection(old_opener_ctx.connection_id)
12584 .expect("old opener registration cleans up");
12585 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
12586 register_capability_manifest(
12587 &handler,
12588 &new_opener_ctx,
12589 &mut new_opener_rx,
12590 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12591 4,
12592 )
12593 .await;
12594
12595 assert_capability_denied_push(
12596 client_rx
12597 .try_recv()
12598 .expect("HELLO deny addition must emit route.closed")
12599 .frame,
12600 "target",
12601 );
12602 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12603 assert!(matches!(
12604 target_rx.try_recv(),
12605 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
12606 ));
12607 }
12608
12609 #[tokio::test]
12610 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
12611 let registry = Arc::new(Registry::default());
12612 let forwarding = Arc::new(ForwardingTable::default());
12613 let supervisor = SupervisorHandle::new();
12614 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12615 let handler =
12616 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12617 .with_supervisor(supervisor);
12618 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
12619 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
12620 register_capability_manifest(
12621 &handler,
12622 &target_ctx,
12623 &mut target_rx,
12624 capability_manifest("target", &[], &[]),
12625 1,
12626 )
12627 .await;
12628 register_capability_manifest(
12629 &handler,
12630 &opener_ctx,
12631 &mut opener_rx,
12632 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12633 2,
12634 )
12635 .await;
12636 let (mut client_rx, _) = open_route_for_capability_test(
12637 &handler,
12638 &target_ctx,
12639 &mut target_rx,
12640 722,
12641 3,
12642 "target",
12643 Some(ConsumerIdentity {
12644 module_id: "opener".to_string(),
12645 launch_nonce: "opener-nonce".to_string(),
12646 }),
12647 )
12648 .await;
12649 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12650
12651 let replies = handler
12652 .handle_control_frame(
12653 &target_ctx,
12654 catalog_update_with_capabilities_frame(
12655 4,
12656 CapabilityDeclarations {
12657 provides: vec!["credentials-provider/v1".to_string()],
12658 requires: Vec::new(),
12659 must_never_reach: Vec::new(),
12660 },
12661 ),
12662 )
12663 .await
12664 .expect("claim catalog.update succeeds");
12665 assert!(matches!(
12666 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
12667 Ok(ModuleControlResponseToModule::CatalogUpdate {})
12668 ));
12669 assert_capability_denied_push(
12670 client_rx
12671 .try_recv()
12672 .expect("claim addition must emit route.closed")
12673 .frame,
12674 "target",
12675 );
12676 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12677 assert!(matches!(
12678 target_rx.try_recv(),
12679 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
12680 ));
12681 }
12682
12683 #[tokio::test]
12684 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
12685 let registry = Arc::new(Registry::default());
12686 let forwarding = Arc::new(ForwardingTable::default());
12687 let supervisor = SupervisorHandle::new();
12688 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12689 let handler =
12690 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12691 .with_supervisor(supervisor);
12692 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
12693 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
12694 register_capability_manifest(
12695 &handler,
12696 &target_ctx,
12697 &mut target_rx,
12698 capability_manifest("target", &["credentials-provider/v1"], &[]),
12699 1,
12700 )
12701 .await;
12702 register_capability_manifest(
12703 &handler,
12704 &opener_ctx,
12705 &mut opener_rx,
12706 capability_manifest("opener", &[], &[]),
12707 2,
12708 )
12709 .await;
12710 let (mut client_rx, _) = open_route_for_capability_test(
12711 &handler,
12712 &target_ctx,
12713 &mut target_rx,
12714 732,
12715 3,
12716 "target",
12717 Some(ConsumerIdentity {
12718 module_id: "opener".to_string(),
12719 launch_nonce: "opener-nonce".to_string(),
12720 }),
12721 )
12722 .await;
12723 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12724
12725 handler
12726 .handle_control_frame(
12727 &target_ctx,
12728 catalog_update_with_capabilities_frame(
12729 4,
12730 CapabilityDeclarations {
12731 provides: Vec::new(),
12732 requires: Vec::new(),
12733 must_never_reach: Vec::new(),
12734 },
12735 ),
12736 )
12737 .await
12738 .expect("claim removal catalog.update succeeds");
12739 assert_eq!(
12740 forwarding.active_binding_count().unwrap(),
12741 1,
12742 "removing an attested target claim must leave the route census unchanged"
12743 );
12744 assert!(
12745 client_rx.try_recv().is_err(),
12746 "claim removal must not emit route.closed capability_denied"
12747 );
12748 assert!(
12749 target_rx.try_recv().is_err(),
12750 "claim removal must not send the target a route GOODBYE"
12751 );
12752 }
12753
12754 #[tokio::test]
12757 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
12758 let registry = Arc::new(Registry::default());
12759 let forwarding = Arc::new(ForwardingTable::default());
12760 let supervisor = SupervisorHandle::new();
12761 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12762 let handler =
12763 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12764 .with_supervisor(supervisor);
12765 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
12766 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
12767 register_capability_manifest(
12768 &handler,
12769 &target_ctx,
12770 &mut target_rx,
12771 capability_manifest("target", &["credentials-provider/v1"], &[]),
12772 1,
12773 )
12774 .await;
12775 register_capability_manifest(
12776 &handler,
12777 &opener_ctx,
12778 &mut opener_rx,
12779 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12780 2,
12781 )
12782 .await;
12783
12784 let (_client_rx, bind) = open_route_for_capability_test(
12785 &handler,
12786 &target_ctx,
12787 &mut target_rx,
12788 742,
12789 3,
12790 "target",
12791 None,
12792 )
12793 .await;
12794 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
12795 panic!("direct scope-honesty route must bind");
12796 };
12797 assert_eq!(principal, Some(Principal::Direct));
12798 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12799 }
12800
12801 #[tokio::test]
12804 async fn must_never_reach_self_route_is_capability_forbidden() {
12805 let registry = Arc::new(Registry::default());
12806 let forwarding = Arc::new(ForwardingTable::default());
12807 let supervisor = SupervisorHandle::new();
12808 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
12809 let handler =
12810 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12811 .with_supervisor(supervisor);
12812 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
12813 register_capability_manifest(
12814 &handler,
12815 &self_ctx,
12816 &mut self_rx,
12817 capability_manifest(
12818 "self-provider",
12819 &["credentials-provider/v1"],
12820 &["credentials-provider/v1"],
12821 ),
12822 1,
12823 )
12824 .await;
12825
12826 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
12827 let replies = handler
12828 .handle_control_frame(
12829 &client_ctx,
12830 route_open_frame_with_admission_facts(
12831 2,
12832 "self-provider",
12833 unique_project_root("admission-facts"),
12834 Some(ConsumerIdentity {
12835 module_id: "self-provider".to_string(),
12836 launch_nonce: "self-nonce".to_string(),
12837 }),
12838 None,
12839 ),
12840 )
12841 .await
12842 .expect("self-route refusal returns a typed frame");
12843 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
12844 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12845 assert!(
12846 self_rx.try_recv().is_err(),
12847 "self denial must not relay route.bind"
12848 );
12849 }
12850
12851 #[test]
12852 fn unsupported_channel_zero_frame_returns_error() {
12853 let handler = ControlHandler::default();
12854 let request = Frame::build(
12855 FrameType::Request,
12856 control_flags(),
12857 0,
12858 0,
12859 21,
12860 b"opaque".to_vec(),
12861 )
12862 .unwrap();
12863
12864 let response = handler
12865 .handle_control(ConnectionId::new(1), request)
12866 .unwrap();
12867
12868 assert_eq!(response[0].header.ty, FrameType::Error);
12869 assert_eq!(
12870 parse_error(&response[0])["code"],
12871 "unsupported_control_frame"
12872 );
12873 }
12874
12875 mod swap {
12880 use super::*;
12881
12882 const INCUMBENT: ConnectionId = ConnectionId::new(30);
12883 const CANDIDATE: ConnectionId = ConnectionId::new(40);
12884
12885 struct Swap {
12886 registry: Arc<Registry>,
12887 forwarding: Arc<ForwardingTable>,
12888 handler: ControlHandler,
12889 incumbent_ctx: RouteCtx,
12890 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12891 candidate_ctx: RouteCtx,
12892 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12893 }
12894
12895 async fn swap_with_incumbent() -> Swap {
12896 let registry = Arc::new(Registry::default());
12897 let forwarding = Arc::new(ForwardingTable::default());
12898 let handler =
12899 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12900 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
12901 hello_via_sink(
12902 &handler,
12903 &incumbent_ctx,
12904 &mut incumbent_rx,
12905 hello_frame("aft", PROTOCOL_VERSION, 7),
12906 )
12907 .await;
12908 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
12909 Swap {
12910 registry,
12911 forwarding,
12912 handler,
12913 incumbent_ctx,
12914 incumbent_rx,
12915 candidate_ctx,
12916 candidate_rx,
12917 }
12918 }
12919
12920 fn register_candidate(swap: &Swap, ready: Option<bool>) {
12921 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
12922 candidate_manifest.ready = ready;
12923 let registration = swap
12924 .registry
12925 .register_candidate_with_control_ops(
12926 candidate_manifest,
12927 PROTOCOL_VERSION,
12928 CANDIDATE,
12929 module_baseline_control_ops(),
12930 )
12931 .unwrap();
12932 swap.forwarding
12933 .register_candidate_module_connection(
12934 CANDIDATE,
12935 "aft".to_string(),
12936 PROTOCOL_VERSION,
12937 manifest_concurrency(®istration.manifest),
12938 swap.candidate_ctx.egress.clone(),
12939 )
12940 .unwrap();
12941 }
12942
12943 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
12944 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
12945 swap.registry.promote_candidate("aft").unwrap().unwrap();
12946 cutover.incumbent.unwrap()
12947 }
12948
12949 fn keyed_total(counters: &Value, key: &str) -> u64 {
12950 counters[key]
12951 .as_object()
12952 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
12953 .unwrap_or(0)
12954 }
12955
12956 #[tokio::test]
12957 async fn replacement_logs_old_end_and_new_admission_once() {
12958 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12959 let mut swap = swap_with_incumbent().await;
12960 swap.handler
12961 .supervisor
12962 .open_swap("aft", "candidate-nonce".to_string());
12963 hello_via_sink(
12964 &swap.handler,
12965 &swap.candidate_ctx,
12966 &mut swap.candidate_rx,
12967 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
12968 )
12969 .await;
12970 cutover(&swap);
12971 swap.handler.cleanup_connection(INCUMBENT).unwrap();
12972
12973 let captured = crate::router::test_log::captured_logs(&logs);
12974 println!("captured replacement registry logs:\n{captured}");
12975 let new_admissions: Vec<_> = captured
12976 .lines()
12977 .filter(|line| {
12978 line.contains("connection_id=40")
12979 && line.contains("swap candidate registered; not routable until cutover")
12980 })
12981 .collect();
12982 assert_eq!(
12983 new_admissions.len(),
12984 1,
12985 "unexpected admission logs: {captured}"
12986 );
12987 assert!(new_admissions[0].contains("module_id=aft"));
12988 assert!(new_admissions[0].contains("connection_id=40"));
12989 assert!(
12990 new_admissions[0].contains("swap candidate registered; not routable until cutover")
12991 );
12992 assert!(new_admissions[0].contains("ready=true"));
12993
12994 let old_admissions: Vec<_> = captured
12995 .lines()
12996 .filter(|line| {
12997 line.contains("module registered module_id=aft ")
12998 && line.contains("connection_id=30")
12999 })
13000 .collect();
13001 assert_eq!(
13002 old_admissions.len(),
13003 1,
13004 "unexpected incumbent admission: {captured}"
13005 );
13006
13007 let old_ends: Vec<_> = captured
13008 .lines()
13009 .filter(|line| {
13010 line.contains("connection_id=30") && line.contains("module registration ended")
13011 })
13012 .collect();
13013 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
13014 assert!(old_ends[0].contains("module_id=aft"));
13015 assert!(old_ends[0].contains("connection_id=30"));
13016 assert!(old_ends[0].contains("reason=replaced"));
13017 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
13018 assert_eq!(
13019 captured
13020 .lines()
13021 .filter(|line| line.contains("module registration promoted"))
13022 .count(),
13023 1,
13024 "unexpected promotion logs: {captured}"
13025 );
13026 }
13027
13028 #[tokio::test]
13034 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
13035 let mut swap = swap_with_incumbent().await;
13036 let handler = swap.handler.clone();
13037
13038 let cotenant = ConnectionId::new(31);
13040 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
13041 let (cotenant_task, cotenant_bind) = relay_route_open(
13042 &handler,
13043 cotenant,
13044 &cotenant_ctx.egress,
13045 &mut swap.incumbent_rx,
13046 100,
13047 "aft",
13048 "swap-cotenant",
13049 )
13050 .await;
13051 handler
13052 .handle_control_frame(
13053 &swap.incumbent_ctx,
13054 route_bind_ack(cotenant_bind.header.corr),
13055 )
13056 .await
13057 .unwrap();
13058 assert!(cotenant_task.await.unwrap().is_empty());
13059 let (cotenant_channel, cotenant_epoch) =
13060 published_route(&cotenant_rx.recv().await.unwrap());
13061
13062 let caller = ConnectionId::new(32);
13064 let (caller_ctx, mut caller_rx) = route_ctx(caller);
13065 let (caller_task, caller_bind) = relay_route_open(
13066 &handler,
13067 caller,
13068 &caller_ctx.egress,
13069 &mut swap.incumbent_rx,
13070 101,
13071 "aft",
13072 "swap-caller",
13073 )
13074 .await;
13075 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
13076
13077 register_candidate(&swap, None);
13078 cutover(&swap);
13079
13080 let ack = handler
13082 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
13083 .await;
13084 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
13085 if module_loop_error.is_some() {
13086 handler.cleanup_connection(INCUMBENT).unwrap();
13089 }
13090
13091 assert!(
13093 cotenant_rx.try_recv().is_err(),
13094 "the co-tenant route on the incumbent was torn down by one late ack: \
13095 {module_loop_error:?}"
13096 );
13097 assert!(matches!(
13098 swap.forwarding
13099 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
13100 .unwrap(),
13101 DataRoute::Client(DataRouteState::Bound(_))
13102 ));
13103 assert_eq!(module_loop_error, None);
13104 assert!(swap
13105 .registry
13106 .get_module_by_connection(INCUMBENT)
13107 .unwrap()
13108 .is_some());
13109
13110 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
13112 .await
13113 .expect("the incumbent is told to drop the abandoned binding")
13114 .unwrap()
13115 .frame;
13116 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13117 assert_eq!(goodbye.header.channel, abandoned_channel);
13118 assert_eq!(goodbye.header.epoch, abandoned_epoch);
13119 assert!(swap.incumbent_rx.try_recv().is_err());
13120
13121 let response = caller_task.await.unwrap();
13123 assert_eq!(response.len(), 1);
13124 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
13125 assert!(caller_rx.try_recv().is_err());
13126
13127 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
13131 let counters = handler.counters().snapshot();
13132 assert_eq!(
13133 keyed_total(&counters, "route_open_accepted_by_principal"),
13134 1
13135 );
13136 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
13137 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
13138 }
13139
13140 #[tokio::test]
13144 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
13145 let mut swap = swap_with_incumbent().await;
13146 register_candidate(&swap, None);
13147 let incumbent = cutover(&swap);
13148 swap.forwarding
13149 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
13150 .unwrap()
13151 .expect("the incumbent is still registered");
13152
13153 let client = ConnectionId::new(33);
13154 let (client_ctx, mut client_rx) = route_ctx(client);
13155 let route_handler = swap.handler.clone();
13156 let open_ctx = RouteCtx {
13157 connection_id: client,
13158 egress: client_ctx.egress.clone(),
13159 };
13160 let mut route_task = tokio::spawn(async move {
13161 route_handler
13162 .handle_control_frame(
13163 &open_ctx,
13164 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
13165 )
13166 .await
13167 .unwrap()
13168 });
13169 let bind = tokio::select! {
13170 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
13171 response = &mut route_task => {
13172 let response = response.unwrap();
13173 panic!(
13174 "post-cutover route.open was refused instead of relayed to the candidate: {}",
13175 parse_error(&response[0])["code"]
13176 );
13177 }
13178 };
13179 swap.handler
13180 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
13181 .await
13182 .unwrap();
13183 assert!(route_task.await.unwrap().is_empty());
13184 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
13185 match swap
13186 .forwarding
13187 .lookup_data_route(client, channel, epoch)
13188 .unwrap()
13189 {
13190 DataRoute::Client(DataRouteState::Bound(route)) => {
13191 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
13192 }
13193 other => panic!("expected a bound route on the candidate, got {other:?}"),
13194 }
13195 assert!(swap.incumbent_rx.try_recv().is_err());
13196 }
13197
13198 #[tokio::test]
13203 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
13204 let swap = swap_with_incumbent().await;
13205 register_candidate(&swap, Some(false));
13206 let update = Frame::build(
13207 FrameType::Request,
13208 control_flags(),
13209 0,
13210 0,
13211 55,
13212 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
13213 provides: manifest("aft", PROTOCOL_VERSION).provides,
13214 capabilities: None,
13215 ready: Some(true),
13216 })
13217 .unwrap(),
13218 )
13219 .unwrap();
13220
13221 let replies = swap
13222 .handler
13223 .handle_control_frame(&swap.candidate_ctx, update)
13224 .await
13225 .unwrap();
13226
13227 assert_eq!(replies.len(), 1);
13228 assert_eq!(
13229 replies[0].header.ty,
13230 FrameType::Response,
13231 "candidate catalog.update was refused: {:?}",
13232 serde_json::from_slice::<Value>(&replies[0].body).ok()
13233 );
13234 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
13235 assert_eq!(
13236 swap.registry
13237 .get_module("aft")
13238 .unwrap()
13239 .unwrap()
13240 .connection_id,
13241 INCUMBENT
13242 );
13243 }
13244 }
13245
13246 mod swap_admission {
13250 use super::*;
13251
13252 const INCUMBENT_NONCE: &str = "incumbent-nonce";
13253 const CANDIDATE_NONCE: &str = "candidate-nonce";
13254
13255 fn handler_with_incumbent(
13256 module_id: &str,
13257 reserved: bool,
13258 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
13259 let registry = Arc::new(Registry::default());
13260 let supervisor = SupervisorHandle::new();
13261 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
13262 if reserved {
13263 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
13264 }
13265 let handler =
13266 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
13267 let incumbent = handler
13268 .handle_control(
13269 ConnectionId::new(1),
13270 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
13271 )
13272 .unwrap();
13273 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
13274 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
13275 (registry, supervisor, handler)
13276 }
13277
13278 #[test]
13284 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
13285 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
13286
13287 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
13288 let replies = handler
13289 .handle_control(
13290 ConnectionId::new(connection),
13291 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
13292 )
13293 .unwrap();
13294 assert_eq!(replies[0].header.ty, FrameType::Error);
13295 assert_eq!(
13296 parse_error(&replies[0])["code"],
13297 "swap_token_invalid",
13298 "nonce {nonce:?}"
13299 );
13300 }
13301 assert!(registry.get_candidate("aft").unwrap().is_none());
13302 assert_eq!(
13303 registry.get_module("aft").unwrap().unwrap().connection_id,
13304 ConnectionId::new(1)
13305 );
13306
13307 let admitted = handler
13310 .handle_control(
13311 ConnectionId::new(4),
13312 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
13313 )
13314 .unwrap();
13315 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
13316 assert_eq!(
13317 registry
13318 .get_candidate("aft")
13319 .unwrap()
13320 .unwrap()
13321 .connection_id,
13322 ConnectionId::new(4)
13323 );
13324 assert_eq!(
13325 registry.get_module("aft").unwrap().unwrap().connection_id,
13326 ConnectionId::new(1),
13327 "the candidate must not take the active slot"
13328 );
13329 let replayed = handler
13330 .handle_control(
13331 ConnectionId::new(5),
13332 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
13333 )
13334 .unwrap();
13335 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
13336
13337 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
13341 let squatter = handler
13342 .handle_control(
13343 ConnectionId::new(6),
13344 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
13345 )
13346 .unwrap();
13347 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
13348 assert!(
13349 registry.get_module("aft").unwrap().is_none(),
13350 "a squatter took the active slot of an id being swapped"
13351 );
13352 }
13353
13354 #[test]
13359 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
13360 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
13361
13362 let replies = handler
13363 .handle_control(
13364 ConnectionId::new(2),
13365 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
13366 )
13367 .unwrap();
13368
13369 assert_eq!(
13370 replies[0].header.ty,
13371 FrameType::HelloAck,
13372 "reserved candidate refused: {:?}",
13373 serde_json::from_slice::<Value>(&replies[0].body).ok()
13374 );
13375 assert_eq!(
13376 registry
13377 .get_candidate("vault")
13378 .unwrap()
13379 .unwrap()
13380 .connection_id,
13381 ConnectionId::new(2)
13382 );
13383 }
13384
13385 #[test]
13388 fn without_an_open_swap_the_ordinary_gates_decide() {
13389 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
13390 supervisor.close_swap("vault");
13391
13392 let candidate = handler
13393 .handle_control(
13394 ConnectionId::new(2),
13395 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
13396 )
13397 .unwrap();
13398 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
13399 let duplicate = handler
13400 .handle_control(
13401 ConnectionId::new(3),
13402 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
13403 )
13404 .unwrap();
13405 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
13406 assert!(registry.get_candidate("vault").unwrap().is_none());
13407 }
13408 }
13409
13410 mod scopes {
13414 mod activity {
13415 include!("activity_scope_tests.rs");
13416 }
13417 mod delta {
13418 include!("scope_handler_delta_tests.rs");
13419 }
13420
13421 use subc_protocol::scope::{
13422 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
13423 ScopeStatus,
13424 };
13425
13426 use super::*;
13427
13428 const OWNER: &str = "prefrontal-core";
13429
13430 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
13431 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
13432 }
13433
13434 async fn call(
13435 handler: &ControlHandler,
13436 ctx: &RouteCtx,
13437 request: &ModuleControlRequestFromModule,
13438 ) -> Frame {
13439 let body = serde_json::to_vec(request).unwrap();
13440 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
13441 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
13442 assert_eq!(replies.len(), 1, "{replies:?}");
13443 replies.pop().unwrap()
13444 }
13445
13446 async fn sync(
13447 handler: &ControlHandler,
13448 ctx: &RouteCtx,
13449 generation: u64,
13450 scopes: Vec<ScopeRecord>,
13451 ) -> Result<ModuleControlResponseToModule, String> {
13452 let reply = call(
13453 handler,
13454 ctx,
13455 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
13456 )
13457 .await;
13458 match reply.header.ty {
13459 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
13460 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
13461 }
13462 }
13463
13464 async fn describe(
13465 handler: &ControlHandler,
13466 ctx: &RouteCtx,
13467 owner: &str,
13468 scope_ref: &str,
13469 ) -> ModuleControlResponseToModule {
13470 let reply = call(
13471 handler,
13472 ctx,
13473 &ModuleControlRequestFromModule::ScopeDescribe {
13474 owner: Principal::Reserved {
13475 module_id: owner.to_string(),
13476 },
13477 scope_ref: scope_ref.to_string(),
13478 },
13479 )
13480 .await;
13481 assert_eq!(
13482 reply.header.ty,
13483 FrameType::Response,
13484 "{:?}",
13485 parse_error(&reply)
13486 );
13487 serde_json::from_slice(&reply.body).unwrap()
13488 }
13489
13490 async fn module(
13492 handler: &ControlHandler,
13493 connection: u64,
13494 module_id: &str,
13495 nonce: Option<&str>,
13496 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
13497 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
13498 hello_via_sink(
13499 handler,
13500 &ctx,
13501 &mut rx,
13502 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
13503 )
13504 .await;
13505 (ctx, rx)
13506 }
13507
13508 #[tokio::test]
13511 async fn a_client_connection_cannot_sync_or_describe() {
13512 let handler = ControlHandler::new(Arc::new(Registry::default()));
13513 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
13514 for request in [
13515 ModuleControlRequestFromModule::ScopeSync {
13516 generation: 1,
13517 scopes: vec![head("s", 1)],
13518 },
13519 ModuleControlRequestFromModule::ScopeDescribe {
13520 owner: Principal::Direct,
13521 scope_ref: "s".to_string(),
13522 },
13523 ] {
13524 let reply = call(&handler, &ctx, &request).await;
13525 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
13526 }
13527 assert!(
13528 !handler
13529 .scopes
13530 .read()
13531 .unwrap()
13532 .describe(
13533 &Principal::Reserved {
13534 module_id: OWNER.to_string()
13535 },
13536 "s"
13537 )
13538 .owner_synced
13539 );
13540 }
13541
13542 #[tokio::test]
13545 async fn a_module_without_a_supervised_launch_cannot_sync() {
13546 let handler = ControlHandler::new(Arc::new(Registry::default()));
13547 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
13548 assert_eq!(
13549 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
13550 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13551 );
13552 }
13553
13554 #[tokio::test]
13555 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
13556 let supervisor = SupervisorHandle::new();
13557 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13558 let handler = ControlHandler::new(Arc::new(Registry::default()))
13559 .with_supervisor(supervisor.clone());
13560 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
13561 sync(&handler, &incumbent, 1, vec![head("s", 1)])
13562 .await
13563 .expect("the current launch syncs");
13564
13565 supervisor.open_swap(OWNER, "n2".to_string());
13568 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
13569 assert_eq!(
13570 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
13571 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13572 );
13573 sync(&handler, &incumbent, 2, vec![head("s", 1)])
13574 .await
13575 .expect("the serving owner syncs during the swap");
13576
13577 supervisor.close_swap(OWNER);
13580 assert_eq!(
13581 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
13582 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13583 );
13584 sync(&handler, &incumbent, 3, vec![head("s", 1)])
13585 .await
13586 .expect("the serving owner syncs after the rollback");
13587 handler.cleanup_connection(candidate.connection_id).unwrap();
13588
13589 supervisor.open_swap(OWNER, "n3".to_string());
13594 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
13595 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
13596 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
13597 .await
13598 .expect("the promoted launch takes authority");
13599 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
13600 panic!("unexpected reply {reply:?}");
13601 };
13602 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
13603 assert_eq!(
13604 sync(&handler, &incumbent, 4, Vec::new()).await,
13605 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13606 );
13607 }
13608
13609 #[tokio::test]
13612 async fn closing_the_authority_connection_frees_sync_authority() {
13613 let supervisor = SupervisorHandle::new();
13614 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13615 let handler = ControlHandler::new(Arc::new(Registry::default()))
13616 .with_supervisor(supervisor.clone());
13617 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
13618 sync(&handler, &first, 10, vec![head("s", 1)])
13619 .await
13620 .unwrap();
13621 handler.cleanup_connection(first.connection_id).unwrap();
13622
13623 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
13624 sync(&handler, &second, 1, vec![head("s", 1)])
13625 .await
13626 .expect("the next connection takes the released authority");
13627 }
13628
13629 #[tokio::test]
13630 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
13631 let supervisor = SupervisorHandle::new();
13632 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13633 let handler =
13634 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
13635 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
13636 sync(&handler, &first, 10, vec![head("s", 1)])
13637 .await
13638 .unwrap();
13639 handler
13640 .handle_control_frame(
13641 &first,
13642 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
13643 )
13644 .await
13645 .unwrap();
13646 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
13647 sync(&handler, &second, 1, vec![head("s", 1)])
13648 .await
13649 .expect("GOODBYE releases authority even if the old socket remains open");
13650 }
13651
13652 #[tokio::test]
13653 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
13654 let registry = Arc::new(Registry::default());
13655 let supervisor_handle = SupervisorHandle::new();
13656 let supervisor =
13657 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
13658 .with_handle(supervisor_handle.clone())
13659 .with_daemon_incarnation("incarnation-7".to_string());
13660 supervisor
13663 .supervise_configured(
13664 ModuleSpec {
13665 module_id: OWNER.to_string(),
13666 program: PathBuf::from("/nonexistent/prefrontal-core"),
13667 args: Vec::new(),
13668 env: Vec::new(),
13669 reserved: false,
13670 reserved_prefixes: Vec::new(),
13671 protocol: ModuleProtocol::Subc,
13672 overlap: Default::default(),
13673 },
13674 false,
13675 )
13676 .unwrap();
13677 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
13678 let handler =
13679 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13680 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
13681
13682 let ModuleControlResponseToModule::ScopeDescribe {
13684 status,
13685 daemon_incarnation,
13686 owner_synced,
13687 owner_configured,
13688 scope,
13689 ..
13690 } = describe(&handler, &reader, OWNER, "s").await
13691 else {
13692 panic!("not a describe reply");
13693 };
13694 assert_eq!(status, ScopeStatus::NotLive);
13695 assert_eq!(daemon_incarnation, "incarnation-7");
13696 assert!(!owner_synced);
13697 assert!(owner_configured);
13698 assert!(scope.is_none());
13699
13700 let ModuleControlResponseToModule::ScopeDescribe {
13703 status,
13704 owner_configured,
13705 ..
13706 } = describe(&handler, &reader, "ghost", "s").await
13707 else {
13708 panic!("not a describe reply");
13709 };
13710 assert_eq!(status, ScopeStatus::NotLive);
13711 assert!(!owner_configured);
13712
13713 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
13715 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
13716 let ModuleControlResponseToModule::ScopeDescribe {
13717 status,
13718 scope_epoch,
13719 owner_synced,
13720 scope,
13721 ..
13722 } = describe(&handler, &reader, OWNER, "s").await
13723 else {
13724 panic!("not a describe reply");
13725 };
13726 assert_eq!(status, ScopeStatus::Live);
13727 assert_eq!(scope_epoch, Some(4));
13728 assert!(owner_synced);
13729 let stamp = scope.expect("a live scope carries its stamp");
13730 assert!(
13731 stamp.owner_authorized,
13732 "prefrontal-core is the default authority"
13733 );
13734 assert_eq!(stamp.kind, ScopeKind::Head);
13735 }
13736
13737 #[tokio::test]
13738 async fn scope_authority_owners_decides_owner_authorized() {
13739 let supervisor = SupervisorHandle::new();
13740 supervisor.set_spawn_nonce("broca", "b1".to_string());
13741 let handler = ControlHandler::new(Arc::new(Registry::default()))
13742 .with_supervisor(supervisor)
13743 .with_scope_authority_owners(vec!["broca".to_string()]);
13744 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
13745 let mut gated = head("s", 1);
13746 gated.attributes.agent_id = Some("agent".to_string());
13747 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
13748 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
13749 describe(&handler, &broca, "broca", "s").await
13750 else {
13751 panic!("not a describe reply");
13752 };
13753 assert!(scope.unwrap().owner_authorized);
13754 }
13755
13756 #[tokio::test]
13760 async fn scope_ops_and_the_scopes_capability_are_advertised() {
13761 let handler = ControlHandler::new(Arc::new(Registry::default()));
13762 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
13763 let ack = hello_via_sink(
13764 &handler,
13765 &ctx,
13766 &mut rx,
13767 hello_frame("m", PROTOCOL_VERSION, 1),
13768 )
13769 .await;
13770 let ack = parse_ack(&ack);
13771 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
13772 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
13773 }
13774 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
13775
13776 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
13777 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13778 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13779 let reply = handler
13780 .handle_control_frame(&client, frame)
13781 .await
13782 .unwrap()
13783 .pop()
13784 .unwrap();
13785 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13786 serde_json::from_slice(&reply.body).unwrap()
13787 else {
13788 panic!("not a server.describe reply");
13789 };
13790 assert!(
13791 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
13792 "{capabilities:?}"
13793 );
13794 }
13795
13796 const PLEXUS: &str = "plexus";
13799 const OTHER: &str = "other";
13800 const AFT: &str = "aft";
13801 const BROCA: &str = "broca";
13802 const MAGIC: &str = "magic-context";
13803
13804 fn nonce(module_id: &str) -> String {
13805 format!("nonce-{module_id}")
13806 }
13807
13808 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
13809 let (tx, rx) = mpsc::channel(64);
13810 (
13811 RouteCtx {
13812 connection_id: ConnectionId::new(connection),
13813 egress: FrameSink::new(tx),
13814 },
13815 rx,
13816 )
13817 }
13818
13819 struct Rig {
13823 handler: ControlHandler,
13824 forwarding: Arc<ForwardingTable>,
13825 owner: RouteCtx,
13826 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13827 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
13828 generation: u64,
13829 next_connection: u64,
13830 _supervisor: Supervisor,
13831 }
13832
13833 async fn rig() -> Rig {
13834 rig_with_flow_support(true).await
13835 }
13836
13837 async fn rig_with_flow_support(flow_support: bool) -> Rig {
13838 rig_with_scope_capabilities(flow_support, false).await
13839 }
13840
13841 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
13842 let registry = Arc::new(Registry::default());
13843 let forwarding = Arc::new(ForwardingTable::default());
13844 let supervisor_handle = SupervisorHandle::new();
13845 let supervisor =
13846 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
13847 .with_handle(supervisor_handle.clone());
13848 supervisor
13849 .supervise_configured(
13850 ModuleSpec {
13851 module_id: OWNER.to_string(),
13852 program: PathBuf::from("/nonexistent/prefrontal-core"),
13853 args: Vec::new(),
13854 env: Vec::new(),
13855 reserved: false,
13856 reserved_prefixes: Vec::new(),
13857 protocol: ModuleProtocol::Subc,
13858 overlap: Default::default(),
13859 },
13860 false,
13861 )
13862 .unwrap();
13863 for module_id in [OWNER, AFT, BROCA, MAGIC] {
13864 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
13865 }
13866 let handler =
13867 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13868 .with_supervisor(supervisor_handle);
13869 let (owner, mut owner_rx) = wide_ctx(1);
13870 hello_via_sink(
13871 &handler,
13872 &owner,
13873 &mut owner_rx,
13874 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
13875 )
13876 .await;
13877 let mut modules = BTreeMap::new();
13878 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
13879 let (ctx, mut rx) = wide_ctx(connection);
13880 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
13881 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
13882 body["manifest"]["provenance"] =
13886 serde_json::json!({"wire_crate_version": "0.29.0"});
13887 if flow_support || run_support {
13888 let mut provides = Vec::new();
13889 if flow_support {
13890 provides.push("flow-scopes/v1");
13891 }
13892 if run_support {
13893 provides.push("agent-run-scopes/v1");
13894 }
13895 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
13896 }
13897 let hello = Frame::build(
13898 FrameType::Hello,
13899 control_flags(),
13900 0,
13901 0,
13902 connection,
13903 serde_json::to_vec(&body).unwrap(),
13904 )
13905 .unwrap();
13906 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
13907 modules.insert(module_id.to_string(), (ctx, rx));
13908 }
13909 Rig {
13910 handler,
13911 forwarding,
13912 owner,
13913 _owner_rx: owner_rx,
13914 modules,
13915 generation: 0,
13916 next_connection: 100,
13917 _supervisor: supervisor,
13918 }
13919 }
13920
13921 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
13922 ScopeCarrier::new(Principal::Reserved {
13923 module_id: module_id.to_string(),
13924 })
13925 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
13926 }
13927
13928 fn session(scope_epoch: u64) -> ScopeRecord {
13931 let mut record = head("s", scope_epoch);
13932 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
13933 record.attributes.agent_id = Some("agent-1".to_string());
13934 record.attributes.delegates = true;
13935 record
13936 }
13937
13938 impl Rig {
13939 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
13940 self.generation += 1;
13941 sync(&self.handler, &self.owner, self.generation, scopes)
13942 .await
13943 .expect("the owner's sync is accepted");
13944 }
13945
13946 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
13947 ScopeSelector {
13948 owner: Principal::Reserved {
13949 module_id: OWNER.to_string(),
13950 },
13951 scope_ref: scope_ref.to_string(),
13952 scope_epoch,
13953 }
13954 }
13955
13956 fn open_frame(
13957 &mut self,
13958 opener: Option<&str>,
13959 target: &str,
13960 scope: Option<ScopeSelector>,
13961 ) -> (
13962 RouteCtx,
13963 mpsc::Receiver<crate::router::OutboundFrame>,
13964 Frame,
13965 ) {
13966 self.next_connection += 1;
13967 let (ctx, rx) = wide_ctx(self.next_connection);
13968 let root = unique_project_root("scoped-open");
13969 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
13970 target: RouteTarget::ToolProvider {
13971 module_id: target.to_string(),
13972 },
13973 identity: BindIdentity::new(
13974 root.path().to_path_buf(),
13975 "unit".to_string(),
13976 "session".to_string(),
13977 ),
13978 consumer_identity: opener.map(|module_id| ConsumerIdentity {
13979 module_id: module_id.to_string(),
13980 launch_nonce: nonce(module_id),
13981 }),
13982 consumer_capabilities: None,
13983 role_versions: None,
13984 admission_facts: None,
13985 scope,
13986 })
13987 .unwrap();
13988 let frame = Frame::build(
13989 FrameType::Request,
13990 control_flags(),
13991 0,
13992 0,
13993 self.next_connection,
13994 body,
13995 )
13996 .unwrap();
13997 (ctx, rx, frame)
13998 }
13999
14000 async fn refused(
14002 &mut self,
14003 opener: Option<&str>,
14004 target: &str,
14005 scope: Option<ScopeSelector>,
14006 ) -> String {
14007 self.refusal_body(opener, target, scope).await["code"]
14008 .as_str()
14009 .unwrap()
14010 .to_string()
14011 }
14012
14013 async fn refusal_body(
14014 &mut self,
14015 opener: Option<&str>,
14016 target: &str,
14017 scope: Option<ScopeSelector>,
14018 ) -> Value {
14019 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
14020 let replies = tokio::time::timeout(
14021 Duration::from_secs(2),
14022 self.handler.handle_control_frame(&ctx, frame),
14023 )
14024 .await
14025 .expect("the open must be refused before waiting for a bind ack")
14026 .unwrap();
14027 assert_eq!(replies.len(), 1, "{replies:?}");
14028 assert_eq!(replies[0].header.ty, FrameType::Error);
14029 let (_, module_rx) = self.modules.get_mut(target).unwrap();
14030 assert!(
14031 module_rx.try_recv().is_err(),
14032 "a refused open relays nothing"
14033 );
14034 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
14035 parse_error(&replies[0])
14036 }
14037
14038 async fn relayed(
14040 &mut self,
14041 opener: Option<&str>,
14042 target: &str,
14043 scope: Option<ScopeSelector>,
14044 ) -> Relayed {
14045 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
14046 let handler = self.handler.clone();
14047 let task_ctx = ctx.clone();
14048 let task = tokio::spawn(async move {
14049 handler
14050 .handle_control_frame(&task_ctx, frame)
14051 .await
14052 .unwrap()
14053 });
14054 let (_, module_rx) = self.modules.get_mut(target).unwrap();
14055 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
14056 .await
14057 .expect("the target receives the relayed route.bind")
14058 .unwrap()
14059 .frame;
14060 Relayed {
14061 target: target.to_string(),
14062 client: ctx,
14063 client_rx: rx,
14064 task,
14065 bind,
14066 }
14067 }
14068
14069 async fn ack(&self, relayed: &Relayed) {
14070 let (module, _) = &self.modules[&relayed.target];
14071 self.handler
14072 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
14073 .await
14074 .unwrap();
14075 }
14076
14077 async fn bound(
14079 &mut self,
14080 opener: Option<&str>,
14081 target: &str,
14082 scope: Option<ScopeSelector>,
14083 ) -> Bound {
14084 let relayed = self.relayed(opener, target, scope).await;
14085 self.ack(&relayed).await;
14086 let Relayed {
14087 target,
14088 client,
14089 mut client_rx,
14090 task,
14091 bind,
14092 } = relayed;
14093 assert!(
14094 task.await.unwrap().is_empty(),
14095 "the open is answered by commit"
14096 );
14097 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
14098 Bound {
14099 target,
14100 client,
14101 client_rx,
14102 channel,
14103 epoch,
14104 bind,
14105 }
14106 }
14107
14108 fn live(&self, route: &Bound) -> bool {
14109 matches!(
14110 self.forwarding
14111 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
14112 .unwrap(),
14113 DataRoute::Client(DataRouteState::Bound(_))
14114 )
14115 }
14116 }
14117
14118 struct Relayed {
14119 target: String,
14120 client: RouteCtx,
14121 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14122 task: tokio::task::JoinHandle<Vec<Frame>>,
14123 bind: Frame,
14124 }
14125
14126 struct Bound {
14127 target: String,
14128 client: RouteCtx,
14129 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14130 channel: u16,
14131 epoch: u32,
14132 bind: Frame,
14133 }
14134
14135 impl Bound {
14136 fn closed_reason(&mut self) -> RouteCloseReason {
14139 let mut reason = None;
14140 let mut goodbye = false;
14141 while let Ok(outbound) = self.client_rx.try_recv() {
14142 let frame = outbound.frame;
14143 match frame.header.ty {
14144 FrameType::Goodbye => {
14145 assert_eq!(
14146 (frame.header.channel, frame.header.epoch),
14147 (self.channel, self.epoch)
14148 );
14149 goodbye = true;
14150 }
14151 FrameType::Push => {
14152 let ClientControlPush::RouteClosed {
14153 reason: r,
14154 module_id,
14155 ..
14156 } = serde_json::from_slice(&frame.body).unwrap()
14157 else {
14158 panic!("unexpected push");
14159 };
14160 assert_eq!(module_id, self.target);
14161 reason = Some(r);
14162 }
14163 other => panic!("unexpected frame {other:?}"),
14164 }
14165 }
14166 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
14167 reason.expect("the client is told why the route closed")
14168 }
14169
14170 fn untouched(&mut self) -> bool {
14171 self.client_rx.try_recv().is_err()
14172 }
14173
14174 fn stamp(&self) -> Option<ScopeStamp> {
14175 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
14176 ModuleControlRequest::RouteBind { scope, .. } => scope,
14177 other => panic!("expected a route.bind, got {other:?}"),
14178 }
14179 }
14180 }
14181
14182 #[tokio::test]
14183 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
14184 ) {
14185 let mut rig = rig().await;
14186 let mut record = session(1);
14187 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
14188 record.child_owners = vec![Principal::Reserved {
14189 module_id: MAGIC.to_string(),
14190 }];
14191 rig.sync(vec![record]).await;
14192 let scope = || Some(rig_selector("s", Some(1)));
14193
14194 rig.bound(Some(OWNER), PLEXUS, scope()).await;
14197 rig.bound(Some(AFT), OTHER, scope()).await;
14198 rig.bound(Some(BROCA), PLEXUS, scope()).await;
14199
14200 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
14204 assert_eq!(
14205 rig.refused(opener, target, scope()).await,
14206 error_codes::SCOPE_NOT_CARRIER,
14207 "{opener:?} -> {target}"
14208 );
14209 }
14210 }
14211
14212 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
14213 ScopeSelector {
14214 owner: Principal::Reserved {
14215 module_id: OWNER.to_string(),
14216 },
14217 scope_ref: scope_ref.to_string(),
14218 scope_epoch,
14219 }
14220 }
14221
14222 #[tokio::test]
14223 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
14224 let mut rig = rig().await;
14225 rig.sync(vec![session(1)]).await;
14226 for opener in [OWNER, AFT] {
14227 assert_eq!(
14228 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
14229 .await,
14230 error_codes::SCOPE_EPOCH_REQUIRED,
14231 "{opener}"
14232 );
14233 }
14234 }
14235
14236 #[tokio::test]
14237 async fn admission_separates_not_synced_not_live_and_ended() {
14238 let mut rig = rig().await;
14239 let code = rig
14241 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14242 .await;
14243 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
14244 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
14245
14246 let ghost = ScopeSelector {
14248 owner: Principal::Reserved {
14249 module_id: "ghost".to_string(),
14250 },
14251 scope_ref: "s".to_string(),
14252 scope_epoch: Some(1),
14253 };
14254 assert_eq!(
14255 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
14256 error_codes::SCOPE_NOT_LIVE
14257 );
14258
14259 rig.sync(vec![session(2)]).await;
14260 assert_eq!(
14261 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
14262 .await,
14263 error_codes::SCOPE_NOT_LIVE
14264 );
14265 for epoch in [1, 3] {
14266 assert_eq!(
14267 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
14268 .await,
14269 error_codes::SCOPE_ENDED,
14270 "epoch {epoch}"
14271 );
14272 }
14273 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
14275 .await;
14276 }
14277
14278 #[tokio::test]
14279 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
14280 let mut rig = rig().await;
14281 rig.sync(vec![session(1)]).await;
14282 let route = rig
14283 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14284 .await;
14285 let stamp = route.stamp().expect("a scoped bind carries the stamp");
14286 assert_eq!(stamp.scope_ref, "s");
14287 assert_eq!(stamp.scope_epoch, 1);
14288 assert_eq!(stamp.kind, ScopeKind::Head);
14289 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
14290 assert!(stamp.attributes.delegates);
14291 assert!(stamp.owner_authorized);
14292 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
14293 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
14294
14295 let (broca, mut broca_rx) = wide_ctx(50);
14298 hello_via_sink(
14299 &rig.handler,
14300 &broca,
14301 &mut broca_rx,
14302 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
14303 )
14304 .await;
14305 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
14306 .await
14307 .unwrap();
14308 let own = ScopeSelector {
14309 owner: Principal::Reserved {
14310 module_id: BROCA.to_string(),
14311 },
14312 scope_ref: "b".to_string(),
14313 scope_epoch: Some(1),
14314 };
14315 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
14316 assert!(!route.stamp().unwrap().owner_authorized);
14317 }
14318
14319 #[tokio::test]
14320 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
14321 let mut rig = rig().await;
14322 let mut record = head("s", 1);
14323 record.carriers = vec![carrier(AFT, None)];
14324 let flow_id = "Flow:run-7/step_2!~";
14325 record.attributes.flow_id = Some(flow_id.to_string());
14326 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
14327 .await
14328 .unwrap();
14329 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
14330 panic!("not a sync reply");
14331 };
14332 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
14333 let route = rig
14334 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14335 .await;
14336 assert!(rig.live(&route), "the stamped bind committed");
14337 let stamp = route.stamp().expect("a flow scope carries a stamp");
14338 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
14339 assert_eq!(stamp.attributes.agent_id, None);
14340 assert!(!stamp.attributes.delegates);
14341 assert!(stamp.owner_authorized);
14342 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
14343 assert_eq!(unscoped.stamp(), None);
14344 }
14345
14346 #[tokio::test]
14347 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
14348 let mut rig = rig_with_flow_support(false).await;
14349 let mut record = session(1);
14350 record.attributes.flow_id = Some("flow:7".to_string());
14351 rig.sync(vec![record]).await;
14352 for opener in [OWNER, AFT] {
14353 let body = rig
14354 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
14355 .await;
14356 assert_eq!(body["code"], "target_flow_unsupported");
14357 let message = body["message"].as_str().unwrap();
14358 for required in [PLEXUS, "flow-scopes/v1"] {
14359 assert!(message.contains(required), "{message}");
14360 }
14361 }
14362 }
14363
14364 #[tokio::test]
14365 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
14366 let mut rig = rig_with_flow_support(true).await;
14367 let mut record = session(1);
14368 record.attributes.flow_id = Some("flow:7".to_string());
14369 rig.sync(vec![record]).await;
14370 for opener in [OWNER, AFT] {
14371 let route = rig
14372 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
14373 .await;
14374 assert!(rig.live(&route));
14375 assert_eq!(
14376 route.stamp().unwrap().attributes.flow_id.as_deref(),
14377 Some("flow:7")
14378 );
14379 }
14380 }
14381
14382 #[tokio::test]
14383 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
14384 use std::future::Future;
14385
14386 let mut rig = rig_with_flow_support(true).await;
14387 let mut record = session(1);
14388 record.attributes.flow_id = Some("flow:7".to_string());
14389 rig.sync(vec![record]).await;
14390 let (client, mut client_rx, frame) =
14391 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
14392 for _ in 0..64 {
14395 client.egress.try_send(route_bind_ack(1)).unwrap();
14396 }
14397 let handler = rig.handler.clone();
14398 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
14399 std::future::poll_fn(|cx| {
14400 assert!(open.as_mut().poll(cx).is_pending());
14401 std::task::Poll::Ready(())
14402 })
14403 .await;
14404 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
14405
14406 let old_connection = rig.modules[PLEXUS].0.connection_id;
14407 rig.handler.cleanup_connection(old_connection).unwrap();
14408 let (replacement, mut replacement_rx) = wide_ctx(200);
14409 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
14410 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
14411 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
14412 let hello = Frame::build(
14413 FrameType::Hello,
14414 control_flags(),
14415 0,
14416 0,
14417 200,
14418 serde_json::to_vec(&body).unwrap(),
14419 )
14420 .unwrap();
14421 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
14422
14423 client_rx.try_recv().unwrap();
14424 let replies = tokio::time::timeout(Duration::from_secs(2), open)
14425 .await
14426 .expect("the replacement is refused without waiting for a bind ack")
14427 .unwrap();
14428 assert_eq!(replies.len(), 1);
14429 let body = parse_error(&replies[0]);
14430 assert_eq!(body["code"], "target_flow_unsupported");
14431 for required in [PLEXUS, "flow-scopes/v1"] {
14432 assert!(body["message"].as_str().unwrap().contains(required));
14433 }
14434 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
14435 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
14436 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
14437 assert!(rig
14438 .handler
14439 .registry
14440 .get_module_by_connection(replacement.connection_id)
14441 .unwrap()
14442 .is_some());
14443 }
14444
14445 #[tokio::test]
14446 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
14447 {
14448 let mut rig = rig_with_flow_support(false).await;
14449 rig.sync(vec![session(1)]).await;
14450 let route = rig
14451 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14452 .await;
14453 assert!(rig.live(&route));
14454 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
14455 let mut record = session(1);
14456 record.attributes.flow_id = Some("flow:7".to_string());
14457 rig.sync(vec![record]).await;
14458 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
14459 assert!(rig.live(&unscoped));
14460 assert_eq!(unscoped.stamp(), None);
14461 }
14462
14463 #[tokio::test]
14464 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
14465 let mut rig = rig().await;
14466 let mut record = session(1);
14467 record.attributes.flow_id = Some("flow:7".to_string());
14468 rig.sync(vec![record.clone()]).await;
14469 let before = rig.forwarding.published_scope_tag(OWNER, "s");
14470 let mut owner_route = rig
14471 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
14472 .await;
14473 let mut carrier_route = rig
14474 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
14475 .await;
14476 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
14477 rig.sync(vec![record.clone()]).await;
14478 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
14479 assert!(rig.live(&owner_route) && owner_route.untouched());
14480 assert!(rig.live(&carrier_route) && carrier_route.untouched());
14481
14482 record.attributes.flow_id = Some("flow:8".to_string());
14483 rig.sync(vec![record]).await;
14484 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
14485 let before = before.unwrap();
14486 assert_eq!(after.scope_epoch, before.scope_epoch);
14487 assert!(after.version > before.version);
14488 for route in [&mut owner_route, &mut carrier_route] {
14489 assert!(!rig.live(route));
14490 assert_eq!(
14491 route.closed_reason(),
14492 RouteCloseReason::ScopeDelegationChanged
14493 );
14494 }
14495 assert!(rig.live(&unscoped) && unscoped.untouched());
14496 for target in [PLEXUS, OTHER] {
14499 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
14500 let goodbye = module_rx
14501 .try_recv()
14502 .expect("the provider sees the drain")
14503 .frame;
14504 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14505 assert!(module_rx.try_recv().is_err());
14506 }
14507 let rebound = rig
14508 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14509 .await;
14510 assert_eq!(
14511 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
14512 Some("flow:8")
14513 );
14514 }
14515
14516 #[tokio::test]
14521 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
14522 let mut rig = rig().await;
14523 rig.sync(vec![session(1)]).await;
14524 let mut cotenant = rig
14525 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
14526 .await;
14527
14528 let mut changed = session(1);
14529 changed.child_owners.push(Principal::Reserved {
14530 module_id: MAGIC.to_string(),
14531 });
14532 let mut ended = None;
14533 for (code, next) in [
14534 (error_codes::SCOPE_CHANGED, vec![changed]),
14535 (error_codes::SCOPE_ENDED, Vec::new()),
14536 ] {
14537 let relayed = rig
14538 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14539 .await;
14540 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
14541 ended = Some(next.is_empty());
14542 rig.sync(next).await;
14543 rig.ack(&relayed).await;
14544 let replies = relayed.task.await.unwrap();
14545 assert_eq!(replies.len(), 1, "{replies:?}");
14546 assert_eq!(parse_error(&replies[0])["code"], code);
14547 assert_eq!(
14548 subc_protocol::error_codes::is_retryable_route_open(code),
14549 code == error_codes::SCOPE_CHANGED
14550 );
14551 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
14552 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
14554 let mut goodbyes = Vec::new();
14557 while let Ok(outbound) = plexus_rx.try_recv() {
14558 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
14559 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
14560 }
14561 assert!(
14562 goodbyes.contains(&(bind_channel, bind_epoch)),
14563 "{goodbyes:?}"
14564 );
14565 assert!(rig
14566 .handler
14567 .registry
14568 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
14569 .unwrap()
14570 .is_some());
14571 }
14572 assert_eq!(ended, Some(true));
14573 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
14576 }
14577
14578 #[tokio::test]
14581 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
14582 struct Case {
14583 name: &'static str,
14584 change: fn(&mut ScopeRecord),
14585 closed: [Option<RouteCloseReason>; 4],
14588 }
14589 use RouteCloseReason::*;
14590 let cases = [
14591 Case {
14592 name: "a carrier entry removed",
14593 change: |r| {
14594 r.carriers.retain(|c| {
14595 c.principal
14596 != Principal::Reserved {
14597 module_id: AFT.to_string(),
14598 }
14599 })
14600 },
14601 closed: [None, Some(ScopeCarrierRemoved), None, None],
14602 },
14603 Case {
14604 name: "a target removed from a carrier",
14605 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
14606 closed: [None, None, None, Some(ScopeCarrierRemoved)],
14607 },
14608 Case {
14609 name: "a bare carrier narrowed to targets",
14610 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
14611 closed: [None, Some(ScopeCarrierRemoved), None, None],
14612 },
14613 Case {
14614 name: "delegates turned off",
14615 change: |r| r.attributes.delegates = false,
14616 closed: [Some(ScopeDelegationChanged); 4],
14617 },
14618 Case {
14619 name: "agent_id changed",
14620 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
14621 closed: [Some(ScopeDelegationChanged); 4],
14622 },
14623 Case {
14624 name: "a carrier added, child owners changed, the record re-sent",
14625 change: |r| {
14626 r.carriers.push(carrier(MAGIC, None));
14627 r.child_owners.push(Principal::Reserved {
14628 module_id: MAGIC.to_string(),
14629 });
14630 },
14631 closed: [None; 4],
14632 },
14633 Case {
14634 name: "a target added",
14635 change: |r| {
14636 r.carriers[1]
14637 .targets
14638 .as_mut()
14639 .unwrap()
14640 .push("third".to_string())
14641 },
14642 closed: [None; 4],
14643 },
14644 Case {
14645 name: "delegates turned on",
14646 change: |r| r.attributes.delegates = true,
14647 closed: [None; 4],
14648 },
14649 ];
14650 for case in cases {
14651 let mut rig = rig().await;
14652 rig.sync(vec![session(1)]).await;
14653 let scope = || Some(rig_selector("s", Some(1)));
14654 let mut routes = [
14655 rig.bound(Some(OWNER), PLEXUS, scope()).await,
14656 rig.bound(Some(AFT), PLEXUS, scope()).await,
14657 rig.bound(Some(BROCA), PLEXUS, scope()).await,
14658 rig.bound(Some(BROCA), OTHER, scope()).await,
14659 ];
14660 let mut record = session(1);
14661 (case.change)(&mut record);
14662 rig.sync(vec![record]).await;
14663 for (index, expected) in case.closed.iter().enumerate() {
14664 let route = &mut routes[index];
14665 match expected {
14666 Some(reason) => {
14667 assert!(!rig.live(route), "{}: route {index} still live", case.name);
14668 assert_eq!(
14669 route.closed_reason(),
14670 *reason,
14671 "{}: route {index}",
14672 case.name
14673 );
14674 }
14675 None => {
14676 assert!(rig.live(route), "{}: route {index} closed", case.name);
14677 assert!(
14678 route.untouched(),
14679 "{}: route {index} was told something",
14680 case.name
14681 );
14682 }
14683 }
14684 }
14685 }
14686 }
14687
14688 #[tokio::test]
14689 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
14690 for next in [Vec::new(), vec![session(2)]] {
14692 let mut rig = rig().await;
14693 rig.sync(vec![session(1)]).await;
14694 let mut route = rig
14695 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14696 .await;
14697 rig.sync(next).await;
14698 assert!(!rig.live(&route));
14699 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
14700 }
14701
14702 let mut rig = rig().await;
14705 let mut child = session(1);
14706 child.scope_ref = "child".to_string();
14707 child.kind = ScopeKind::Worker;
14708 child.parent = Some(ScopeParent::new(
14709 Principal::Reserved {
14710 module_id: OWNER.to_string(),
14711 },
14712 "s".to_string(),
14713 1,
14714 ));
14715 rig.sync(vec![session(1), child.clone()]).await;
14716 let mut child_route = rig
14717 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
14718 .await;
14719 assert_eq!(
14720 child_route.stamp().unwrap().parent_state,
14721 Some(ParentState::Linked)
14722 );
14723 rig.sync(vec![child]).await;
14724 assert!(!rig.live(&child_route));
14725 assert_eq!(
14726 child_route.closed_reason(),
14727 RouteCloseReason::ScopeParentEnded
14728 );
14729 }
14730
14731 #[tokio::test]
14732 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
14733 ) {
14734 let mut rig = rig().await;
14735 rig.sync(vec![session(1)]).await;
14736 let mut route = rig
14737 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14738 .await;
14739 let before = rig.forwarding.published_scope_tag(OWNER, "s");
14740 rig.sync(vec![session(1)]).await;
14741 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
14742 assert!(rig.live(&route) && route.untouched());
14743
14744 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
14748 .forwarding
14749 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
14750 .unwrap()
14751 else {
14752 panic!("the route is bound");
14753 };
14754 binding.flow.acquire_tagged(9, false).await.unwrap();
14755 let mut widened = session(1);
14756 widened.carriers.push(carrier(MAGIC, None));
14757 rig.sync(vec![widened]).await;
14758 assert!(rig.live(&route) && route.untouched());
14759 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
14760 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
14761 assert_eq!(binding.flow.in_flight(), 1);
14764 binding
14765 .flow
14766 .acquire_tagged(10, false)
14767 .await
14768 .expect("the flow is still open");
14769 }
14770
14771 #[tokio::test]
14774 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
14775 let mut rig = rig().await;
14776 rig.sync(vec![session(1)]).await;
14777 let mut on_incumbent = rig
14778 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14779 .await;
14780
14781 let (candidate, _candidate_rx) = wide_ctx(9);
14784 let registration = rig
14785 .handler
14786 .registry
14787 .register_candidate_with_control_ops(
14788 manifest(PLEXUS, PROTOCOL_VERSION),
14789 PROTOCOL_VERSION,
14790 candidate.connection_id,
14791 module_baseline_control_ops(),
14792 )
14793 .unwrap();
14794 rig.forwarding
14795 .register_candidate_module_connection(
14796 candidate.connection_id,
14797 PLEXUS.to_string(),
14798 PROTOCOL_VERSION,
14799 manifest_concurrency(®istration.manifest),
14800 candidate.egress.clone(),
14801 )
14802 .unwrap();
14803 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
14804 rig.handler
14805 .registry
14806 .promote_candidate(PLEXUS)
14807 .unwrap()
14808 .unwrap();
14809 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
14810
14811 rig.sync(Vec::new()).await;
14812 assert!(!rig.live(&on_incumbent));
14813 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
14814 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
14815 let goodbye = incumbent_rx
14816 .try_recv()
14817 .expect("the superseded endpoint is told")
14818 .frame;
14819 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14820 }
14821 }
14822}
14823
14824#[cfg(test)]
14825mod concurrency_default_exposure_tests {
14826 use super::*;
14827
14828 fn hello_body(role_json: &str) -> Vec<u8> {
14829 format!(
14830 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
14831 )
14832 .into_bytes()
14833 }
14834
14835 fn manifest_from(body: &[u8]) -> ModuleManifest {
14836 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
14837 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
14838 .expect("manifest parses")
14839 }
14840
14841 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
14842
14843 #[test]
14844 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
14845 let body = hello_body(&format!(
14846 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
14847 ));
14848 let manifest = manifest_from(&body);
14849 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
14852 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
14853 }
14854
14855 #[test]
14856 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
14857 let body = hello_body(&format!(
14858 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
14859 ));
14860 let manifest = manifest_from(&body);
14861 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
14862 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
14863 }
14864
14865 #[test]
14866 fn non_management_roles_are_never_reported() {
14867 let body = hello_body(
14868 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
14869 );
14870 let manifest = manifest_from(&body);
14871 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
14872 }
14873}