1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex, RwLock},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
21 error_codes,
22 manifest::{
23 validate_hello_capability_grammar, validate_hello_event_declarations,
24 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
25 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
26 },
27 scope::{
28 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
29 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
30 SCOPE_SYNC_OP,
31 },
32 session::{
33 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
34 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
35 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
36 ROLE_VERSIONS_FIELD,
37 },
38 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
39 Priority, RouteTarget, PROTOCOL_VERSION,
40};
41use tokio::time::{timeout_at, Instant};
42use tracing::{debug, info, warn};
43
44use crate::{
45 capability_requirements::{
46 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
47 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
48 RuntimeModule,
49 },
50 daemon_config::RestartRequiredSection,
51 forwarding::{
52 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
53 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
54 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
55 },
56 observability::{
57 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
58 },
59 provenance::{
60 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
61 },
62 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
63 router::{RouteCtx, RouterError},
64 scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
65 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
66 stderr_tail::{CaptureState, TailEntry},
67 supervise::{
68 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
69 SupervisorHandle, SwapHelloAdmission,
70 },
71 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
72};
73
74pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
80
81const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
82const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
83const CAP_PING_PONG: &str = "ping_pong_v1";
84const CAP_SESSION_ATTACH: &str = "session_attach_v1";
85const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
86
87const SUBC_CONTROL_OPS: &[&str] = &[
88 ops::ACTIVITY_RECORD,
89 ops::SERVER_DESCRIBE,
90 ops::CATALOG_LIST,
91 ops::ROUTE_OPEN,
92 ops::ROUTE_POLL,
93 ops::ROUTE_CLOSING,
94 ops::ROUTE_CLOSED,
95 ops::SUPERVISOR_LIST,
96 ops::SUPERVISOR_RESTART,
97 ops::SUPERVISOR_SWAP,
98 ops::SUPERVISOR_RELOAD,
99 ops::SUPERVISOR_RESCAN,
100 ops::SUPERVISOR_RELEASE_RESERVED,
101 ops::SUPERVISOR_SET_ENABLED,
102 ops::SUPERVISOR_HEALTH_PROBE,
103 ops::SUPERVISOR_HEALTH,
104 ops::SUPERVISOR_STDERR_TAIL,
105 ops::SUPERVISOR_TERMINALS,
106 ops::SUPERVISOR_ROUTES,
107 ops::SUPERVISOR_PROVENANCE,
108 ops::SUPERVISOR_SPAWN_SNAPSHOT,
109 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
110];
111
112const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
113 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
114 "supervisor.live_roots",
115 SCOPE_SYNC_OP,
116 SCOPE_APPLY_OP,
117 SCOPE_DESCRIBE_OP,
118 "operator.confirm",
119];
120
121const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
125
126const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
127
128pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
136
137pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
152
153pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
171
172const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
173const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
174
175fn reload_verdict(
176 configured: &Path,
177 spawned_from: Option<&Path>,
178 image: subc_control::RunningImageAgreement,
179) -> PendingReloadVerdict {
180 let path = match spawned_from {
181 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
182 ReloadPathAgreement::Match
183 }
184 Some(spawned_from) => ReloadPathAgreement::Mismatch {
185 configured: configured.to_path_buf(),
186 spawned_from: spawned_from.to_path_buf(),
187 },
188 None => ReloadPathAgreement::Unavailable {
189 reason: if matches!(
190 image,
191 subc_control::RunningImageAgreement::Unavailable {
192 reason: subc_control::RunningImageUnavailableReason::NotRunning
193 }
194 ) {
195 ReloadPathUnavailableReason::NotRunning
196 } else {
197 ReloadPathUnavailableReason::SpawnedPathUnavailable
198 },
199 },
200 };
201 PendingReloadVerdict { path, image }
202}
203
204fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
205 if configured == spawned_from {
206 return true;
207 }
208 #[cfg(windows)]
209 {
210 match (
216 std::fs::canonicalize(configured),
217 std::fs::canonicalize(spawned_from),
218 ) {
219 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
220 _ => false,
221 }
222 }
223 #[cfg(not(windows))]
224 {
225 false
226 }
227}
228
229#[derive(Clone)]
230struct DaemonProvenanceFacts {
231 build: DaemonBuildProvenance,
232 pid: Option<u32>,
233 started_at_ms: Option<u64>,
234 start_clock: Option<crate::clock::StartClock>,
235 executable_path: Option<PathBuf>,
236 executable_identity: Option<SpawnedFileIdentity>,
237 process_start_time: Option<u64>,
238 probe: ExecutableIdentityProbe,
239}
240
241impl Default for DaemonProvenanceFacts {
242 fn default() -> Self {
243 Self {
244 build: DaemonBuildProvenance {
245 build_git_sha: None,
246 build_lock_digest: None,
247 },
248 pid: None,
249 started_at_ms: None,
250 start_clock: None,
251 executable_path: None,
252 executable_identity: None,
253 process_start_time: None,
254 probe: ExecutableIdentityProbe::default(),
255 }
256 }
257}
258
259#[derive(Debug, Clone)]
260struct SupervisorRescanContext {
261 supervisor: Supervisor,
262 config_path: PathBuf,
263 configured_port: Option<u16>,
264 storage_config: Option<crate::daemon_config::StorageConfig>,
265 admission_facts_carrier_module_id: Option<String>,
266 admission_facts_targets: Option<Vec<String>>,
267 scope_authority_owners: Vec<String>,
268}
269
270const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
291 "reloading",
292 "supervisor_not_live",
293 "registration_not_active",
294 "no_forwarding_connection",
295 "relay_send_failed",
296];
297
298#[derive(Clone)]
300pub struct ControlHandler {
301 registry: Arc<Registry>,
302 activity: Option<crate::activity::ActivityLedger>,
303 forwarding: Arc<ForwardingTable>,
304 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
305 supervisor: SupervisorHandle,
306 subc_capabilities: Arc<[String]>,
307 route_bind_relay_timeout: Duration,
311 route_bind_relay_timeouts: BTreeMap<String, Duration>,
315 route_bind_breakers: RouteBindBreakers,
318 route_bind_concurrency: RouteBindConcurrency,
321 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
326 route_bind_breaker_threshold: u32,
328 route_bind_breaker_cooldown: Duration,
330 health_probe_timeout: Duration,
331 storage_config: Option<crate::daemon_config::StorageConfig>,
334 machine_id: Option<crate::machine_id::MachineId>,
338 admission_facts_carrier_module_id: Option<String>,
339 admission_facts_targets: Option<Vec<String>>,
340 scopes: Arc<RwLock<ScopeTable>>,
344 scope_deadline_added: Arc<tokio::sync::Notify>,
348 scope_authority_owners: Vec<String>,
351 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
354 rescan: Option<SupervisorRescanContext>,
355 connected_clients: ConnectedClients,
356 counters: DaemonCounters,
357 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
358 daemon_provenance: DaemonProvenanceFacts,
359 #[cfg(test)]
360 control_dispatch_delay: Option<Duration>,
361 #[cfg(test)]
362 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
363 #[cfg(test)]
367 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
368}
369
370impl fmt::Debug for ControlHandler {
371 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
372 f.debug_struct("ControlHandler")
373 .field("registry", &self.registry)
374 .field("forwarding", &self.forwarding)
375 .field("process_liveness", &self.process_liveness.is_some())
376 .field("supervisor", &self.supervisor)
377 .field("subc_capabilities", &self.subc_capabilities)
378 .finish()
379 }
380}
381
382struct RouteOpenRequest {
383 target: RouteTarget,
384 identity: BindIdentity,
385 consumer_identity: Option<ConsumerIdentity>,
386 consumer_capabilities: Option<Vec<String>>,
387 role_versions: Option<BTreeMap<String, String>>,
388 admission_facts: Option<serde_json::Value>,
389 scope: Option<ScopeSelector>,
390}
391
392struct RouteBindReservationGuard {
393 forwarding: Arc<ForwardingTable>,
394 endpoint: ModuleEndpointId,
395 relay_corr: u64,
396 armed: bool,
397}
398
399struct ModuleControlRpcGuard {
400 forwarding: Arc<ForwardingTable>,
401 endpoint: ModuleEndpointId,
402 corr: u64,
403 armed: bool,
404}
405
406impl ModuleControlRpcGuard {
407 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
408 Self {
409 forwarding,
410 endpoint,
411 corr,
412 armed: true,
413 }
414 }
415
416 fn disarm(&mut self) {
417 self.armed = false;
418 }
419}
420
421impl Drop for ModuleControlRpcGuard {
422 fn drop(&mut self) {
423 if self.armed {
424 let _ = self
425 .forwarding
426 .cancel_module_control_rpc(self.endpoint, self.corr);
427 }
428 }
429}
430
431impl RouteBindReservationGuard {
432 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
433 Self {
434 forwarding,
435 endpoint,
436 relay_corr,
437 armed: true,
438 }
439 }
440
441 fn release_and_disarm(&mut self) {
442 if !self.armed {
443 return;
444 }
445 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
446 self.endpoint,
447 self.relay_corr,
448 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
449 ) {
450 send_goodbye_target_best_effort(
451 &self.forwarding.counters(),
452 &target,
453 "canceled route.bind",
454 );
455 }
456 self.armed = false;
457 }
458
459 fn disarm(&mut self) {
460 self.armed = false;
461 }
462}
463
464impl Drop for RouteBindReservationGuard {
465 fn drop(&mut self) {
466 self.release_and_disarm();
467 }
468}
469
470#[derive(Debug, Clone, Default)]
500pub(crate) struct RouteBindBreakers {
501 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
502}
503
504#[derive(Debug, Clone, Default)]
505pub(crate) struct RouteBindConcurrency {
506 modules: Arc<Mutex<HashMap<String, usize>>>,
507}
508
509struct RouteBindConcurrencyGuard {
510 concurrency: RouteBindConcurrency,
511 module_id: String,
512}
513
514impl RouteBindConcurrency {
515 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
518 let mut modules = self
519 .modules
520 .lock()
521 .expect("route.bind concurrency mutex poisoned");
522 let in_flight = modules.entry(module_id.to_string()).or_default();
523 if *in_flight >= limit {
524 return Err(*in_flight);
525 }
526 *in_flight += 1;
527 Ok(RouteBindConcurrencyGuard {
528 concurrency: self.clone(),
529 module_id: module_id.to_string(),
530 })
531 }
532}
533
534impl Drop for RouteBindConcurrencyGuard {
535 fn drop(&mut self) {
536 let mut modules = self
537 .concurrency
538 .modules
539 .lock()
540 .expect("route.bind concurrency mutex poisoned");
541 let remove = {
542 let in_flight = modules
543 .get_mut(&self.module_id)
544 .expect("admitted route.bind has a concurrency entry");
545 *in_flight -= 1;
546 *in_flight == 0
547 };
548 if remove {
549 modules.remove(&self.module_id);
550 }
551 }
552}
553
554#[derive(Debug, Default)]
555struct ModuleBreakerState {
556 consecutive_timeouts: u32,
558 cooldown_until: Option<Instant>,
561 probe_in_flight: Option<Arc<()>>,
566}
567
568enum RouteBindAdmission<'a> {
570 Admitted {
571 guard: RouteBindBreakerGuard<'a>,
572 probe: bool,
575 },
576 Refused {
577 consecutive_timeouts: u32,
578 retry_in: Duration,
582 probe_in_flight: bool,
583 },
584}
585
586struct RouteBindBreakerGuard<'a> {
593 breakers: RouteBindBreakers,
594 module_id: &'a str,
595 probe_token: Option<Arc<()>>,
596 settled: bool,
597}
598
599impl RouteBindBreakerGuard<'_> {
600 fn record_accepted(&mut self) -> bool {
604 self.settled = true;
605 self.breakers.record_accepted(self.module_id)
606 }
607
608 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
611 self.settled = true;
612 self.breakers.record_timeout(
613 self.module_id,
614 self.probe_token.as_ref(),
615 threshold,
616 cooldown,
617 )
618 }
619
620 fn record_inconclusive(&mut self) {
629 self.settled = true;
630 self.breakers
631 .record_inconclusive(self.module_id, self.probe_token.as_ref());
632 }
633}
634
635impl Drop for RouteBindBreakerGuard<'_> {
636 fn drop(&mut self) {
637 if !self.settled {
638 self.breakers
639 .record_inconclusive(self.module_id, self.probe_token.as_ref());
640 }
641 }
642}
643
644struct BreakerOpened {
648 consecutive_timeouts: u32,
649 reopened_after_probe: bool,
652}
653
654impl RouteBindBreakers {
655 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
656 self.modules
657 .lock()
658 .expect("route.bind breaker mutex poisoned")
659 }
660
661 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
664 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
665 probe: probe_token.is_some(),
666 guard: RouteBindBreakerGuard {
667 breakers: self.clone(),
668 module_id,
669 probe_token,
670 settled: false,
671 },
672 };
673
674 let mut modules = self.lock();
675 let Some(state) = modules.get_mut(module_id) else {
676 return admitted(None);
677 };
678 let Some(cooldown_until) = state.cooldown_until else {
679 return admitted(None);
680 };
681 if state.probe_in_flight.is_some() {
682 return RouteBindAdmission::Refused {
683 consecutive_timeouts: state.consecutive_timeouts,
684 retry_in: Duration::ZERO,
685 probe_in_flight: true,
686 };
687 }
688 let now = Instant::now();
689 if now < cooldown_until {
690 return RouteBindAdmission::Refused {
691 consecutive_timeouts: state.consecutive_timeouts,
692 retry_in: cooldown_until - now,
693 probe_in_flight: false,
694 };
695 }
696 let token = Arc::new(());
697 state.probe_in_flight = Some(Arc::clone(&token));
698 admitted(Some(token))
699 }
700
701 fn record_accepted(&self, module_id: &str) -> bool {
702 self.lock()
703 .remove(module_id)
704 .is_some_and(|state| state.cooldown_until.is_some())
705 }
706
707 fn record_timeout(
708 &self,
709 module_id: &str,
710 probe_token: Option<&Arc<()>>,
711 threshold: u32,
712 cooldown: Duration,
713 ) -> Option<BreakerOpened> {
714 let mut modules = self.lock();
715 let state = modules.entry(module_id.to_string()).or_default();
716 let was_open = state.cooldown_until.is_some();
717 let was_probe = Self::owns_probe(state, probe_token);
718 if was_probe {
719 state.probe_in_flight = None;
720 }
721 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
722 if state.consecutive_timeouts < threshold {
723 return None;
724 }
725 state.cooldown_until = Some(Instant::now() + cooldown);
726 Some(BreakerOpened {
727 consecutive_timeouts: state.consecutive_timeouts,
728 reopened_after_probe: was_open && was_probe,
729 })
730 }
731
732 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
733 state
737 .probe_in_flight
738 .as_ref()
739 .zip(token)
740 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
741 }
742
743 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
744 if let Some(state) = self.lock().get_mut(module_id) {
745 if Self::owns_probe(state, probe_token) {
746 state.probe_in_flight = None;
747 }
748 }
749 }
750
751 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
766 self.lock()
767 .remove(module_id)
768 .map(|state| state.consecutive_timeouts)
769 .filter(|discarded| *discarded > 0)
770 }
771
772 fn open_snapshot(&self) -> Option<serde_json::Value> {
779 let now = Instant::now();
780 let modules = self.lock();
781 let open = modules
782 .iter()
783 .filter_map(|(module_id, state)| {
784 let cooldown_until = state.cooldown_until?;
785 Some((
786 module_id.clone(),
787 serde_json::json!({
788 "consecutive_timeouts": state.consecutive_timeouts,
789 "cooldown_remaining_ms":
790 cooldown_until.saturating_duration_since(now).as_millis() as u64,
791 "probe_in_flight": state.probe_in_flight.is_some(),
792 }),
793 ))
794 })
795 .collect::<serde_json::Map<String, serde_json::Value>>();
796 (!open.is_empty()).then_some(serde_json::Value::Object(open))
797 }
798}
799
800impl ControlHandler {
801 pub fn new(registry: Arc<Registry>) -> Self {
802 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
803 }
804
805 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
806 let counters = forwarding.counters();
807 let route_bind_breakers = forwarding.route_bind_breakers();
811 let route_bind_concurrency = forwarding.route_bind_concurrency();
812 let route_outages = forwarding.route_outages();
813 Self {
814 registry,
815 activity: None,
816 forwarding,
817 process_liveness: None,
818 supervisor: SupervisorHandle::new(),
819 subc_capabilities: Arc::from([
820 CAP_MANIFEST_REGISTRATION.to_string(),
821 CAP_CHANNEL_LIFECYCLE.to_string(),
822 CAP_PING_PONG.to_string(),
823 CAP_SESSION_ATTACH.to_string(),
824 CAP_ADMISSION_FACTS_RELAY.to_string(),
825 CAP_SCOPES_V1.to_string(),
826 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
827 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
828 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
829 ]),
830 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
831 route_bind_relay_timeouts: BTreeMap::new(),
832 route_bind_breakers,
833 route_bind_concurrency,
834 route_outages,
835 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
836 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
837 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
838 storage_config: None,
839 machine_id: None,
840 admission_facts_carrier_module_id: None,
841 admission_facts_targets: None,
842 scopes: Arc::new(RwLock::new(ScopeTable::new(
843 crate::daemon_config::default_scope_authority_owners(),
844 ))),
845 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
846 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
847 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
848 rescan: None,
849 connected_clients: ConnectedClients::new(),
850 counters,
851 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
852 daemon_provenance: DaemonProvenanceFacts::default(),
853 #[cfg(test)]
854 control_dispatch_delay: None,
855 #[cfg(test)]
856 provenance_probe_override: None,
857 #[cfg(test)]
858 route_bind_relay_sent_hold: None,
859 }
860 }
861
862 pub fn with_storage_config(
865 mut self,
866 storage_config: Option<crate::daemon_config::StorageConfig>,
867 ) -> Self {
868 self.storage_config = storage_config;
869 self
870 }
871
872 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
875 self.activity = Some(crate::activity::ActivityLedger::open(
876 run_dir.join("activity.jsonl"),
877 incarnation,
878 ));
879 self
880 }
881
882 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
885 self.machine_id = machine_id;
886 self
887 }
888
889 pub fn with_admission_facts_config(
893 mut self,
894 carrier_module_id: Option<String>,
895 targets: Option<Vec<String>>,
896 ) -> Self {
897 self.admission_facts_carrier_module_id = carrier_module_id;
898 self.admission_facts_targets = targets;
899 self
900 }
901
902 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
906 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
907 self.scope_authority_owners = owners;
908 self
909 }
910
911 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
914 self.route_bind_relay_timeout = timeout;
915 self
916 }
917
918 pub fn with_route_bind_relay_timeouts(
924 mut self,
925 timeouts: impl IntoIterator<Item = (String, Duration)>,
926 ) -> Self {
927 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
928 self
929 }
930
931 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
937 self.route_bind_relay_timeouts
938 .get(module_id)
939 .copied()
940 .unwrap_or(self.route_bind_relay_timeout)
941 }
942
943 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
951 self.route_bind_breaker_threshold = threshold.max(1);
952 self.route_bind_breaker_cooldown = cooldown;
953 self
954 }
955
956 #[cfg(test)]
957 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
958 self.health_probe_timeout = timeout;
959 self
960 }
961
962 #[cfg(test)]
963 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
964 self.control_dispatch_delay = Some(delay);
965 self
966 }
967
968 #[cfg(test)]
972 pub(crate) fn with_route_bind_relay_sent_hold(
973 mut self,
974 hold: Arc<tokio::sync::Barrier>,
975 ) -> Self {
976 self.route_bind_relay_sent_hold = Some(hold);
977 self
978 }
979
980 pub fn with_process_liveness(
981 mut self,
982 process_liveness: Arc<dyn ModuleProcessLiveness>,
983 ) -> Self {
984 self.process_liveness = Some(process_liveness);
985 self
986 }
987
988 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
989 self.supervisor = supervisor;
990 self
991 }
992
993 pub fn with_daemon_provenance(
994 mut self,
995 pid: u32,
996 started_at_ms: u64,
997 executable_path: Option<PathBuf>,
998 build_git_sha: Option<String>,
999 build_lock_digest: Option<String>,
1000 ) -> Self {
1001 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1002 let process_start_time = process_start_time(pid);
1003 self.daemon_provenance = DaemonProvenanceFacts {
1004 build: DaemonBuildProvenance {
1005 build_git_sha,
1006 build_lock_digest,
1007 },
1008 pid: Some(pid),
1009 started_at_ms: Some(started_at_ms),
1010 start_clock: None,
1011 executable_path,
1012 executable_identity,
1013 process_start_time,
1014 probe: ExecutableIdentityProbe::default(),
1015 };
1016 self
1017 }
1018
1019 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1020 self.daemon_provenance.start_clock = Some(clock);
1021 self
1022 }
1023
1024 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1026 self.scopes
1027 .write()
1028 .unwrap_or_else(|poisoned| poisoned.into_inner())
1029 .set_wall_clock(Arc::new(clock));
1030 self
1031 }
1032
1033 #[cfg(test)]
1034 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1035 self.provenance_probe_override = Some(result);
1036 self
1037 }
1038
1039 pub fn with_capability_config(
1043 self,
1044 modules: impl IntoIterator<Item = (String, bool)>,
1045 reserved_capabilities: BTreeMap<String, String>,
1046 ) -> Self {
1047 self.capability_evaluator
1048 .configure(modules, reserved_capabilities);
1049 self
1050 }
1051
1052 pub fn with_supervisor_rescan(
1053 mut self,
1054 supervisor: Supervisor,
1055 config_path: impl Into<PathBuf>,
1056 configured_port: Option<u16>,
1057 ) -> Self {
1058 self.rescan = Some(SupervisorRescanContext {
1059 supervisor,
1060 config_path: config_path.into(),
1061 configured_port,
1062 storage_config: self.storage_config.clone(),
1063 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1064 admission_facts_targets: self.admission_facts_targets.clone(),
1065 scope_authority_owners: self.scope_authority_owners.clone(),
1066 });
1067 self
1068 }
1069
1070 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1071 self.connected_clients = connected_clients;
1072 self
1073 }
1074
1075 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1076 Arc::clone(&self.forwarding)
1077 }
1078
1079 pub(crate) fn counters(&self) -> DaemonCounters {
1080 self.counters.clone()
1081 }
1082
1083 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1086 tokio::spawn(async move {
1087 loop {
1088 self.capability_evaluator
1089 .wait_for_change_or_deadline()
1090 .await;
1091 self.refresh_capability_requirements();
1092 }
1093 });
1094 }
1095
1096 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1102 tokio::spawn(async move {
1103 loop {
1104 let has_deadlines = self
1105 .scopes
1106 .read()
1107 .unwrap_or_else(|poisoned| poisoned.into_inner())
1108 .has_deadlines();
1109 if !has_deadlines {
1110 self.scope_deadline_added.notified().await;
1111 continue;
1112 }
1113 tokio::time::sleep(Duration::from_secs(1)).await;
1114 if let Err(error) = self.sweep_expired_scopes() {
1115 warn!(%error, "scope expiry sweep failed");
1116 }
1117 }
1118 });
1119 }
1120
1121 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1122 let mut table = self
1123 .scopes
1124 .write()
1125 .unwrap_or_else(|poisoned| poisoned.into_inner());
1126 let swept = table.sweep_expired();
1127 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1128 drop(table);
1129 self.close_scope_drained_routes(drained);
1130 Ok(())
1131 }
1132
1133 fn publish_scope_changes(
1134 &self,
1135 changes: &[crate::scopes::ScopeTagChange],
1136 expired: &[crate::scopes::ScopeExpired],
1137 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1138 if changes.is_empty() {
1139 return Ok(Vec::new());
1140 }
1141 let drained = self
1142 .forwarding
1143 .publish_scope_changes(changes)
1144 .map_err(RouterError::Forwarding)?;
1145 let mut counts = HashMap::new();
1146 for route in &drained {
1147 *counts
1148 .entry((
1149 route.scope.owner.as_str(),
1150 route.scope.scope_ref.as_str(),
1151 route.scope.tag.scope_epoch,
1152 ))
1153 .or_insert(0usize) += 1;
1154 }
1155 for scope in expired {
1156 let routes_closed = counts
1157 .get(&(
1158 scope.owner.as_str(),
1159 scope.scope_ref.as_str(),
1160 scope.scope_epoch,
1161 ))
1162 .copied()
1163 .unwrap_or(0);
1164 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1165 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1166 routes_closed, "scope expired");
1167 }
1168 Ok(drained)
1169 }
1170
1171 fn runtime_capability_snapshot(
1172 &self,
1173 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1174 let runtime = self
1175 .supervisor
1176 .list()
1177 .into_iter()
1178 .map(|module| {
1179 let status = module.status().map_err(|err| {
1180 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1181 })?;
1182 Ok(RuntimeModule {
1183 module_id: status.module_id,
1184 state: status.state,
1185 enabled: status.enabled,
1186 })
1187 })
1188 .collect::<Result<Vec<_>, RouterError>>()?;
1189 let (_, registrations) = self.registry.list_modules().map_err(|err| {
1190 RouterError::backend(
1191 0,
1192 0,
1193 format!("failed to list capability registrations: {err}"),
1194 )
1195 })?;
1196 let registrations = registrations
1197 .into_iter()
1198 .map(|registration| RegisteredModule {
1199 module_id: registration.manifest.module_id,
1200 module_version: registration.manifest.module_version,
1201 capabilities: registration.manifest.capabilities,
1202 })
1203 .collect();
1204 Ok((runtime, registrations))
1205 }
1206
1207 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1214 let cached_registration = RegisteredModule {
1215 module_id: registration.manifest.module_id.clone(),
1216 module_version: registration.manifest.module_version.clone(),
1217 capabilities: registration.manifest.capabilities.clone(),
1218 };
1219 if self.capability_evaluator.record_hello(&cached_registration) {
1220 warn!(
1221 module_id = %cached_registration.module_id,
1222 "capability claims drifted from the cached manifest"
1223 );
1224 }
1225 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1226 self.enforce_capability_denies();
1227 }
1228 self.refresh_capability_requirements();
1229 }
1230
1231 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1235 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1236 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1237 self.supervisor.set_swap_promotion_observer(observer);
1238 }
1239
1240 pub fn refresh_capability_requirements(&self) {
1241 match self.runtime_capability_snapshot() {
1242 Ok((runtime, registrations)) => {
1243 log_requirement_events(
1244 self.capability_evaluator
1245 .evaluate_now(&runtime, ®istrations),
1246 );
1247 }
1248 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1249 }
1250 }
1251
1252 fn enforce_capability_denies(&self) {
1256 let (_, registrations) = match self.registry.list_modules() {
1257 Ok(snapshot) => snapshot,
1258 Err(err) => {
1259 warn!(error = %err, "failed to read registrations for capability deny census");
1260 return;
1261 }
1262 };
1263 let manifests = registrations
1264 .into_iter()
1265 .map(|registration| {
1266 (
1267 registration.manifest.module_id.clone(),
1268 registration.manifest,
1269 )
1270 })
1271 .collect::<BTreeMap<_, _>>();
1272 let census = match self.forwarding.route_census(None) {
1273 Ok(census) => census,
1274 Err(err) => {
1275 warn!(error = %err, "failed to read route census for capability deny enforcement");
1276 return;
1277 }
1278 };
1279
1280 for (target_module_id, routes) in census {
1281 let Some(target_manifest) = manifests.get(&target_module_id) else {
1282 continue;
1283 };
1284 let mut closed_routes = Vec::new();
1285 let mut module_goodbyes = Vec::new();
1286 for route in routes {
1287 let Principal::Reserved {
1288 module_id: opening_module_id,
1289 } = &route.principal
1290 else {
1291 continue;
1292 };
1293 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1294 continue;
1295 };
1296 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1297 continue;
1298 };
1299
1300 match self.forwarding.release_client_route(
1301 route.goodbye_target.connection_id,
1302 route.goodbye_target.channel,
1303 route.goodbye_target.epoch,
1304 ) {
1305 Ok(RouteRelease::Removed(module_goodbye)) => {
1306 warn!(
1307 opening_module_id,
1308 target_module_id,
1309 capability,
1310 "force-closing route because an attested capability deny edge now matches"
1311 );
1312 closed_routes.push(route);
1313 module_goodbyes.push(module_goodbye);
1314 }
1315 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1316 Err(err) => warn!(
1317 opening_module_id,
1318 target_module_id,
1319 capability,
1320 error = %err,
1321 "failed to force-close capability-denied route"
1322 ),
1323 }
1324 }
1325
1326 if closed_routes.is_empty() {
1327 continue;
1328 }
1329 send_route_control_pushes(
1330 &self.forwarding,
1331 closed_routes,
1332 ClientControlPush::RouteClosed {
1333 module_id: target_module_id,
1334 channels: Vec::new(),
1335 reason: RouteCloseReason::CapabilityDenied,
1336 drained: false,
1337 abandoned: 0,
1338 excluded_subscriptions: 0,
1339 terminal: Some(false),
1340 },
1341 );
1342 self.emit_route_goodbyes(module_goodbyes);
1343 }
1344 }
1345
1346 fn not_ready_reason(
1352 &self,
1353 registration: &crate::registry::ModuleRegistration,
1354 ) -> Option<NotReadyReason> {
1355 if !registration.ready {
1356 return Some(NotReadyReason {
1357 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1358 capability: None,
1359 });
1360 }
1361 self.first_unprovided_required_capability(registration)
1362 .map(|capability| NotReadyReason {
1363 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1364 capability: Some(capability),
1365 })
1366 }
1367
1368 fn first_unprovided_required_capability(
1386 &self,
1387 registration: &crate::registry::ModuleRegistration,
1388 ) -> Option<String> {
1389 let required = registration
1390 .manifest
1391 .capabilities
1392 .iter()
1393 .flat_map(|declarations| declarations.requires.iter())
1394 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1395 .map(|requirement| requirement.capability.as_str())
1396 .collect::<BTreeSet<_>>();
1397 if required.is_empty() {
1398 return None;
1399 }
1400 let module_id = registration.manifest.module_id.as_str();
1401 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1402 if required
1403 .iter()
1404 .any(|capability| verdict(capability).is_none())
1405 {
1406 self.refresh_capability_requirements();
1407 }
1408 required
1409 .into_iter()
1410 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1411 .map(str::to_string)
1412 }
1413
1414 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1415 self.capability_evaluator
1416 .statuses()
1417 .into_iter()
1418 .map(capability_requirement_status)
1419 .collect()
1420 }
1421
1422 fn deregister_connection(
1429 &self,
1430 connection_id: ConnectionId,
1431 reason: RegistrationEndReason,
1432 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1433 self.registry
1434 .deregister_connection_with_reason(connection_id, reason)
1435 }
1436
1437 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1438 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1439 return None;
1440 }
1441 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1442 parse_client_control_request(&frame.body)
1443 else {
1444 return None;
1445 };
1446 Some(target_module_id(&target).to_string())
1447 }
1448
1449 pub(crate) fn route_open_capacity_refusal(
1450 &self,
1451 ctx: &RouteCtx,
1452 frame: &Frame,
1453 target_module_id: &str,
1454 in_flight: usize,
1455 limit: usize,
1456 ) -> Result<Frame, RouterError> {
1457 self.route_open_admission_refusal_frame(
1458 ctx,
1459 frame,
1460 target_module_id,
1461 "open_admission_full",
1462 (in_flight, limit),
1463 format!(
1464 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1465 ),
1466 )
1467 }
1468
1469 fn route_open_target_capacity_refusal(
1470 &self,
1471 ctx: &RouteCtx,
1472 frame: &Frame,
1473 target_module_id: &str,
1474 in_flight: usize,
1475 ) -> Result<Frame, RouterError> {
1476 self.route_open_admission_refusal_frame(
1477 ctx,
1478 frame,
1479 target_module_id,
1480 "target_binds_full",
1481 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1482 format!(
1483 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1484 ),
1485 )
1486 }
1487
1488 fn route_open_admission_refusal_frame(
1496 &self,
1497 ctx: &RouteCtx,
1498 frame: &Frame,
1499 target_module_id: &str,
1500 reason: &'static str,
1501 (in_flight, limit): (usize, usize),
1502 message: impl Into<String>,
1503 ) -> Result<Frame, RouterError> {
1504 let code = error_codes::TARGET_UNAVAILABLE;
1505 self.counters.increment_route_open_refused(code);
1506 info!(
1507 target: "control",
1508 code,
1509 reason,
1510 module_id = ?target_module_id,
1511 connection_id = ctx.connection_id.get(),
1512 in_flight,
1513 limit,
1514 "route.open refused"
1515 );
1516 control_error_frame(frame, code, message.into())
1517 }
1518
1519 #[cfg(test)]
1525 pub fn handle_control(
1526 &self,
1527 connection_id: ConnectionId,
1528 frame: Frame,
1529 ) -> Result<Vec<Frame>, RouterError> {
1530 match frame.header.ty {
1531 FrameType::Ping => Ok(vec![pong(&frame)?]),
1532 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1533 FrameType::Goodbye => self.handle_goodbye(connection_id),
1534 ty => Ok(vec![control_error_frame(
1535 &frame,
1536 "unsupported_control_frame",
1537 format!("unsupported channel-0 frame {ty:?}"),
1538 )?]),
1539 }
1540 }
1541
1542 pub async fn handle_control_frame(
1543 &self,
1544 ctx: &RouteCtx,
1545 frame: Frame,
1546 ) -> Result<Vec<Frame>, RouterError> {
1547 self.handle_control_frame_timed(ctx, frame, None).await
1548 }
1549
1550 pub(crate) async fn handle_control_frame_timed(
1551 &self,
1552 ctx: &RouteCtx,
1553 frame: Frame,
1554 dispatch_started_at: Option<StdInstant>,
1555 ) -> Result<Vec<Frame>, RouterError> {
1556 match frame.header.ty {
1557 FrameType::Ping => Ok(vec![pong(&frame)?]),
1558 FrameType::Hello => {
1559 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1560 }
1561 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1562 FrameType::Cancel => {
1563 if self
1566 .forwarding
1567 .operator_confirms()
1568 .cancel(ctx.connection_id, frame.header.corr)
1569 || self
1570 .supervisor
1571 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1572 {
1573 Ok(Vec::new())
1574 } else {
1575 Ok(vec![control_error_frame(
1576 &frame,
1577 "unknown_subscription",
1578 "no supervisor spawn subscription has this correlation id",
1579 )?])
1580 }
1581 }
1582 FrameType::Request => {
1583 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1586 if op
1587 .as_ref()
1588 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1589 {
1590 return self.handle_activity_record(ctx, frame).await;
1591 }
1592 if op
1593 .as_ref()
1594 .is_some_and(|probe| probe.op == "operator.confirm")
1595 {
1596 return self.handle_operator_confirm(ctx, frame);
1597 }
1598 if self
1599 .forwarding
1600 .module_endpoint_for_connection(ctx.connection_id)
1601 .map_err(RouterError::Forwarding)?
1602 .is_some()
1603 {
1604 if !is_known_module_request_op(&frame.body) {
1605 return Ok(vec![control_error_frame(
1606 &frame,
1607 "unsupported_control_frame",
1608 "module-originated channel-0 REQUEST is not supported",
1609 )?]);
1610 }
1611 let request = match parse_module_control_request_from_module(&frame.body) {
1612 Ok(request) => request,
1613 Err((err, ControlRequestBodyError::UnknownOp)) => {
1614 return Ok(vec![control_error_frame(
1615 &frame,
1616 "unsupported_control_frame",
1617 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1618 )?])
1619 }
1620 Err((err, ControlRequestBodyError::InvalidBody)) => {
1621 return Ok(vec![control_error_frame(
1622 &frame,
1623 "invalid_control_body",
1624 format!("malformed module control body: {err}"),
1625 )?])
1626 }
1627 };
1628 let op = module_control_request_op(&request);
1629 let corr = frame.header.corr;
1630 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1631 let result =
1632 self.handle_module_control_request(ctx.connection_id, frame, request);
1633 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1634 return result;
1635 }
1636
1637 if is_known_module_request_op(&frame.body) {
1638 return Ok(vec![control_error_frame(
1639 &frame,
1640 "not_registered",
1641 "catalog.update requires an active module registration owned by this connection",
1642 )?]);
1643 }
1644
1645 let request = match parse_client_control_request(&frame.body) {
1646 Ok(request) => request,
1647 Err((err, ControlRequestBodyError::UnknownOp)) => {
1648 return Ok(vec![control_error_frame(
1649 &frame,
1650 "unknown_control_op",
1651 format!("unknown client control op: {err}"),
1652 )?])
1653 }
1654 Err((err, ControlRequestBodyError::InvalidBody)) => {
1655 return Ok(vec![control_error_frame(
1656 &frame,
1657 "invalid_control_body",
1658 format!("malformed client control body: {err}"),
1659 )?])
1660 }
1661 };
1662 let op = client_control_request_op(&request);
1663 let corr = frame.header.corr;
1664 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1665 #[cfg(test)]
1666 if let Some(delay) = self.control_dispatch_delay {
1667 tokio::time::sleep(delay).await;
1668 }
1669 let result = self
1670 .handle_client_control_request(ctx, frame, request)
1671 .await;
1672 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1673 result
1674 }
1675 FrameType::Push => {
1676 let Some(endpoint) = self
1677 .forwarding
1678 .module_endpoint_for_connection(ctx.connection_id)
1679 .map_err(RouterError::Forwarding)?
1680 else {
1681 return Ok(vec![control_error_frame(
1682 &frame,
1683 "unsupported_control_frame",
1684 "client-originated channel-0 PUSH is not supported",
1685 )?]);
1686 };
1687 self.handle_status_update(endpoint, frame)
1688 }
1689 FrameType::Response | FrameType::Error
1690 if self
1691 .forwarding
1692 .module_endpoint_for_connection(ctx.connection_id)
1693 .map_err(RouterError::Forwarding)?
1694 .is_some() =>
1695 {
1696 self.handle_module_relay_response(ctx.connection_id, frame)
1697 }
1698 ty => Ok(vec![control_error_frame(
1699 &frame,
1700 "unsupported_control_frame",
1701 format!("unsupported channel-0 frame {ty:?}"),
1702 )?]),
1703 }
1704 }
1705
1706 pub fn cleanup_connection(
1707 &self,
1708 connection_id: ConnectionId,
1709 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1710 self.cleanup_connection_with_end_reason(
1711 connection_id,
1712 RegistrationEndReason::ConnectionClosed,
1713 )
1714 }
1715
1716 fn cleanup_connection_with_end_reason(
1717 &self,
1718 connection_id: ConnectionId,
1719 requested_reason: RegistrationEndReason,
1720 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1721 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1722 self.registry
1723 .get_module_by_connection(connection_id)?
1724 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
1725 .and_then(|module| module.registration_end_reason().ok().flatten())
1726 .unwrap_or(requested_reason)
1727 } else {
1728 requested_reason
1729 };
1730 let crash_closed = self
1731 .registry
1732 .get_module_by_connection(connection_id)?
1733 .and_then(|registration| {
1734 self.forwarding
1735 .module_endpoint_for_connection(connection_id)
1736 .ok()
1737 .flatten()
1738 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1739 .map(|routes| (registration.manifest.module_id, routes))
1740 });
1741 let crash_closed = crash_closed.map(|(module_id, routes)| {
1742 let terminal = match self.supervisor.get(&module_id) {
1743 None => false,
1744 Some(module) => match module.will_recover_after_connection_loss() {
1745 Ok(will_recover) => !will_recover,
1746 Err(err) => {
1747 warn!(
1748 %module_id,
1749 error = %err,
1750 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1751 );
1752 false
1753 }
1754 },
1755 };
1756 let (reason, drained) = match self
1760 .forwarding
1761 .daemon_drain_result_for_connection(connection_id)
1762 {
1763 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
1764 Ok(None) => (RouteCloseReason::Crash, false),
1765 Err(err) => {
1766 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1767 (RouteCloseReason::Crash, false)
1768 }
1769 };
1770 (module_id, routes, reason, drained, terminal)
1771 });
1772 let registrations = self.deregister_connection(connection_id, end_reason);
1773 let cleanup = if crash_closed.is_some() {
1774 self.forwarding.cleanup_connection_counted(connection_id)
1775 } else {
1776 self.forwarding
1782 .cleanup_connection(connection_id)
1783 .map(|released| crate::forwarding::ConnectionCleanup {
1784 released,
1785 abandoned_relays: 0,
1786 })
1787 };
1788 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
1793 let abandoned = cleanup
1794 .as_ref()
1795 .map_or(0, |cleanup| cleanup.abandoned_relays);
1796 send_route_control_pushes(
1797 &self.forwarding,
1798 routes,
1799 ClientControlPush::RouteClosed {
1800 module_id,
1801 channels: Vec::new(),
1802 reason,
1803 drained,
1804 abandoned,
1805 excluded_subscriptions: 0,
1806 terminal: Some(terminal),
1807 },
1808 );
1809 }
1810 if let Ok(cleanup) = cleanup {
1811 self.emit_route_goodbyes(cleanup.released);
1812 }
1813 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1817 crate::supervise::notify_registration_release();
1818 self.capability_evaluator.wake_deadline_loop();
1819 self.refresh_capability_requirements();
1820 }
1821 self.supervisor.remove_spawn_subscribers(connection_id);
1822 self.hello_launch_nonces
1825 .lock()
1826 .unwrap_or_else(|poisoned| poisoned.into_inner())
1827 .forget(connection_id);
1828 self.scopes
1829 .write()
1830 .unwrap_or_else(|poisoned| poisoned.into_inner())
1831 .release_connection(connection_id);
1832 registrations
1833 }
1834
1835 pub(crate) fn handle_route_goodbye(
1836 &self,
1837 connection_id: ConnectionId,
1838 route_channel: u16,
1839 route_epoch: u32,
1840 ) -> Result<bool, RouterError> {
1841 debug!(
1842 connection_id = connection_id.get(),
1843 route_channel, route_epoch, "handling route GOODBYE"
1844 );
1845 let RouteRelease::Removed(released_route) = self
1846 .forwarding
1847 .release_client_route(connection_id, route_channel, route_epoch)
1848 .map_err(RouterError::Forwarding)?
1849 else {
1850 return Ok(false);
1851 };
1852 self.emit_route_goodbyes(vec![released_route]);
1853 Ok(true)
1854 }
1855
1856 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1857 for released in released_routes {
1858 let frame = match Frame::build_with_version(
1859 released.negotiated_ver,
1860 FrameType::Goodbye,
1861 control_flags(),
1862 released.channel,
1863 released.epoch,
1864 0,
1865 Vec::new(),
1866 ) {
1867 Ok(frame) => frame,
1868 Err(err) => {
1869 warn!(
1870 route_channel = released.channel,
1871 error = %err,
1872 "failed to build route GOODBYE frame"
1873 );
1874 continue;
1875 }
1876 };
1877 if !released.close_on_delivery_failure() {
1878 crate::forwarding::send_module_route_goodbye(
1879 &self.counters,
1880 &released.sink,
1881 frame,
1882 released.module_id.as_deref(),
1883 "client route released",
1884 );
1885 continue;
1886 }
1887 if let Err(err) = released.sink.try_send(frame) {
1888 warn!(
1889 target_connection_id = released.connection_id.get(),
1890 route_channel = released.channel,
1891 error = %err,
1892 "route GOODBYE was not delivered to client; closing target connection"
1893 );
1894 if self
1895 .forwarding
1896 .escalate_client_delivery_failure(
1897 released.connection_id,
1898 released.channel,
1899 released.epoch,
1900 CloseReason::new(
1901 "route_goodbye_delivery_failed",
1902 format!(
1903 "failed to enqueue route GOODBYE for channel {}: {err}",
1904 released.channel
1905 ),
1906 ),
1907 crate::forwarding::UndeliveredFrame {
1908 module_id: released.module_id.as_deref(),
1909 sink: &released.sink,
1910 },
1911 )
1912 .unwrap_or(false)
1913 {
1914 self.counters.increment_goodbye_relay_client_failed();
1915 }
1916 }
1917 }
1918 }
1919
1920 fn send_abandoned_route_bind_goodbye(
1932 &self,
1933 module_sink: &crate::FrameSink,
1934 negotiated_ver: u8,
1935 module_channel: u16,
1936 module_epoch: u32,
1937 ) {
1938 let frame = match Frame::build_with_version(
1939 negotiated_ver,
1940 FrameType::Goodbye,
1941 control_flags(),
1942 module_channel,
1943 module_epoch,
1944 0,
1945 Vec::new(),
1946 ) {
1947 Ok(frame) => frame,
1948 Err(err) => {
1949 warn!(
1950 route_channel = module_channel,
1951 error = %err,
1952 "failed to build GOODBYE for abandoned route.bind"
1953 );
1954 return;
1955 }
1956 };
1957 crate::forwarding::send_module_route_goodbye(
1958 &self.counters,
1959 module_sink,
1960 frame,
1961 None,
1962 "abandoned route.bind",
1963 );
1964 }
1965
1966 fn handle_hello(
1967 &self,
1968 connection_id: ConnectionId,
1969 sink: Option<crate::FrameSink>,
1970 frame: Frame,
1971 ) -> Result<Vec<Frame>, RouterError> {
1972 debug!(
1973 connection_id = connection_id.get(),
1974 corr = frame.header.corr,
1975 "handling HELLO"
1976 );
1977 if self
1981 .registry
1982 .get_module_by_connection(connection_id)
1983 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
1984 .is_some()
1985 {
1986 return Ok(vec![control_error_frame(
1987 &frame,
1988 "invalid_hello",
1989 "connection is already registered as a module",
1990 )?]);
1991 }
1992 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1993 Ok(value) => value,
1994 Err(err) => {
1995 return Ok(vec![control_error_frame(
1996 &frame,
1997 "invalid_hello",
1998 format!("malformed HELLO body: {err}"),
1999 )?])
2000 }
2001 };
2002 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2003 return Ok(vec![control_error_frame(
2004 &frame,
2005 "invalid_capability_grammar",
2006 err.to_string(),
2007 )?]);
2008 }
2009 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2010 return Ok(vec![control_error_frame(
2011 &frame,
2012 "invalid_manifest",
2013 err.to_string(),
2014 )?]);
2015 }
2016 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2017 return Ok(vec![control_error_frame(
2018 &frame,
2019 "invalid_event_declaration",
2020 err.to_string(),
2021 )?]);
2022 }
2023 if let Some(provenance) = hello_value
2024 .get("manifest")
2025 .and_then(|manifest| manifest.get("provenance"))
2026 {
2027 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2028 return Ok(vec![control_error_frame(
2029 &frame,
2030 "invalid_manifest",
2031 format!("malformed manifest provenance: {err}"),
2032 )?]);
2033 }
2034 }
2035 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2036 Ok(hello) => hello,
2037 Err(err) => {
2038 return Ok(vec![control_error_frame(
2039 &frame,
2040 "invalid_hello",
2041 format!("malformed HELLO body: {err}"),
2042 )?])
2043 }
2044 };
2045
2046 if hello.protocol_ver != hello.manifest.protocol_ver {
2047 return Ok(vec![control_error_frame(
2048 &frame,
2049 "invalid_manifest",
2050 format!(
2051 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2052 hello.protocol_ver, hello.manifest.protocol_ver
2053 ),
2054 )?]);
2055 }
2056
2057 if hello.manifest.module_id.trim().is_empty() {
2058 return Ok(vec![control_error_frame(
2059 &frame,
2060 "invalid_manifest",
2061 "manifest module_id must not be empty",
2062 )?]);
2063 }
2064
2065 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2066 Ok(negotiated_ver) => negotiated_ver,
2067 Err(message) => {
2068 return Ok(vec![control_error_frame(
2069 &frame,
2070 "version_unsupported",
2071 message,
2072 )?])
2073 }
2074 };
2075
2076 let swap_admission = self
2084 .supervisor
2085 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2086 if swap_admission == SwapHelloAdmission::Refused {
2087 warn!(
2088 module_id = %hello.manifest.module_id,
2089 connection_id = connection_id.get(),
2090 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2091 );
2092 return Ok(vec![control_error_frame(
2093 &frame,
2094 "swap_token_invalid",
2095 format!(
2096 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2097 hello.manifest.module_id
2098 ),
2099 )?]);
2100 }
2101 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2102
2103 if let Some(rejection) = (!swap_candidate)
2111 .then(|| {
2112 self.supervisor.reserved_hello_rejection(
2113 &hello.manifest.module_id,
2114 hello.launch_nonce.as_deref(),
2115 )
2116 })
2117 .flatten()
2118 {
2119 let message = match rejection {
2120 ReservedHelloRejection::Exact { module_id } => format!(
2121 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2122 ),
2123 ReservedHelloRejection::Prefix {
2124 prefix,
2125 owner_module_id,
2126 } => format!(
2127 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2128 hello.manifest.module_id
2129 ),
2130 };
2131 return Ok(vec![control_error_frame(
2132 &frame,
2133 "reserved_module",
2134 message,
2135 )?]);
2136 }
2137
2138 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2139 &hello.manifest.module_id,
2140 hello.manifest.capabilities.as_ref(),
2141 );
2142 if let Some(refusal) = reserved_capability_refusals.first() {
2143 let capability = refusal.capability.clone();
2144 let bound_module = refusal.claimants[0].clone();
2145 log_duplicate_claim_events(reserved_capability_refusals);
2146 return Ok(vec![control_error_frame(
2147 &frame,
2148 "reserved_capability",
2149 format!(
2150 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2151 capability, bound_module, hello.manifest.module_id
2152 ),
2153 )?]);
2154 }
2155
2156 if self
2159 .forwarding
2160 .connection_has_client_routes(connection_id)
2161 .map_err(RouterError::Forwarding)?
2162 {
2163 return Ok(vec![control_error_frame(
2164 &frame,
2165 "invalid_hello",
2166 "connection has open client routes and cannot also register as a module",
2167 )?]);
2168 }
2169
2170 self.hello_launch_nonces
2175 .lock()
2176 .unwrap_or_else(|poisoned| poisoned.into_inner())
2177 .record(connection_id, hello.launch_nonce.as_deref());
2178 let control_ops = effective_module_control_ops(hello.control_ops);
2179 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2182 if swap_candidate {
2183 return self.register_swap_candidate(
2184 connection_id,
2185 sink,
2186 &frame,
2187 hello.manifest,
2188 negotiated_ver,
2189 control_ops,
2190 hello_ack,
2191 );
2192 }
2193 let registration = match self.registry.register_with_control_ops(
2194 hello.manifest,
2195 negotiated_ver,
2196 connection_id,
2197 control_ops,
2198 ) {
2199 Ok(registration) => registration,
2200 Err(RegistryError::DuplicateModuleId { module_id }) => {
2201 return Ok(vec![control_error_frame(
2202 &frame,
2203 "duplicate_module_id",
2204 format!(
2205 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2206 ),
2207 )?])
2208 }
2209 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2210 return Ok(vec![control_error_frame(
2211 &frame,
2212 "invalid_module_id",
2213 err.to_string(),
2214 )?])
2215 }
2216 Err(err) => {
2217 return Ok(vec![control_error_frame(
2218 &frame,
2219 "registry_error",
2220 err.to_string(),
2221 )?])
2222 }
2223 };
2224
2225 let reply = if let Some(sink) = sink {
2226 let concurrency = manifest_concurrency(®istration.manifest);
2242 if let Err(err) = self.forwarding.register_module_connection_acked(
2243 connection_id,
2244 registration.manifest.module_id.clone(),
2245 negotiated_ver,
2246 concurrency,
2247 sink,
2248 hello_ack,
2249 ) {
2250 if matches!(
2254 self.deregister_connection(
2255 connection_id,
2256 RegistrationEndReason::RegistrationFailed,
2257 ),
2258 Ok(r) if !r.is_empty()
2259 ) {
2260 crate::supervise::notify_registration_release();
2261 }
2262 return Ok(vec![control_error_frame(
2263 &frame,
2264 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2265 "invalid_hello"
2266 } else {
2267 forwarding_error_code(&err)
2268 },
2269 err.to_string(),
2270 )?]);
2271 }
2272 Vec::new()
2273 } else {
2274 vec![hello_ack]
2277 };
2278
2279 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2288 info!(
2289 module_id = %registration.manifest.module_id,
2290 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2291 );
2292 }
2293
2294 self.apply_registration_capabilities(®istration);
2295
2296 info!(
2297 module_id = %registration.manifest.module_id,
2298 module_version = %registration.manifest.module_version,
2299 negotiated_ver,
2300 routable_provider = manifest_provides_routable_role(®istration.manifest),
2301 connection_id = connection_id.get(),
2302 "module registered"
2303 );
2304
2305 Ok(reply)
2306 }
2307
2308 #[allow(clippy::too_many_arguments)]
2318 fn register_swap_candidate(
2319 &self,
2320 connection_id: ConnectionId,
2321 sink: Option<crate::FrameSink>,
2322 frame: &Frame,
2323 manifest: ModuleManifest,
2324 negotiated_ver: u8,
2325 control_ops: Vec<String>,
2326 hello_ack: Frame,
2327 ) -> Result<Vec<Frame>, RouterError> {
2328 let module_id = manifest.module_id.clone();
2329 let registration = match self.registry.register_candidate_with_control_ops(
2330 manifest,
2331 negotiated_ver,
2332 connection_id,
2333 control_ops,
2334 ) {
2335 Ok(registration) => registration,
2336 Err(RegistryError::DuplicateModuleId { module_id }) => {
2337 return Ok(vec![control_error_frame(
2338 frame,
2339 "duplicate_module_id",
2340 format!(
2341 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2342 ),
2343 )?])
2344 }
2345 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2346 return Ok(vec![control_error_frame(
2347 frame,
2348 "invalid_module_id",
2349 err.to_string(),
2350 )?])
2351 }
2352 Err(err) => {
2353 return Ok(vec![control_error_frame(
2354 frame,
2355 "registry_error",
2356 err.to_string(),
2357 )?])
2358 }
2359 };
2360 let reply = if let Some(sink) = sink {
2361 let concurrency = manifest_concurrency(®istration.manifest);
2365 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2366 connection_id,
2367 module_id.clone(),
2368 negotiated_ver,
2369 concurrency,
2370 sink,
2371 hello_ack,
2372 ) {
2373 if matches!(
2374 self.deregister_connection(
2375 connection_id,
2376 RegistrationEndReason::RegistrationFailed,
2377 ),
2378 Ok(r) if !r.is_empty()
2379 ) {
2380 crate::supervise::notify_registration_release();
2381 }
2382 return Ok(vec![control_error_frame(
2383 frame,
2384 forwarding_error_code(&err),
2385 err.to_string(),
2386 )?]);
2387 }
2388 Vec::new()
2389 } else {
2390 vec![hello_ack]
2391 };
2392 self.supervisor.mark_swap_candidate_admitted(&module_id);
2393 info!(
2394 module_id = %module_id,
2395 module_version = %registration.manifest.module_version,
2396 negotiated_ver,
2397 ready = registration.ready,
2398 connection_id = connection_id.get(),
2399 "swap candidate registered; not routable until cutover"
2400 );
2401 Ok(reply)
2402 }
2403
2404 fn build_hello_ack(
2405 &self,
2406 frame: &Frame,
2407 negotiated_ver: u8,
2408 module_id: &str,
2409 ) -> Result<Frame, RouterError> {
2410 let ack = ModuleHelloAckBody {
2411 negotiated_ver,
2412 subc_ops: module_subc_ops(),
2413 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2414 storage: self
2415 .storage_config
2416 .as_ref()
2417 .map(|cfg| cfg.descriptor_for(module_id)),
2418 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2419 };
2420 let body = serde_json::to_vec(&ack).map_err(|err| {
2421 RouterError::backend(
2422 0,
2423 frame.header.corr,
2424 format!("failed to encode HELLO_ACK: {err}"),
2425 )
2426 })?;
2427
2428 Frame::build_with_version(
2429 negotiated_ver,
2430 FrameType::HelloAck,
2431 control_flags(),
2432 0,
2433 0,
2434 frame.header.corr,
2435 body,
2436 )
2437 .map_err(RouterError::FrameBuild)
2438 }
2439
2440 async fn handle_client_control_request(
2441 &self,
2442 ctx: &RouteCtx,
2443 frame: Frame,
2444 request: ClientControlRequest,
2445 ) -> Result<Vec<Frame>, RouterError> {
2446 match request {
2447 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2448 ClientControlRequest::CatalogList { module_id } => {
2449 self.handle_catalog_list(frame, module_id)
2450 }
2451 ClientControlRequest::RouteOpen {
2452 target,
2453 identity,
2454 consumer_identity,
2455 consumer_capabilities,
2456 role_versions,
2457 admission_facts,
2458 scope,
2459 } => {
2460 self.handle_route_open(
2461 ctx,
2462 frame,
2463 RouteOpenRequest {
2464 target,
2465 identity,
2466 consumer_identity,
2467 consumer_capabilities,
2468 role_versions,
2469 admission_facts,
2470 scope,
2471 },
2472 )
2473 .await
2474 }
2475 ClientControlRequest::RoutePoll {
2476 route_channel,
2477 route_epoch,
2478 kind,
2479 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2480 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2481 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2482 self.handle_supervisor_spawn_snapshot(frame)
2483 }
2484 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2485 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2486 }
2487 ClientControlRequest::SupervisorRestart {
2488 module_id,
2489 drain_timeout_ms,
2490 } => {
2491 self.log_supervisor_request_received(
2492 ctx,
2493 frame.header.corr,
2494 ops::SUPERVISOR_RESTART,
2495 Some(&module_id),
2496 None,
2497 )?;
2498 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2499 .await
2500 }
2501 ClientControlRequest::SupervisorSwap {
2502 module_id,
2503 ready_timeout_ms,
2504 } => {
2505 self.log_supervisor_request_received(
2506 ctx,
2507 frame.header.corr,
2508 ops::SUPERVISOR_SWAP,
2509 Some(&module_id),
2510 None,
2511 )?;
2512 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2513 .await
2514 }
2515 ClientControlRequest::SupervisorReload { module_id } => {
2516 self.log_supervisor_request_received(
2517 ctx,
2518 frame.header.corr,
2519 ops::SUPERVISOR_RELOAD,
2520 Some(&module_id),
2521 None,
2522 )?;
2523 self.handle_supervisor_reload(frame, module_id).await
2524 }
2525 ClientControlRequest::SupervisorRescan { preview } => {
2526 if !preview {
2527 self.log_supervisor_request_received(
2528 ctx,
2529 frame.header.corr,
2530 ops::SUPERVISOR_RESCAN,
2531 None,
2532 None,
2533 )?;
2534 }
2535 self.handle_supervisor_rescan(frame, preview).await
2536 }
2537 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2538 self.log_supervisor_request_received(
2539 ctx,
2540 frame.header.corr,
2541 ops::SUPERVISOR_RELEASE_RESERVED,
2542 Some(&module_id),
2543 None,
2544 )?;
2545 self.handle_supervisor_release_reserved(frame, module_id)
2546 .await
2547 }
2548 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2549 self.log_supervisor_request_received(
2550 ctx,
2551 frame.header.corr,
2552 ops::SUPERVISOR_SET_ENABLED,
2553 Some(&module_id),
2554 Some(enabled),
2555 )?;
2556 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2557 .await
2558 }
2559 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2560 self.handle_supervisor_health_probe(frame, module_id).await
2561 }
2562 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2563 ClientControlRequest::SupervisorRoutes { module_id } => {
2564 self.handle_supervisor_routes(frame, module_id)
2565 }
2566 ClientControlRequest::SupervisorProvenance { module_id } => {
2567 self.handle_supervisor_provenance(frame, module_id).await
2568 }
2569 ClientControlRequest::SupervisorStderrTail {
2570 module_id,
2571 max_lines,
2572 max_bytes,
2573 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2574 ClientControlRequest::SupervisorTerminals { module_id } => {
2575 self.handle_supervisor_terminals(frame, module_id).await
2576 }
2577 }
2578 }
2579
2580 fn log_supervisor_request_received(
2581 &self,
2582 ctx: &RouteCtx,
2583 corr: u64,
2584 op: &'static str,
2585 module_id: Option<&str>,
2586 enabled: Option<bool>,
2587 ) -> Result<(), RouterError> {
2588 let caller = self
2589 .registry
2590 .get_module_by_connection(ctx.connection_id)
2591 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2592 .map(|registration| Principal::Reserved {
2593 module_id: registration.manifest.module_id,
2594 })
2595 .unwrap_or(Principal::Direct);
2596 let caller = principal_label(&caller);
2597 info!(
2600 target: "control",
2601 op,
2602 module_id,
2603 enabled,
2604 connection_id = ctx.connection_id.get(),
2605 caller = %caller,
2606 "supervisor request received"
2607 );
2608 Ok(())
2609 }
2610
2611 fn handle_module_control_request(
2612 &self,
2613 connection_id: ConnectionId,
2614 frame: Frame,
2615 request: ModuleControlRequestFromModule,
2616 ) -> Result<Vec<Frame>, RouterError> {
2617 match request {
2618 ModuleControlRequestFromModule::CatalogUpdate {
2619 provides,
2620 capabilities,
2621 ready,
2622 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2623 ModuleControlRequestFromModule::LiveRoots {} => {
2624 let registered = self
2625 .registry
2626 .get_module_by_connection(connection_id)
2627 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2628 let Some(registration) = registered else {
2629 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2630 };
2631 let response = self
2632 .forwarding
2633 .live_roots(®istration.manifest.module_id)
2634 .map_err(RouterError::Forwarding)?;
2635 Ok(vec![control_response_body_frame(
2636 &frame,
2637 &response,
2638 "ModuleControlResponseToModule::LiveRoots",
2639 )?])
2640 }
2641 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2642 self.handle_scope_sync(connection_id, frame, generation, scopes)
2643 }
2644 ModuleControlRequestFromModule::ScopeApply {
2645 generation,
2646 upsert,
2647 end,
2648 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
2649 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2650 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2651 }
2652 }
2653 }
2654
2655 fn handle_operator_confirm(
2656 &self,
2657 ctx: &RouteCtx,
2658 frame: Frame,
2659 ) -> Result<Vec<Frame>, RouterError> {
2660 use crate::operator_confirm::{audit, Outcome};
2661 let registration = self
2662 .registry
2663 .get_module_by_connection(ctx.connection_id)
2664 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2665 let Some(registration) = registration else {
2666 let outcome = Outcome::refusal("not_registered");
2667 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2668 return Ok(vec![outcome.frame(&frame)]);
2669 };
2670 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
2671 Ok(request) => request,
2672 Err(_) => {
2673 let outcome = Outcome::refusal("invalid_control_body");
2674 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2675 return Ok(vec![outcome.frame(&frame)]);
2676 }
2677 };
2678 let module_id = registration.manifest.module_id;
2679 let nonce = self
2683 .hello_launch_nonces
2684 .lock()
2685 .unwrap_or_else(|p| p.into_inner())
2686 .nonce(ctx.connection_id)
2687 .map(str::to_owned);
2688 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
2689 self.supervisor
2690 .spawned_consumer_authorized(&module_id, nonce)
2691 });
2692 let confirms = self.forwarding.operator_confirms();
2693 self.forwarding
2694 .with_operator_route(
2695 ctx.connection_id,
2696 request.route_channel,
2697 request.route_epoch,
2698 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
2699 )
2700 .map_err(RouterError::Forwarding)
2701 }
2702
2703 fn handle_scope_sync(
2707 &self,
2708 connection_id: ConnectionId,
2709 frame: Frame,
2710 generation: u64,
2711 scopes: Vec<ScopeRecord>,
2712 ) -> Result<Vec<Frame>, RouterError> {
2713 self.handle_scope_change(connection_id, frame, generation, scopes, None)
2714 }
2715
2716 fn handle_scope_change(
2717 &self,
2718 connection_id: ConnectionId,
2719 frame: Frame,
2720 generation: u64,
2721 scopes: Vec<ScopeRecord>,
2722 end: Option<Vec<ScopeEnd>>,
2723 ) -> Result<Vec<Frame>, RouterError> {
2724 let is_apply = end.is_some();
2725 let Some(registration) = self
2726 .registry
2727 .get_module_by_connection(connection_id)
2728 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2729 else {
2730 return Ok(vec![control_error_frame(
2731 &frame,
2732 "not_registered",
2733 "scope.sync and scope.apply require an active module registration owned by this connection",
2734 )?]);
2735 };
2736 let owner = registration.manifest.module_id;
2737 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2738 let is_current_launch = |connection: ConnectionId| {
2739 self.hello_launch_nonces
2740 .lock()
2741 .unwrap_or_else(|poisoned| poisoned.into_inner())
2742 .presented(connection, current_nonce.as_deref())
2743 };
2744 let mut table = self
2749 .scopes
2750 .write()
2751 .unwrap_or_else(|poisoned| poisoned.into_inner());
2752 let outcome = match end {
2753 Some(end) => table.apply(
2754 &owner,
2755 connection_id,
2756 is_current_launch,
2757 generation,
2758 scopes,
2759 end,
2760 ),
2761 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
2762 };
2763 let drained = match &outcome {
2764 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
2765 Err(_) => Vec::new(),
2766 };
2767 let has_deadlines = outcome.is_ok() && table.has_deadlines();
2768 drop(table);
2769 if has_deadlines {
2770 self.scope_deadline_added.notify_one();
2771 }
2772 match outcome {
2773 Ok(applied) => {
2774 let counts = ScopeOutcomeCounts::of(&applied.results);
2775 info!(
2776 owner = %owner,
2777 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
2778 generation,
2779 records = applied.results.len(),
2780 created = counts.created,
2781 replaced = counts.replaced,
2782 updated = counts.updated,
2783 unchanged = counts.unchanged,
2784 refused = counts.refused,
2785 ended = applied.ended.len(),
2786 tag_changes = applied.tag_changes.len(),
2787 routes_closed = drained.len(),
2788 "scope change accepted"
2789 );
2790 for refused in applied
2796 .results
2797 .iter()
2798 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
2799 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
2800 {
2801 warn!(
2802 owner = %owner,
2803 generation,
2804 scope_ref = %refused.scope_ref,
2805 scope_epoch = refused.scope_epoch,
2806 code = refused.code.as_deref().unwrap_or(""),
2807 "scope record refused"
2808 );
2809 }
2810 self.close_scope_drained_routes(drained);
2811 let response = if is_apply {
2812 ModuleControlResponseToModule::ScopeApply {
2813 generation,
2814 results: applied.results,
2815 end_results: applied.end_results,
2816 ended: applied.ended,
2817 }
2818 } else {
2819 ModuleControlResponseToModule::ScopeSync {
2820 generation,
2821 results: applied.results,
2822 ended: applied.ended,
2823 }
2824 };
2825 Ok(vec![control_response_body_frame(
2826 &frame,
2827 &response,
2828 "scope change response",
2829 )?])
2830 }
2831 Err(refusal) => {
2832 info!(
2833 owner = %owner,
2834 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
2835 generation,
2836 code = refusal.code,
2837 "scope change refused"
2838 );
2839 Ok(vec![control_error_frame(
2840 &frame,
2841 refusal.code,
2842 refusal.message,
2843 )?])
2844 }
2845 }
2846 }
2847
2848 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2854 if drained.is_empty() {
2855 return;
2856 }
2857 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2858 BTreeMap::new();
2859 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2860 for route in drained {
2861 warn!(
2862 module_id = %route.module_id,
2863 reason = ?route.reason,
2864 client_connection_id = route.client.connection_id.get(),
2865 route_channel = route.client.channel,
2866 "closing route because its scope changed"
2867 );
2868 pushes
2869 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2870 .or_insert_with(|| (route.reason, Vec::new()))
2871 .1
2872 .push(EndpointRoute {
2873 goodbye_target: route.client.clone(),
2874 principal: Principal::Unverified,
2875 bound_at: Instant::now(),
2876 draining: false,
2877 drain_reason: None,
2878 });
2879 goodbyes.push(route.module);
2880 goodbyes.push(route.client);
2881 }
2882 for ((module_id, _), (reason, routes)) in pushes {
2883 send_route_control_pushes(
2884 &self.forwarding,
2885 routes,
2886 ClientControlPush::RouteClosed {
2887 module_id,
2888 channels: Vec::new(),
2889 reason,
2890 drained: false,
2891 abandoned: 0,
2892 excluded_subscriptions: 0,
2893 terminal: Some(false),
2894 },
2895 );
2896 }
2897 self.emit_route_goodbyes(goodbyes);
2898 }
2899
2900 fn handle_scope_describe(
2903 &self,
2904 connection_id: ConnectionId,
2905 frame: Frame,
2906 owner: Principal,
2907 scope_ref: String,
2908 ) -> Result<Vec<Frame>, RouterError> {
2909 let registered = self
2910 .registry
2911 .get_module_by_connection(connection_id)
2912 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2913 if registered.is_none() {
2914 return Ok(vec![control_error_frame(
2915 &frame,
2916 "not_registered",
2917 "scope.describe requires an active module registration owned by this connection",
2918 )?]);
2919 }
2920 let description = self
2921 .scopes
2922 .read()
2923 .unwrap_or_else(|poisoned| poisoned.into_inner())
2924 .describe(&owner, &scope_ref);
2925 let owner_configured = match &owner {
2926 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
2931 _ => false,
2932 };
2933 let response = ModuleControlResponseToModule::ScopeDescribe {
2934 status: description.status,
2935 scope_epoch: description.scope_epoch,
2936 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2937 owner_synced: description.owner_synced,
2938 owner_configured,
2939 scope: description.stamp,
2940 };
2941 Ok(vec![control_response_body_frame(
2942 &frame,
2943 &response,
2944 "ModuleControlResponseToModule::ScopeDescribe",
2945 )?])
2946 }
2947
2948 fn handle_catalog_update(
2949 &self,
2950 connection_id: ConnectionId,
2951 frame: Frame,
2952 provides: Vec<ProviderRole>,
2953 capabilities: Option<CapabilityDeclarations>,
2954 ready: Option<bool>,
2955 ) -> Result<Vec<Frame>, RouterError> {
2956 self.refresh_capability_requirements();
2957 let Some(registration) = self
2958 .registry
2959 .get_module_by_connection(connection_id)
2960 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2961 else {
2962 return Ok(vec![control_error_frame(
2963 &frame,
2964 "not_registered",
2965 "catalog.update requires an active module registration owned by this connection",
2966 )?]);
2967 };
2968
2969 if let Some(message) =
2970 catalog_update_frozen_field_message(®istration.manifest, &provides)
2971 {
2972 return Ok(vec![control_error_frame(
2973 &frame,
2974 "catalog_update_frozen_field",
2975 message,
2976 )?]);
2977 }
2978
2979 let mut candidate = registration.manifest.clone();
2980 candidate.provides = provides.clone();
2981 candidate.capabilities = capabilities
2982 .clone()
2983 .or_else(|| registration.manifest.capabilities.clone());
2984 if let Err(err) = candidate.validate_capability_grammar() {
2985 return Ok(vec![control_error_frame(
2986 &frame,
2987 "invalid_capability_grammar",
2988 err.to_string(),
2989 )?]);
2990 }
2991
2992 let mut conflicts = self
2995 .capability_evaluator
2996 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
2997 if let Some(conflict) = conflicts.first() {
2998 let message = format!(
2999 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3000 conflict.capability, conflict.claimants[0], candidate.module_id
3001 );
3002 for conflict in &mut conflicts {
3003 conflict.source = DuplicateClaimSource::CatalogUpdate;
3004 }
3005 log_duplicate_claim_events(conflicts);
3006 return Ok(vec![control_error_frame(
3007 &frame,
3008 "reserved_capability",
3009 message,
3010 )?]);
3011 }
3012
3013 let updated = self
3014 .registry
3015 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3016 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3017 if updated.is_none() {
3018 return Ok(vec![control_error_frame(
3019 &frame,
3020 "not_registered",
3021 "catalog.update requires an active module registration owned by this connection",
3022 )?]);
3023 }
3024 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3025 log_duplicate_claim_events(
3026 self.capability_evaluator
3027 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3028 );
3029 }
3030 if capability_census_trigger(
3031 registration.manifest.capabilities.as_ref(),
3032 updated
3033 .as_ref()
3034 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3035 ) {
3036 self.enforce_capability_denies();
3037 }
3038 self.refresh_capability_requirements();
3039
3040 let response = ModuleControlResponseToModule::CatalogUpdate {};
3041 control_response_body_frame(
3042 &frame,
3043 &response,
3044 "ModuleControlResponseToModule::CatalogUpdate",
3045 )
3046 .map(|frame| vec![frame])
3047 }
3048
3049 async fn handle_activity_record(
3050 &self,
3051 ctx: &RouteCtx,
3052 frame: Frame,
3053 ) -> Result<Vec<Frame>, RouterError> {
3054 use subc_control::{
3055 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3056 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3057 };
3058 let result = async {
3059 let module = self
3060 .forwarding
3061 .module_id_for_connection(ctx.connection_id)
3062 .ok()
3063 .flatten()
3064 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3065 let request: ActivityRecordRequest =
3066 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3067 crate::activity::validate(&module, &request)?;
3068 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3069 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3070 let at_ms = std::time::SystemTime::now()
3071 .duration_since(std::time::UNIX_EPOCH)
3072 .unwrap_or_default()
3073 .as_millis() as u64;
3074 ledger.record(module, request, actor, at_ms).await
3075 }
3076 .await;
3077 match result {
3078 Ok(entry) => Ok(vec![control_response_body_frame(
3079 &frame,
3080 &ActivityRecordResponse { seq: entry.seq },
3081 "activity.record",
3082 )?]),
3083 Err(code) => Ok(vec![control_error_frame(
3084 &frame,
3085 code,
3086 "activity recording refused",
3087 )?]),
3088 }
3089 }
3090
3091 fn activity_actor(
3092 &self,
3093 connection: ConnectionId,
3094 module: &str,
3095 on: Option<&subc_control::ActivityOn>,
3096 ) -> Result<subc_control::ActivityActor, &'static str> {
3097 use crate::forwarding::{DataRoute, DataRouteState};
3098 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3099 let principal = |id: &str| Principal::Reserved {
3100 module_id: id.to_owned(),
3101 };
3102 let mut actor = ActivityActor {
3103 principal: principal(module),
3104 scope_owner: None,
3105 agent_id: None,
3106 run_id: None,
3107 flow_id: None,
3108 project_root: None,
3109 };
3110 let scope = match on {
3111 None => return Ok(actor),
3112 Some(ActivityOn::Scope(scope)) => {
3113 let selector = subc_protocol::scope::ScopeSelector {
3114 owner: principal(module),
3115 scope_ref: scope.scope_ref.clone(),
3116 scope_epoch: Some(scope.epoch),
3117 };
3118 self.scopes
3119 .read()
3120 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3121 .admit(&principal(module), module, &selector, true)
3122 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3123 }
3124 Some(ActivityOn::Route(route)) => {
3125 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3126 .forwarding
3127 .lookup_data_route(connection, route.channel, route.epoch)
3128 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3129 else {
3130 return Err(ACTIVITY_ACTOR_UNKNOWN);
3131 };
3132 if binding.module_id != module {
3133 return Err(ACTIVITY_ACTOR_UNKNOWN);
3134 }
3135 actor.principal = binding.principal.clone();
3136 actor.project_root = binding
3137 .project_root
3138 .as_ref()
3139 .map(|root| root.as_path().to_path_buf());
3140 let Some(bound) = &binding.scope else {
3141 return Ok(actor);
3142 };
3143 let owner = principal(&bound.owner);
3144 let selector = subc_protocol::scope::ScopeSelector {
3145 owner: owner.clone(),
3146 scope_ref: bound.scope_ref.clone(),
3147 scope_epoch: Some(bound.tag.scope_epoch),
3148 };
3149 let admitted = self
3150 .scopes
3151 .read()
3152 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3153 .admit(&owner, module, &selector, true)
3154 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3155 if admitted.tag != bound.tag {
3156 return Err(ACTIVITY_ACTOR_UNKNOWN);
3157 }
3158 admitted
3159 }
3160 };
3161 actor.scope_owner = Some(scope.stamp.owner);
3162 actor.agent_id = scope.stamp.attributes.agent_id;
3163 actor.run_id = scope.stamp.attributes.run_id;
3164 actor.flow_id = scope.stamp.attributes.flow_id;
3165 Ok(actor)
3166 }
3167
3168 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3169 self.refresh_capability_requirements();
3170 let mut counters = self.counters.snapshot();
3176 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3177 obj.insert(
3178 "activity_unparseable_lines".into(),
3179 ledger.read_errors().into(),
3180 );
3181 }
3182 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3183 self.forwarding.client_route_concentration(),
3184 counters.as_object_mut(),
3185 ) {
3186 obj.insert(
3187 "client_connections_with_routes".into(),
3188 connections_with_routes.into(),
3189 );
3190 obj.insert("max_routes_on_one_connection".into(), max.into());
3191 }
3192 if let (Some(open_breakers), Some(obj)) = (
3198 self.route_bind_breakers.open_snapshot(),
3199 counters.as_object_mut(),
3200 ) {
3201 obj.insert("route_bind_breakers_open".into(), open_breakers);
3202 }
3203 let response = ClientControlResponse::ServerDescribe {
3204 protocol_ver: PROTOCOL_VERSION,
3205 subc_ops: subc_ops(),
3206 capabilities: self.subc_capabilities.as_ref().to_vec(),
3207 connected_clients: self.connected_clients.count(),
3208 counters: Some(counters),
3209 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3210 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3211 capability_requirements: self.capability_requirement_statuses(),
3212 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3213 };
3214 Ok(vec![control_response_body_frame(
3215 &frame,
3216 &response,
3217 "ClientControlResponse::ServerDescribe",
3218 )?])
3219 }
3220
3221 fn handle_catalog_list(
3222 &self,
3223 frame: Frame,
3224 module_id: Option<String>,
3225 ) -> Result<Vec<Frame>, RouterError> {
3226 let (generation, modules) = self.registry.list_modules().map_err(|err| {
3227 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3228 })?;
3229 let entries = modules
3230 .into_iter()
3231 .filter(|registration| {
3232 module_id
3233 .as_deref()
3234 .map(|wanted| registration.manifest.module_id == wanted)
3235 .unwrap_or(true)
3236 })
3237 .map(|registration| {
3238 let not_ready = self.not_ready_reason(®istration);
3239 let roles = registration.manifest.provides;
3240 CatalogEntry::new(
3241 registration.manifest.module_id,
3242 roles,
3243 registration.control_ops,
3244 )
3245 .with_ready(not_ready.is_none())
3246 .with_not_ready(not_ready)
3247 .with_module_version(Some(registration.manifest.module_version))
3248 .with_capabilities(registration.manifest.capabilities)
3249 .with_self_signals(registration.manifest.self_signals)
3250 .with_events(registration.manifest.events)
3251 })
3252 .collect();
3253 let response = ClientControlResponse::CatalogList {
3254 generation,
3255 modules: entries,
3256 subc_ops: subc_ops(),
3257 };
3258 Ok(vec![control_response_body_frame(
3259 &frame,
3260 &response,
3261 "ClientControlResponse::CatalogList",
3262 )?])
3263 }
3264
3265 fn route_open_principal(
3266 &self,
3267 frame: &Frame,
3268 consumer_identity: Option<ConsumerIdentity>,
3269 ) -> Result<Result<Principal, Frame>, RouterError> {
3270 let Some(consumer_identity) = consumer_identity else {
3271 return Ok(Ok(Principal::Direct));
3272 };
3273
3274 if self.supervisor.spawned_consumer_authorized(
3275 &consumer_identity.module_id,
3276 &consumer_identity.launch_nonce,
3277 ) {
3278 return Ok(Ok(Principal::Reserved {
3279 module_id: consumer_identity.module_id,
3280 }));
3281 }
3282
3283 Ok(Err(control_error_frame(
3284 frame,
3285 "bad_consumer_identity",
3286 format!(
3287 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3288 consumer_identity.module_id
3289 ),
3290 )?))
3291 }
3292
3293 fn route_open_refusal_frame(
3300 &self,
3301 ctx: &RouteCtx,
3302 frame: &Frame,
3303 module_id: &str,
3304 reason: &'static str,
3305 code: &'static str,
3306 message: impl Into<String>,
3307 ) -> Result<Frame, RouterError> {
3308 self.observe_route_open_refusal(ctx, module_id, reason, code);
3309 control_error_frame(frame, code, message.into())
3310 }
3311
3312 fn route_open_breaker_refusal_frame(
3330 &self,
3331 ctx: &RouteCtx,
3332 frame: &Frame,
3333 module_id: &str,
3334 consecutive_timeouts: u32,
3335 retry_in: Duration,
3336 probe_in_flight: bool,
3337 ) -> Result<Frame, RouterError> {
3338 self.counters
3339 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
3340 debug!(
3341 target: "control",
3342 code = "module_timeout",
3343 module_id = ?module_id,
3344 connection_id = ctx.connection_id.get(),
3345 consecutive_timeouts,
3346 retry_in_ms = retry_in.as_millis() as u64,
3347 probe_in_flight,
3348 "route.open refused by open bind-relay breaker"
3349 );
3350 let detail = if probe_in_flight {
3357 "one test route is already being tried; retry once it settles".to_string()
3358 } else {
3359 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
3360 };
3361 control_error_frame(
3362 frame,
3363 "module_timeout",
3364 format!(
3365 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
3366 timed out in a row); {detail}; its established routes are unaffected"
3367 ),
3368 )
3369 }
3370
3371 fn observe_route_open_refusal(
3384 &self,
3385 ctx: &RouteCtx,
3386 module_id: &str,
3387 reason: &'static str,
3388 code: &'static str,
3389 ) {
3390 self.counters.increment_route_open_refused(code);
3391 info!(
3392 target: "control",
3393 code,
3394 reason,
3395 module_id = ?module_id,
3396 connection_id = ctx.connection_id.get(),
3397 "route.open refused"
3398 );
3399 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
3400 self.route_outages.record_not_serving(module_id, reason);
3401 }
3402 }
3403
3404 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
3449 self.route_outages.record_accepted(module_id);
3450 self.counters.increment_route_open_accepted(principal);
3451 info!(
3452 target: "control",
3453 principal,
3454 module_id,
3455 connection_id = ctx.connection_id.get(),
3456 "route.open accepted"
3457 );
3458 }
3459
3460 fn supervised_absent_route_open_refusal_frame(
3461 &self,
3462 ctx: &RouteCtx,
3463 frame: &Frame,
3464 module_id: &str,
3465 code: &'static str,
3466 status: &crate::supervise::ModuleStatus,
3467 ) -> Result<Frame, RouterError> {
3468 self.counters.increment_route_open_refused(code);
3469 info!(
3470 target: "control",
3471 code,
3472 reason = "supervised_not_registered",
3473 module_id = ?module_id,
3474 connection_id = ctx.connection_id.get(),
3475 state = %status.state,
3476 enabled = status.enabled,
3477 live = status.live,
3478 "route.open refused"
3479 );
3480 self.route_outages
3484 .record_not_serving(module_id, "supervised_not_registered");
3485 let message = if let Some(pid) = status.waiting_for_previous_pid {
3486 format!("waiting: previous instance (pid {pid}) is still running but hidden; it cannot be identified")
3487 } else {
3488 format!(
3489 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
3490 status.state, status.enabled, status.live
3491 )
3492 };
3493 control_error_frame(frame, code, message)
3494 }
3495
3496 async fn handle_route_open(
3497 &self,
3498 ctx: &RouteCtx,
3499 frame: Frame,
3500 request: RouteOpenRequest,
3501 ) -> Result<Vec<Frame>, RouterError> {
3502 let RouteOpenRequest {
3503 target,
3504 mut identity,
3505 consumer_identity,
3506 consumer_capabilities,
3507 role_versions,
3508 admission_facts,
3509 scope,
3510 } = request;
3511 let target_module_id = target_module_id(&target).to_string();
3512 if self
3513 .registry
3514 .get_module_by_connection(ctx.connection_id)
3515 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3516 .is_some()
3517 {
3518 return Ok(vec![control_error_frame(
3519 &frame,
3520 "invalid_request",
3521 "module connections cannot open client routes",
3522 )?]);
3523 }
3524 debug!(
3525 connection_id = ctx.connection_id.get(),
3526 corr = frame.header.corr,
3527 module_id = %target_module_id,
3528 "handling route.open"
3529 );
3530
3531 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
3537 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
3538 self.observe_route_open_refusal(
3539 ctx,
3540 &target_module_id,
3541 "invalid_role_versions",
3542 error_codes::INVALID_REQUEST,
3543 );
3544 return Ok(vec![control_error_body_frame(
3545 &frame,
3546 ErrorBody {
3547 code: error_codes::INVALID_REQUEST.to_string(),
3548 message: error.to_string(),
3549 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
3550 },
3551 )?]);
3552 }
3553
3554 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
3555 if status.waiting_for_previous_pid.is_some() {
3556 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3557 ctx,
3558 &frame,
3559 &target_module_id,
3560 "module_warming",
3561 &status,
3562 )?]);
3563 }
3564 }
3565
3566 let Some(registration) = self
3585 .registry
3586 .get_module(&target_module_id)
3587 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3588 else {
3589 if let Some((status, warming)) =
3590 self.supervisor_status(&target_module_id, frame.header.corr)?
3591 {
3592 if status.protocol == ModuleProtocol::None {
3599 return Ok(vec![self.route_open_refusal_frame(
3600 ctx,
3601 &frame,
3602 &target_module_id,
3603 "protocol_none",
3604 error_codes::MODULE_NO_PROTOCOL,
3605 format!(
3606 "module_id '{target_module_id}' is declared protocol: none; \
3607 it speaks no subc wire and serves no routes"
3608 ),
3609 )?]);
3610 }
3611 let code = if warming {
3612 "module_warming"
3613 } else {
3614 "target_unavailable"
3615 };
3616 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3617 ctx,
3618 &frame,
3619 &target_module_id,
3620 code,
3621 &status,
3622 )?]);
3623 }
3624 if let Some(removed_ago_ms) =
3625 self.supervisor.removal_tombstone_age_ms(&target_module_id)
3626 {
3627 return Ok(vec![self.route_open_refusal_frame(
3628 ctx,
3629 &frame,
3630 &target_module_id,
3631 "removed",
3632 error_codes::MODULE_REMOVED,
3633 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
3634 )?]);
3635 }
3636 return Ok(vec![self.route_open_refusal_frame(
3637 ctx,
3638 &frame,
3639 &target_module_id,
3640 "not_registered",
3641 error_codes::UNKNOWN_MODULE,
3642 format!("module_id '{target_module_id}' is not registered"),
3643 )?]);
3644 };
3645
3646 if !registration.ready {
3650 self.counters
3651 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
3652 info!(
3653 target: "control",
3654 code = error_codes::MODULE_WARMING,
3655 module_id = ?target_module_id,
3656 connection_id = ctx.connection_id.get(),
3657 reason = "declared_not_ready",
3658 "route.open refused"
3659 );
3660 self.route_outages
3663 .record_not_serving(&target_module_id, "declared_not_ready");
3664 return Ok(vec![control_error_body_frame(
3665 &frame,
3666 ErrorBody {
3667 code: error_codes::MODULE_WARMING.to_string(),
3668 message: format!(
3669 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
3670 ),
3671 detail: Some(serde_json::json!({
3672 "reason": "declared_not_ready"
3673 })),
3674 },
3675 )?]);
3676 }
3677
3678 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
3697 self.counters
3698 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3699 info!(
3700 target: "control",
3701 code = error_codes::MODULE_WARMING,
3702 module_id = ?target_module_id,
3703 connection_id = ctx.connection_id.get(),
3704 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3705 capability = %capability,
3706 "route.open refused"
3707 );
3708 return Ok(vec![control_error_body_frame(
3709 &frame,
3710 ErrorBody {
3711 code: error_codes::MODULE_WARMING.to_string(),
3712 message: format!(
3713 "module_id '{target_module_id}' requires capability '{capability}', \
3714 which no registered module provides; retry"
3715 ),
3716 detail: Some(serde_json::json!({
3717 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3718 "capability": capability,
3719 })),
3720 },
3721 )?]);
3722 }
3723
3724 if !target_has_required_role(&target, ®istration.manifest.provides) {
3725 return Ok(vec![self.route_open_refusal_frame(
3726 ctx,
3727 &frame,
3728 &target_module_id,
3729 "role_not_provided",
3730 "target_unavailable",
3731 format!("module_id '{target_module_id}' does not provide the requested target"),
3732 )?]);
3733 }
3734
3735 if registration.state != ChannelState::Active {
3736 return Ok(vec![self.route_open_refusal_frame(
3737 ctx,
3738 &frame,
3739 &target_module_id,
3740 "registration_not_active",
3741 "target_unavailable",
3742 format!("module_id '{target_module_id}' is not active"),
3743 )?]);
3744 }
3745
3746 if self
3747 .forwarding
3748 .module_is_draining(&target_module_id)
3749 .map_err(RouterError::Forwarding)?
3750 {
3751 return Ok(vec![self.route_open_refusal_frame(
3752 ctx,
3753 &frame,
3754 &target_module_id,
3755 "reloading",
3756 "module_reloading",
3757 format!("module_id '{target_module_id}' is reloading"),
3758 )?]);
3759 }
3760
3761 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
3762 process_liveness.process_live(&target_module_id) == Some(false)
3763 }) {
3764 if process_liveness.process_replacing(&target_module_id) {
3771 return Ok(vec![self.route_open_refusal_frame(
3772 ctx,
3773 &frame,
3774 &target_module_id,
3775 "reloading",
3776 "module_reloading",
3777 format!("module_id '{target_module_id}' is reloading"),
3778 )?]);
3779 }
3780 return Ok(vec![self.route_open_refusal_frame(
3781 ctx,
3782 &frame,
3783 &target_module_id,
3784 "supervisor_not_live",
3785 "target_unavailable",
3786 format!("module_id '{target_module_id}' is not live"),
3787 )?]);
3788 }
3789
3790 if !self
3791 .forwarding
3792 .has_live_module_connection(&target_module_id)
3793 .map_err(RouterError::Forwarding)?
3794 {
3795 return Ok(vec![self.route_open_refusal_frame(
3796 ctx,
3797 &frame,
3798 &target_module_id,
3799 "no_forwarding_connection",
3800 "target_unavailable",
3801 format!("module_id '{target_module_id}' has no live forwarding connection"),
3802 )?]);
3803 }
3804
3805 if let Some(error) =
3806 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
3807 {
3808 self.observe_route_open_refusal(
3809 ctx,
3810 &target_module_id,
3811 "op_not_allowed",
3812 "op_not_allowed",
3813 );
3814 return Ok(vec![error]);
3815 }
3816
3817 let principal = match self.route_open_principal(&frame, consumer_identity)? {
3818 Ok(principal) => principal,
3819 Err(error) => {
3820 self.observe_route_open_refusal(
3821 ctx,
3822 &target_module_id,
3823 "bad_consumer_identity",
3824 "bad_consumer_identity",
3825 );
3826 return Ok(vec![error]);
3827 }
3828 };
3829
3830 if let Principal::Reserved {
3834 module_id: opening_module_id,
3835 } = &principal
3836 {
3837 if let Some(opening_registration) = self
3838 .registry
3839 .get_module(opening_module_id)
3840 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3841 {
3842 if let Some(capability) =
3843 denied_capability(&opening_registration.manifest, ®istration.manifest)
3844 {
3845 warn!(
3846 opening_module_id,
3847 target_module_id,
3848 capability,
3849 "refusing route.open because an attested capability deny edge matches"
3850 );
3851 return Ok(vec![self.route_open_refusal_frame(
3852 ctx,
3853 &frame,
3854 &target_module_id,
3855 "capability_deny_edge",
3856 "capability_forbidden",
3857 format!(
3858 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
3859 ),
3860 )?]);
3861 }
3862 }
3863 }
3864
3865 if admission_facts.is_some() {
3866 let carrier_matches = matches!(
3867 &principal,
3868 Principal::Reserved { module_id }
3869 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
3870 );
3871 if !carrier_matches {
3872 return Ok(vec![self.route_open_refusal_frame(
3873 ctx,
3874 &frame,
3875 &target_module_id,
3876 "admission_facts_carrier_not_permitted",
3877 "admission_facts_not_permitted",
3878 "admission facts may only be carried by the configured reserved module",
3879 )?]);
3880 }
3881
3882 let target_allowed = self
3883 .admission_facts_targets
3884 .as_ref()
3885 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
3886 if !target_allowed {
3887 return Ok(vec![self.route_open_refusal_frame(
3888 ctx,
3889 &frame,
3890 &target_module_id,
3891 "admission_facts_target_not_listed",
3892 "admission_facts_target_not_allowed",
3893 format!(
3894 "admission facts are not permitted for target module_id '{target_module_id}'"
3895 ),
3896 )?]);
3897 }
3898
3899 }
3903
3904 let (bound_scope, mut scope_stamp) = match scope {
3910 None => (None, None),
3911 Some(selector) => {
3912 let owner_configured = match &selector.owner {
3913 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3919 _ => false,
3920 };
3921 let admitted = self
3922 .scopes
3923 .read()
3924 .unwrap_or_else(|poisoned| poisoned.into_inner())
3925 .admit(&principal, &target_module_id, &selector, owner_configured)
3926 .and_then(|admission| {
3927 crate::scopes::check_target_flow_support(
3928 &admission.stamp,
3929 &target_module_id,
3930 registration.manifest.capabilities.as_ref(),
3931 )?;
3932 crate::scopes::check_target_agent_run_support(
3933 &admission.stamp,
3934 &target_module_id,
3935 registration.manifest.capabilities.as_ref(),
3936 )?;
3937 Ok(admission)
3938 });
3939 match admitted {
3940 Ok(admission) => (
3941 Some(BoundScope {
3942 owner: admission.owner,
3943 scope_ref: admission.stamp.scope_ref.clone(),
3944 tag: admission.tag,
3945 }),
3946 Some(admission.stamp),
3947 ),
3948 Err(refusal) => {
3949 return Ok(vec![self.route_open_refusal_frame(
3950 ctx,
3951 &frame,
3952 &target_module_id,
3953 refusal.code,
3954 refusal.code,
3955 refusal.message,
3956 )?]);
3957 }
3958 }
3959 }
3960 };
3961
3962 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
3984 Ok(project_root) => project_root,
3985 Err(err) => {
3986 return Ok(vec![control_error_frame(
3987 &frame,
3988 "invalid_project_root",
3989 err.to_string(),
3990 )?])
3991 }
3992 };
3993 identity.project_root = project_root.as_path().to_path_buf();
3994
3995 let _concurrency_guard = match self
4006 .route_bind_concurrency
4007 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4008 {
4009 Ok(guard) => guard,
4010 Err(in_flight) => {
4011 return Ok(vec![self.route_open_target_capacity_refusal(
4012 ctx,
4013 &frame,
4014 &target_module_id,
4015 in_flight,
4016 )?]);
4017 }
4018 };
4019
4020 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4023 RouteBindAdmission::Admitted { guard, probe } => {
4024 if probe {
4025 info!(
4026 module_id = %target_module_id,
4027 connection_id = ctx.connection_id.get(),
4028 "route.bind breaker half-open: admitting one probe"
4029 );
4030 }
4031 guard
4032 }
4033 RouteBindAdmission::Refused {
4034 consecutive_timeouts,
4035 retry_in,
4036 probe_in_flight,
4037 } => {
4038 return Ok(vec![self.route_open_breaker_refusal_frame(
4039 ctx,
4040 &frame,
4041 &target_module_id,
4042 consecutive_timeouts,
4043 retry_in,
4044 probe_in_flight,
4045 )?]);
4046 }
4047 };
4048
4049 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4054 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4055 let pending = match self
4056 .forwarding
4057 .begin_route_bind_relay_for(
4058 ctx.connection_id,
4059 ctx.egress.clone(),
4060 response_version(&frame),
4061 frame.header.corr,
4062 &target_module_id,
4063 principal.clone(),
4064 bound_scope,
4065 Some(project_root),
4066 relay_deadline,
4067 )
4068 .await
4069 {
4070 Ok(pending) => pending,
4071 Err(err) => {
4072 return Ok(vec![self.route_open_refusal_frame(
4073 ctx,
4074 &frame,
4075 &target_module_id,
4076 "relay_reservation_failed",
4077 forwarding_error_code(&err),
4078 err.to_string(),
4079 )?])
4080 }
4081 };
4082 let crate::forwarding::PendingRouteBindRelay {
4083 endpoint,
4084 module_sink,
4085 negotiated_ver,
4086 client_channel,
4087 client_epoch,
4088 module_channel,
4089 module_epoch,
4090 corr: relay_corr,
4091 receiver,
4092 } = pending;
4093 let mut reservation =
4094 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4095
4096 let relay_registration = self
4101 .registry
4102 .get_module_by_connection(endpoint.connection_id)
4103 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4104 let relay_capabilities = relay_registration
4105 .as_ref()
4106 .and_then(|registration| registration.manifest.capabilities.as_ref());
4107 if let Some(stamp) = scope_stamp
4108 .as_ref()
4109 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4110 {
4111 if let Err(refusal) = crate::scopes::check_target_flow_support(
4112 stamp,
4113 &target_module_id,
4114 relay_capabilities,
4115 )
4116 .and_then(|()| {
4117 crate::scopes::check_target_agent_run_support(
4118 stamp,
4119 &target_module_id,
4120 relay_capabilities,
4121 )
4122 }) {
4123 reservation.release_and_disarm();
4124 return Ok(vec![self.route_open_refusal_frame(
4125 ctx,
4126 &frame,
4127 &target_module_id,
4128 refusal.code,
4129 refusal.code,
4130 refusal.message,
4131 )?]);
4132 }
4133 }
4134
4135 let call_mode = if relay_capabilities
4138 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4139 {
4140 Some(derive_call_mode(scope_stamp.as_ref()))
4141 } else {
4142 if let Some(stamp) = scope_stamp.as_mut() {
4143 stamp.expires_at_ms = None;
4144 }
4145 None
4146 };
4147
4148 debug!(
4149 connection_id = ctx.connection_id.get(),
4150 client_channel,
4151 client_epoch,
4152 module_channel,
4153 module_epoch,
4154 "reserved route handle pair"
4155 );
4156 let principal_label = principal_label(&principal);
4159 let relay = ModuleControlRequest::RouteBind {
4160 route_channel: module_channel,
4161 epoch: module_epoch,
4162 target,
4163 identity,
4164 principal: Some(principal),
4165 consumer_capabilities,
4166 role_versions,
4167 admission_facts,
4168 scope: scope_stamp,
4169 call_mode,
4170 };
4171 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4172 RouterError::backend(
4173 0,
4174 frame.header.corr,
4175 format!("failed to encode route.bind request: {err}"),
4176 )
4177 })?;
4178 let relay_frame = Frame::build_with_version(
4179 negotiated_ver,
4180 FrameType::Request,
4181 control_flags(),
4182 0,
4183 0,
4184 relay_corr,
4185 relay_body,
4186 )
4187 .map_err(RouterError::FrameBuild)?;
4188
4189 if let Err(err) = module_sink.send(relay_frame).await {
4190 reservation.release_and_disarm();
4191 return Ok(vec![self.route_open_refusal_frame(
4192 ctx,
4193 &frame,
4194 &target_module_id,
4195 "relay_send_failed",
4196 "target_unavailable",
4197 err.to_string(),
4198 )?]);
4199 }
4200
4201 #[cfg(test)]
4202 if let Some(hold) = &self.route_bind_relay_sent_hold {
4203 hold.wait().await;
4207 hold.wait().await;
4208 }
4209
4210 if self
4215 .forwarding
4216 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4217 .map_err(RouterError::Forwarding)?
4218 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4219 {
4220 self.send_abandoned_route_bind_goodbye(
4221 &module_sink,
4222 negotiated_ver,
4223 module_channel,
4224 module_epoch,
4225 );
4226 }
4227
4228 match timeout_at(relay_deadline, receiver).await {
4229 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4230 reservation.disarm();
4231 if breaker.record_accepted() {
4232 info!(
4233 module_id = %target_module_id,
4234 "route.bind breaker closed: the probe was accepted"
4235 );
4236 }
4237 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
4238 Ok(Vec::new())
4239 }
4240 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4241 reservation.release_and_disarm();
4242 breaker.record_inconclusive();
4246 let scope_code = match body.code.as_str() {
4250 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4251 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4252 _ => None,
4253 };
4254 if let Some(code) = scope_code {
4255 self.observe_route_open_refusal(
4256 ctx,
4257 &target_module_id,
4258 "scope_changed_before_commit",
4259 code,
4260 );
4261 return Ok(vec![control_error_body_frame(&frame, body)?]);
4262 }
4263 self.counters
4264 .increment_route_open_refused("module_rejected");
4265 info!(
4266 target: "control",
4267 code = "module_rejected",
4268 module_code = ?body.code,
4269 module_id = ?target_module_id,
4270 connection_id = ctx.connection_id.get(),
4271 "route.open refused"
4272 );
4273 Ok(vec![control_error_body_frame(&frame, body)?])
4274 }
4275 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
4276 reservation.release_and_disarm();
4277 breaker.record_inconclusive();
4278 tracing::warn!(
4285 module_id = %target_module_id,
4286 "route.bind relay abandoned: {message}"
4287 );
4288 Ok(vec![self.route_open_refusal_frame(
4289 ctx,
4290 &frame,
4291 &target_module_id,
4292 "relay_abandoned",
4293 "target_unavailable",
4294 message,
4295 )?])
4296 }
4297 Ok(Err(_)) => {
4298 reservation.release_and_disarm();
4299 breaker.record_inconclusive();
4300 Ok(vec![self.route_open_refusal_frame(
4301 ctx,
4302 &frame,
4303 &target_module_id,
4304 "relay_waiter_canceled",
4305 "target_unavailable",
4306 "route.bind relay waiter was canceled before the module responded",
4307 )?])
4308 }
4309 Err(_) => {
4310 reservation.release_and_disarm();
4311 if let Some(opened) = breaker.record_timeout(
4315 self.route_bind_breaker_threshold,
4316 self.route_bind_breaker_cooldown,
4317 ) {
4318 warn!(
4319 module_id = %target_module_id,
4320 consecutive_timeouts = opened.consecutive_timeouts,
4321 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
4322 reopened_after_probe = opened.reopened_after_probe,
4323 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
4324 );
4325 }
4326 tracing::warn!(
4334 module_id = %target_module_id,
4335 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
4336 "route.bind relay timed out: module did not ack within budget"
4337 );
4338 Ok(vec![self.route_open_refusal_frame(
4339 ctx,
4340 &frame,
4341 &target_module_id,
4342 "relay_timed_out",
4343 "module_timeout",
4344 format!(
4345 "module_id '{target_module_id}' did not answer route.bind within {:?}",
4346 route_bind_relay_timeout
4347 ),
4348 )?])
4349 }
4350 }
4351 }
4352
4353 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4354 let response = ClientControlResponse::SupervisorSpawnSnapshot {
4355 snapshot: self.supervisor.spawn_snapshot(),
4356 };
4357 Ok(vec![control_response_body_frame(
4358 &frame,
4359 &response,
4360 "ClientControlResponse::SupervisorSpawnSnapshot",
4361 )?])
4362 }
4363
4364 fn handle_supervisor_spawn_subscribe(
4365 &self,
4366 ctx: &RouteCtx,
4367 frame: Frame,
4368 since: Option<SpawnCursor>,
4369 ) -> Result<Vec<Frame>, RouterError> {
4370 match self.supervisor.subscribe_spawns(
4371 ctx.connection_id,
4372 frame.header.corr,
4373 response_version(&frame),
4374 since,
4375 ctx.egress.clone(),
4376 ) {
4377 Ok(()) => Ok(Vec::new()),
4378 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
4379 Ok(vec![control_error_body_frame(
4380 &frame,
4381 ErrorBody {
4382 code: "spawn_cursor_incarnation_mismatch".to_string(),
4383 message: "spawn cursor belongs to a different daemon incarnation"
4384 .to_string(),
4385 detail: Some(serde_json::json!({
4386 "current_daemon_incarnation": current
4387 })),
4388 },
4389 )?])
4390 }
4391 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
4392 &frame,
4393 ErrorBody {
4394 code: "spawn_cursor_too_old".to_string(),
4395 message: "spawn cursor predates the retained event ring".to_string(),
4396 detail: Some(serde_json::json!({
4397 "oldest_retained_cursor": oldest
4398 })),
4399 },
4400 )?]),
4401 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
4402 0,
4403 frame.header.corr,
4404 format!("failed to open supervisor spawn subscription: {error}"),
4405 )),
4406 }
4407 }
4408
4409 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4410 let generation = self
4411 .registry
4412 .generation()
4413 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4414 let mut modules = Vec::new();
4415 for module in self.supervisor.list() {
4416 let status = module.status_for_control("list").map_err(|err| {
4417 RouterError::backend(
4418 0,
4419 frame.header.corr,
4420 format!("failed to read supervisor status: {err}"),
4421 )
4422 })?;
4423 let (configured, _) = module.configuration().map_err(|err| {
4424 RouterError::backend(
4425 0,
4426 frame.header.corr,
4427 format!("failed to read module configuration: {err}"),
4428 )
4429 })?;
4430 let image = module.running_image_agreement().await;
4432 let resources = Some(module.child_resource_usage());
4435 let pending_reload = Some(reload_verdict(
4436 &configured.program,
4437 status.spawned_from.as_deref(),
4438 image,
4439 ));
4440 modules.push(
4443 SupervisorEntry::new(
4444 status.module_id,
4445 status.state.to_string(),
4446 status.enabled,
4447 status.live,
4448 status.health.status,
4449 )
4450 .with_launch_nonce_env(Some(!cfg!(unix)))
4451 .with_launch_nonce_source(status.launch_nonce_source)
4452 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
4453 .with_protocol(status.protocol)
4454 .with_pending_reload(pending_reload)
4455 .with_last_probe_ms(status.health.last_probe_ms)
4456 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
4457 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
4458 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
4459 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
4460 .with_restart_count(Some(status.restart_count))
4461 .with_max_restarts(Some(status.max_restarts))
4462 .with_lifetime_restarts(Some(status.lifetime_restarts))
4463 .with_spawn_generation(Some(status.spawn_generation))
4464 .with_restart_window_secs(Some(status.restart_window.as_secs()))
4465 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
4466 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
4467 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
4468 .with_resources(resources),
4469 );
4470 }
4471 let response = ClientControlResponse::SupervisorList {
4472 generation,
4473 modules,
4474 };
4475 Ok(vec![control_response_body_frame(
4476 &frame,
4477 &response,
4478 "ClientControlResponse::SupervisorList",
4479 )?])
4480 }
4481
4482 fn handle_supervisor_stderr_tail(
4483 &self,
4484 frame: Frame,
4485 module_id: String,
4486 max_lines: Option<u32>,
4487 max_bytes: Option<u32>,
4488 ) -> Result<Vec<Frame>, RouterError> {
4489 let Some(module) = self.supervisor.get(&module_id) else {
4490 return Ok(vec![control_error_frame(
4491 &frame,
4492 "unknown_module",
4493 format!("module_id '{module_id}' is not supervised"),
4494 )?]);
4495 };
4496
4497 let snapshot = module.stderr_tail(
4498 max_lines.map(|value| value as usize),
4499 max_bytes.map(|value| value as usize),
4500 );
4501
4502 let response = ClientControlResponse::SupervisorStderrTail {
4503 module_id,
4504 tail: StderrTail {
4505 capture: match snapshot.capture {
4506 CaptureState::Captured => StderrCaptureState::Captured,
4507 CaptureState::Incomplete { reason } => {
4508 StderrCaptureState::Incomplete { reason }
4509 }
4510 CaptureState::NotCaptured { reason } => {
4511 StderrCaptureState::NotCaptured { reason }
4512 }
4513 },
4514 entries: snapshot
4515 .entries
4516 .into_iter()
4517 .map(|entry| match entry {
4518 TailEntry::Line {
4519 text,
4520 truncated,
4521 at_ms,
4522 } => StderrTailEntry::Line {
4523 text,
4524 truncated,
4525 at_ms,
4526 },
4527 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
4528 })
4529 .collect(),
4530 dropped_lines: snapshot.dropped_lines,
4531 },
4532 };
4533 Ok(vec![control_response_body_frame(
4534 &frame,
4535 &response,
4536 "ClientControlResponse::SupervisorStderrTail",
4537 )?])
4538 }
4539
4540 async fn handle_supervisor_terminals(
4541 &self,
4542 frame: Frame,
4543 module_id: String,
4544 ) -> Result<Vec<Frame>, RouterError> {
4545 let Some(module) = self.supervisor.get(&module_id) else {
4546 return Ok(vec![control_error_frame(
4547 &frame,
4548 "unknown_module",
4549 format!("module_id '{module_id}' is not supervised"),
4550 )?]);
4551 };
4552
4553 let terminals = module
4556 .read_durable_terminal_history()
4557 .await
4558 .map_err(|error| {
4559 RouterError::backend(
4560 0,
4561 frame.header.corr,
4562 format!("failed to read terminal history: {error}"),
4563 )
4564 })?;
4565 let response = ClientControlResponse::SupervisorTerminals {
4566 module_id,
4567 terminals,
4568 };
4569 Ok(vec![control_response_body_frame(
4570 &frame,
4571 &response,
4572 "ClientControlResponse::SupervisorTerminals",
4573 )?])
4574 }
4575
4576 fn handle_supervisor_routes(
4577 &self,
4578 frame: Frame,
4579 module_id: Option<String>,
4580 ) -> Result<Vec<Frame>, RouterError> {
4581 let modules = self
4582 .forwarding
4583 .route_census(module_id.as_deref())
4584 .map_err(RouterError::Forwarding)?
4585 .into_iter()
4586 .map(|(module_id, routes)| SupervisorRouteModule {
4587 module_id,
4588 routes: routes
4589 .into_iter()
4590 .map(|route| SupervisorRoute {
4591 consumer: match route.principal {
4592 Principal::Reserved { module_id } => {
4593 SupervisorRouteConsumer::Reserved { module_id }
4594 }
4595 Principal::Direct | Principal::Unverified => {
4596 SupervisorRouteConsumer::Direct {
4597 connection_id: route.goodbye_target.connection_id.get(),
4598 }
4599 }
4600 },
4601 age_ms: Instant::now()
4602 .saturating_duration_since(route.bound_at)
4603 .as_millis()
4604 .try_into()
4605 .unwrap_or(u64::MAX),
4606 draining: route.draining,
4607 drain_reason: route.drain_reason,
4608 })
4609 .collect(),
4610 })
4611 .collect();
4612 let response = ClientControlResponse::SupervisorRoutes { modules };
4613 Ok(vec![control_response_body_frame(
4614 &frame,
4615 &response,
4616 "ClientControlResponse::SupervisorRoutes",
4617 )?])
4618 }
4619
4620 async fn handle_supervisor_provenance(
4621 &self,
4622 frame: Frame,
4623 module_id: Option<String>,
4624 ) -> Result<Vec<Frame>, RouterError> {
4625 let mut selected = if let Some(module_id) = module_id {
4626 let Some(module) = self.supervisor.get(&module_id) else {
4627 return Ok(vec![control_error_frame(
4628 &frame,
4629 "unknown_module",
4630 format!("module_id '{module_id}' is not supervised"),
4631 )?]);
4632 };
4633 vec![module]
4634 } else {
4635 self.supervisor.list()
4636 };
4637
4638 let mut modules = Vec::with_capacity(selected.len());
4639 for module in selected.drain(..) {
4640 let (status, observed_image) = module
4641 .status_and_running_image_agreement()
4642 .await
4643 .map_err(|err| {
4644 RouterError::backend(
4645 0,
4646 frame.header.corr,
4647 format!("failed to read supervisor status: {err}"),
4648 )
4649 })?;
4650 let module_declared = self
4651 .registry
4652 .get_module(&status.module_id)
4653 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4654 .and_then(|registration| registration.manifest.provenance)
4655 .map(|build| ModuleDeclaredProvenance::Reported { build })
4656 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
4657 #[cfg(test)]
4658 let running_image = match &self.provenance_probe_override {
4659 Some(result) => result.clone(),
4660 None => observed_image,
4661 };
4662 #[cfg(not(test))]
4663 let running_image = observed_image;
4664 modules.push(SupervisorModuleProvenance {
4665 module_id: status.module_id,
4666 module_declared,
4667 daemon_observed: SupervisorObservedProcess {
4668 launch_nonce_source: status.launch_nonce_source,
4669 pid: status.pid,
4670 spawned_at_ms: status.spawned_at_ms,
4671 spawned_from: status.spawned_from,
4672 running_image,
4673 },
4674 });
4675 }
4676 let daemon = SupervisorDaemonProvenance {
4677 daemon_build: self.daemon_provenance.build.clone(),
4678 daemon_observed: DaemonObservedProcess {
4679 pid: self.daemon_provenance.pid,
4680 started_at_ms: self
4681 .daemon_provenance
4682 .start_clock
4683 .map(|clock| clock.started_at_ms())
4684 .or(self.daemon_provenance.started_at_ms),
4685 running_image: self
4686 .daemon_provenance
4687 .probe
4688 .observe(
4689 self.daemon_provenance.pid,
4690 self.daemon_provenance.executable_path.as_deref(),
4691 self.daemon_provenance.executable_identity.clone(),
4692 self.daemon_provenance.process_start_time,
4693 )
4694 .await,
4695 },
4696 };
4697 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
4698 Ok(vec![control_response_body_frame(
4699 &frame,
4700 &response,
4701 "ClientControlResponse::SupervisorProvenance",
4702 )?])
4703 }
4704
4705 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4706 self.refresh_capability_requirements();
4707 let generation = self
4708 .registry
4709 .generation()
4710 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4711 let modules = self
4712 .supervisor
4713 .list()
4714 .into_iter()
4715 .map(|module| {
4716 let status = module.status_for_control("health").map_err(|err| {
4717 RouterError::backend(
4718 0,
4719 frame.header.corr,
4720 format!("failed to read supervisor health: {err}"),
4721 )
4722 })?;
4723 let module_id = status.module_id;
4724 let capability_detail = self
4725 .capability_evaluator
4726 .required_problem_detail(&module_id);
4727 Ok(SupervisorHealthEntry {
4728 module_id,
4729 status: status.health.status,
4730 detail: append_capability_problem_detail(
4731 status.health.detail,
4732 capability_detail,
4733 ),
4734 metrics: status.health.metrics,
4735 consecutive_failures: status.health.consecutive_failures,
4736 late_answer_count: status.health.late_answer_count,
4737 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
4738 last_action: status.health.last_action,
4739 last_action_ms: status.health.last_action_ms,
4740 last_probe_ms: status.health.last_probe_ms,
4741 })
4742 })
4743 .collect::<Result<Vec<_>, RouterError>>()?;
4744 let response = ClientControlResponse::SupervisorHealth {
4745 generation,
4746 modules,
4747 };
4748 Ok(vec![control_response_body_frame(
4749 &frame,
4750 &response,
4751 "ClientControlResponse::SupervisorHealth",
4752 )?])
4753 }
4754
4755 async fn handle_supervisor_restart(
4756 &self,
4757 frame: Frame,
4758 module_id: String,
4759 drain_timeout_ms: Option<u64>,
4760 ) -> Result<Vec<Frame>, RouterError> {
4761 let operation_lock = self.supervisor.operation_lock();
4762 let _operation_guard = operation_lock.lock().await;
4763 let Some(module) = self.supervisor.get(&module_id) else {
4764 return Ok(vec![control_error_frame(
4765 &frame,
4766 "unknown_module",
4767 format!("module_id '{module_id}' is not supervised"),
4768 )?]);
4769 };
4770
4771 self.route_outages.mark_operator_action(&module_id);
4772 if let Err(err) = module.restart(drain_timeout_ms).await {
4773 self.route_outages
4774 .operator_action_ended_unrefused(&module_id);
4775 let (code, message) = match err {
4776 crate::supervise::SuperviseError::Disabled { .. } => {
4777 ("module_disabled", err.to_string())
4778 }
4779 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4780 ("swap_in_progress", err.to_string())
4781 }
4782 _ => (
4783 "target_unavailable",
4784 format!("failed to restart module_id '{module_id}': {err}"),
4785 ),
4786 };
4787 return Ok(vec![control_error_frame(&frame, code, message)?]);
4788 }
4789
4790 let response = ClientControlResponse::SupervisorAck {
4791 module_id,
4792 applied: true,
4793 };
4794 Ok(vec![control_response_body_frame(
4795 &frame,
4796 &response,
4797 "ClientControlResponse::SupervisorAck",
4798 )?])
4799 }
4800
4801 async fn handle_supervisor_swap(
4805 &self,
4806 frame: Frame,
4807 module_id: String,
4808 ready_timeout_ms: Option<u64>,
4809 ) -> Result<Vec<Frame>, RouterError> {
4810 let module = {
4817 let operation_lock = self.supervisor.operation_lock();
4818 let _operation_guard = operation_lock.lock().await;
4819 self.supervisor.get(&module_id)
4820 };
4821 let Some(module) = module else {
4822 return Ok(vec![control_error_frame(
4823 &frame,
4824 "unknown_module",
4825 format!("module_id '{module_id}' is not supervised"),
4826 )?]);
4827 };
4828
4829 self.route_outages.mark_operator_action(&module_id);
4830 if let Err(err) = module
4831 .swap(ready_timeout_ms.map(Duration::from_millis))
4832 .await
4833 {
4834 self.route_outages
4835 .operator_action_ended_unrefused(&module_id);
4836 use crate::supervise::SuperviseError;
4837 let message = err.to_string();
4838 let error = match err {
4839 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
4840 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
4841 code: "swap_refused".to_string(),
4842 message,
4843 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
4844 },
4845 SuperviseError::SwapFailed {
4846 arm,
4847 candidate_exit,
4848 ..
4849 } => ErrorBody {
4850 code: "swap_failed".to_string(),
4851 message,
4852 detail: Some(serde_json::json!({
4853 "arm": arm.as_str(),
4854 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
4855 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
4856 })),
4857 },
4858 _ => ErrorBody::new(
4859 "target_unavailable",
4860 format!("failed to swap module_id '{module_id}': {message}"),
4861 ),
4862 };
4863 return Ok(vec![control_error_body_frame(&frame, error)?]);
4864 }
4865 self.route_outages
4869 .operator_action_ended_unrefused(&module_id);
4870
4871 let response = ClientControlResponse::SupervisorAck {
4872 module_id,
4873 applied: true,
4874 };
4875 Ok(vec![control_response_body_frame(
4876 &frame,
4877 &response,
4878 "ClientControlResponse::SupervisorAck",
4879 )?])
4880 }
4881
4882 async fn handle_supervisor_reload(
4883 &self,
4884 frame: Frame,
4885 module_id: String,
4886 ) -> Result<Vec<Frame>, RouterError> {
4887 let operation_lock = self.supervisor.operation_lock();
4888 let _operation_guard = operation_lock.lock().await;
4889 let Some(module) = self.supervisor.get(&module_id) else {
4890 return Ok(vec![control_error_frame(
4891 &frame,
4892 "unknown_module",
4893 format!("module_id '{module_id}' is not supervised"),
4894 )?]);
4895 };
4896
4897 self.route_outages.mark_operator_action(&module_id);
4898 if let Err(err) = module.reload().await {
4899 self.route_outages
4900 .operator_action_ended_unrefused(&module_id);
4901 let (code, message) = match err {
4902 crate::supervise::SuperviseError::Disabled { .. } => {
4903 ("module_disabled", err.to_string())
4904 }
4905 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4906 ("swap_in_progress", err.to_string())
4907 }
4908 _ => (
4909 "reload_failed",
4910 format!("failed to reload module_id '{module_id}': {err}"),
4911 ),
4912 };
4913 return Ok(vec![control_error_frame(&frame, code, message)?]);
4914 }
4915
4916 let response = ClientControlResponse::SupervisorAck {
4917 module_id,
4918 applied: true,
4919 };
4920 Ok(vec![control_response_body_frame(
4921 &frame,
4922 &response,
4923 "ClientControlResponse::SupervisorAck",
4924 )?])
4925 }
4926
4927 async fn handle_supervisor_rescan(
4928 &self,
4929 frame: Frame,
4930 preview: bool,
4931 ) -> Result<Vec<Frame>, RouterError> {
4932 let Some(context) = self.rescan.clone() else {
4933 return Ok(vec![control_error_frame(
4934 &frame,
4935 "rescan_unavailable",
4936 "the daemon was not started with a reloadable config path".to_string(),
4937 )?]);
4938 };
4939
4940 let operation_lock = self.supervisor.operation_lock();
4941 let _operation_guard = operation_lock.lock().await;
4942 let loaded = match crate::daemon_config::load(&context.config_path) {
4943 Ok(config) => config,
4944 Err(err) => {
4945 return Ok(vec![control_error_frame(
4946 &frame,
4947 "invalid_daemon_config",
4948 format!("supervisor rescan rejected daemon config: {err}"),
4949 )?])
4950 }
4951 };
4952 let Some(config) = loaded else {
4960 return Ok(vec![control_error_frame(
4961 &frame,
4962 "invalid_daemon_config",
4963 format!(
4964 "daemon config not found at {}; refusing to rescan (an absent config would \
4965 retire every supervised module)",
4966 context.config_path.display()
4967 ),
4968 )?]);
4969 };
4970 let (
4971 configured_port,
4972 storage_config,
4973 admission_facts_carrier_module_id,
4974 admission_facts_targets,
4975 scope_authority_owners,
4976 modules,
4977 reserved_capabilities,
4978 ) = (
4979 config.port,
4980 config.storage,
4981 config.admission_facts_carrier_module_id,
4982 config.admission_facts_targets,
4983 config.scope_authority_owners,
4984 config.modules,
4985 config.reserved_capabilities,
4986 );
4987
4988 let mut restart_required = Vec::new();
4997 for section in RestartRequiredSection::ALL {
4998 let changed = match section {
4999 RestartRequiredSection::Port => configured_port != context.configured_port,
5000 RestartRequiredSection::Storage => storage_config != context.storage_config,
5001 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5002 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5003 }
5004 RestartRequiredSection::AdmissionFactsTargets => {
5005 admission_facts_targets != context.admission_facts_targets
5006 }
5007 RestartRequiredSection::ScopeAuthorityOwners => {
5008 scope_authority_owners != context.scope_authority_owners
5009 }
5010 };
5011 if changed {
5012 restart_required.push(section.label().to_string());
5013 }
5014 }
5015 if !restart_required.is_empty() {
5016 warn!(
5017 config_path = %context.config_path.display(),
5018 sections = %restart_required.join(", "),
5019 "daemon config changed outside the modules section; restart the daemon to apply those changes"
5020 );
5021 }
5022
5023 for configured in &modules {
5024 if let Err(err) = validate_spec(&configured.module_spec()) {
5025 return Ok(vec![control_error_frame(
5026 &frame,
5027 "invalid_daemon_config",
5028 format!("supervisor rescan rejected daemon config: {err}"),
5029 )?]);
5030 }
5031 }
5032
5033 let configured_capabilities = modules
5034 .iter()
5035 .map(|module| (module.module_id.clone(), module.enabled))
5036 .collect::<Vec<_>>();
5037 let preview_capability_warnings = if preview {
5038 let (_, registrations) = self.runtime_capability_snapshot()?;
5039 let current_modules = self
5040 .supervisor
5041 .list()
5042 .into_iter()
5043 .map(|module| module.module_id().to_string())
5044 .collect::<BTreeSet<_>>();
5045 let resulting_modules = configured_capabilities.clone();
5046 let removed = current_modules
5047 .into_iter()
5048 .filter(|module_id| {
5049 !resulting_modules
5050 .iter()
5051 .any(|(configured_id, _)| configured_id == module_id)
5052 })
5053 .collect::<Vec<_>>();
5054 self.capability_evaluator.preview_removal_warnings(
5055 resulting_modules,
5056 &removed,
5057 ®istrations,
5058 )
5059 } else {
5060 Vec::new()
5061 };
5062 let result = match self
5063 .reconcile_supervised_modules(&context.supervisor, modules, preview)
5064 .await
5065 {
5066 Ok(result) => result,
5067 Err(message) => {
5068 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5069 }
5070 };
5071 if !preview {
5072 self.capability_evaluator
5073 .configure(configured_capabilities, reserved_capabilities);
5074 self.capability_evaluator.wake_deadline_loop();
5075 self.refresh_capability_requirements();
5076 }
5077 let mut result = result;
5078 result.restart_required = restart_required;
5079 result.capability_warnings = preview_capability_warnings;
5080 let response = ClientControlResponse::SupervisorRescan { result };
5081 Ok(vec![control_response_body_frame(
5082 &frame,
5083 &response,
5084 "ClientControlResponse::SupervisorRescan",
5085 )?])
5086 }
5087
5088 async fn handle_supervisor_release_reserved(
5089 &self,
5090 frame: Frame,
5091 module_id: String,
5092 ) -> Result<Vec<Frame>, RouterError> {
5093 let Some(context) = self.rescan.clone() else {
5094 return Ok(vec![control_error_frame(
5095 &frame,
5096 "release_unavailable",
5097 "reserved-id release requires a daemon started with a reloadable config path",
5098 )?]);
5099 };
5100 let operation_lock = self.supervisor.operation_lock();
5101 let _operation_guard = operation_lock.lock().await;
5102 let loaded = match crate::daemon_config::load(&context.config_path) {
5103 Ok(Some(config)) => config,
5104 Ok(None) => {
5105 return Ok(vec![control_error_frame(
5106 &frame,
5107 "invalid_daemon_config",
5108 format!(
5109 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5110 context.config_path.display()
5111 ),
5112 )?])
5113 }
5114 Err(err) => {
5115 return Ok(vec![control_error_frame(
5116 &frame,
5117 "invalid_daemon_config",
5118 format!("unable to verify reserved-id release against daemon config: {err}"),
5119 )?])
5120 }
5121 };
5122 if loaded
5123 .modules
5124 .iter()
5125 .any(|configured| configured.module_id == module_id)
5126 {
5127 return Ok(vec![control_error_frame(
5128 &frame,
5129 "reserved_module_configured",
5130 format!(
5131 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5132 ),
5133 )?]);
5134 }
5135 if !self.supervisor.release_retained_reserved_gate(&module_id) {
5136 return Ok(vec![control_error_frame(
5137 &frame,
5138 "reserved_gate_not_retained",
5139 format!(
5140 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
5141 ),
5142 )?]);
5143 }
5144
5145 let response = ClientControlResponse::SupervisorAck {
5146 module_id,
5147 applied: true,
5148 };
5149 Ok(vec![control_response_body_frame(
5150 &frame,
5151 &response,
5152 "ClientControlResponse::SupervisorAck",
5153 )?])
5154 }
5155
5156 async fn reconcile_supervised_modules(
5165 &self,
5166 supervisor: &Supervisor,
5167 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
5168 preview: bool,
5169 ) -> Result<SupervisorRescanResult, String> {
5170 let mut current = BTreeMap::new();
5171 for module in self.supervisor.list() {
5172 let (spec, health) = module.configuration().map_err(|err| {
5173 format!(
5174 "failed to read configuration for module_id '{}': {err}",
5175 module.module_id()
5176 )
5177 })?;
5178 let enabled = module
5179 .status()
5180 .map_err(|err| {
5181 format!(
5182 "failed to read status for module_id '{}': {err}",
5183 module.module_id()
5184 )
5185 })?
5186 .enabled;
5187 current.insert(
5188 module.module_id().to_string(),
5189 (module, spec, health, enabled),
5190 );
5191 }
5192 let configured = configured_modules
5193 .into_iter()
5194 .map(|module| (module.module_id.clone(), module))
5195 .collect::<BTreeMap<_, _>>();
5196
5197 let added = configured
5198 .keys()
5199 .filter(|module_id| !current.contains_key(*module_id))
5200 .cloned()
5201 .collect::<Vec<_>>();
5202 let removed = current
5203 .keys()
5204 .filter(|module_id| !configured.contains_key(*module_id))
5205 .cloned()
5206 .collect::<Vec<_>>();
5207 let mut changed_pending_reload = Vec::new();
5208 let mut configuration_changes = BTreeSet::new();
5209 let mut enabled_changes = BTreeSet::new();
5210 let mut unchanged = 0_u32;
5211
5212 for (module_id, configured_module) in &configured {
5213 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5214 else {
5215 continue;
5216 };
5217 let launch_changed = *current_spec != configured_module.module_spec();
5221 let configuration_changed =
5222 launch_changed || *current_health != configured_module.health;
5223 let enabled_changed = *current_enabled != configured_module.enabled;
5224 if configuration_changed {
5225 configuration_changes.insert(module_id.clone());
5226 }
5227 if launch_changed {
5228 changed_pending_reload.push(module_id.clone());
5229 }
5230 if enabled_changed {
5231 enabled_changes.insert(module_id.clone());
5232 }
5233 if !configuration_changed && !enabled_changed {
5234 unchanged = unchanged.saturating_add(1);
5235 }
5236 }
5237
5238 if preview {
5244 return Ok(SupervisorRescanResult {
5245 added,
5246 removed,
5247 changed_pending_reload,
5248 enabled_changes: enabled_changes.iter().cloned().collect(),
5249 unchanged,
5250 preview: true,
5251 restart_required: Vec::new(),
5255 capability_warnings: Vec::new(),
5256 });
5257 }
5258
5259 for module_id in &removed {
5260 let module = ¤t
5261 .get(module_id)
5262 .expect("removed module came from current supervisor state")
5263 .0;
5264 module.retire().await.map_err(|err| {
5265 format!("failed to retire module_id '{module_id}' during rescan: {err}")
5266 })?;
5267 self.supervisor.record_rescan_removal(module_id);
5301 self.supervisor.retire(module_id);
5302 self.route_outages.forget(module_id);
5303 }
5304
5305 for module_id in configured.keys() {
5306 let Some((module, _, _, _)) = current.get(module_id) else {
5307 continue;
5308 };
5309 let configured_module = configured
5310 .get(module_id)
5311 .expect("configured module id came from configured map");
5312 if configuration_changes.contains(module_id) {
5313 module
5314 .update_configuration(
5315 configured_module.module_spec(),
5316 configured_module.health.clone(),
5317 configured_module.drain_timeout_ms,
5318 )
5319 .await
5320 .map_err(|err| {
5321 format!(
5322 "failed to update module_id '{module_id}' configuration during rescan: {err}"
5323 )
5324 })?;
5325 }
5326 if enabled_changes.contains(module_id) {
5327 self.route_outages.mark_operator_action(module_id);
5330 module
5331 .set_enabled(configured_module.enabled)
5332 .await
5333 .map_err(|err| {
5334 self.route_outages.operator_action_ended_unrefused(module_id);
5335 format!(
5336 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
5337 configured_module.enabled
5338 )
5339 })?;
5340 }
5341 }
5342
5343 for module_id in &added {
5344 let configured_module = configured
5345 .get(module_id)
5346 .expect("added module id came from configured map");
5347 supervisor
5348 .supervise_configured_with_health(
5349 configured_module.module_spec(),
5350 configured_module.enabled,
5351 configured_module.health.clone(),
5352 configured_module.drain_timeout_ms,
5353 configured_module.restart,
5354 )
5355 .map_err(|err| {
5356 format!("failed to add module_id '{module_id}' during rescan: {err}")
5357 })?;
5358 }
5359
5360 Ok(SupervisorRescanResult {
5361 added,
5362 removed,
5363 changed_pending_reload,
5364 enabled_changes: enabled_changes.iter().cloned().collect(),
5365 unchanged,
5366 preview: false,
5367 restart_required: Vec::new(),
5370 capability_warnings: Vec::new(),
5371 })
5372 }
5373
5374 async fn handle_supervisor_set_enabled(
5375 &self,
5376 frame: Frame,
5377 module_id: String,
5378 enabled: bool,
5379 ) -> Result<Vec<Frame>, RouterError> {
5380 let operation_lock = self.supervisor.operation_lock();
5381 let _operation_guard = operation_lock.lock().await;
5382 let Some(module) = self.supervisor.get(&module_id) else {
5383 return Ok(vec![control_error_frame(
5384 &frame,
5385 "unknown_module",
5386 format!("module_id '{module_id}' is not supervised"),
5387 )?]);
5388 };
5389
5390 self.route_outages.mark_operator_action(&module_id);
5393 let applied = match module.set_enabled(enabled).await {
5394 Ok(applied) => applied,
5395 Err(err) => {
5396 self.route_outages
5397 .operator_action_ended_unrefused(&module_id);
5398 return Ok(vec![control_error_frame(
5399 &frame,
5400 "target_unavailable",
5401 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
5402 )?]);
5403 }
5404 };
5405 if !applied {
5406 self.route_outages
5409 .operator_action_ended_unrefused(&module_id);
5410 }
5411
5412 self.capability_evaluator.wake_deadline_loop();
5413 self.refresh_capability_requirements();
5414 let response = ClientControlResponse::SupervisorAck { module_id, applied };
5415 Ok(vec![control_response_body_frame(
5416 &frame,
5417 &response,
5418 "ClientControlResponse::SupervisorAck",
5419 )?])
5420 }
5421
5422 async fn handle_supervisor_health_probe(
5423 &self,
5424 frame: Frame,
5425 module_id: String,
5426 ) -> Result<Vec<Frame>, RouterError> {
5427 self.refresh_capability_requirements();
5428 let Some(registration) = self
5429 .registry
5430 .get_module(&module_id)
5431 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5432 else {
5433 return Ok(vec![control_error_frame(
5434 &frame,
5435 "unknown_module",
5436 format!("module_id '{module_id}' is not registered"),
5437 )?]);
5438 };
5439
5440 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
5453 {
5454 return Ok(vec![control_error_frame(
5455 &frame,
5456 "health_not_advertised",
5457 format!("module_id '{module_id}' did not advertise health.check"),
5458 )?]);
5459 }
5460
5461 let deadline = Instant::now() + self.health_probe_timeout;
5462 let pending = match self.forwarding.begin_module_control_rpc_for(
5463 &module_id,
5464 MODULE_CONTROL_OP_HEALTH_CHECK,
5465 deadline,
5466 ) {
5467 Ok(pending) => pending,
5468 Err(err) => {
5469 return Ok(vec![control_error_frame(
5470 &frame,
5471 forwarding_error_code(&err),
5472 err.to_string(),
5473 )?])
5474 }
5475 };
5476
5477 let PendingModuleControlRpc {
5478 endpoint,
5479 module_sink,
5480 negotiated_ver,
5481 corr: probe_corr,
5482 receiver,
5483 } = pending;
5484 let mut guard =
5485 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
5486 let probe_body =
5487 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5488 RouterError::backend(
5489 0,
5490 frame.header.corr,
5491 format!("failed to encode health.check request: {err}"),
5492 )
5493 })?;
5494 let probe_frame = Frame::build_with_version(
5495 negotiated_ver,
5496 FrameType::Request,
5497 control_flags(),
5498 0,
5499 0,
5500 probe_corr,
5501 probe_body,
5502 )
5503 .map_err(RouterError::FrameBuild)?;
5504
5505 if let Err(err) = module_sink.send(probe_frame).await {
5506 return Ok(vec![control_error_frame(
5507 &frame,
5508 "target_unavailable",
5509 err.to_string(),
5510 )?]);
5511 }
5512
5513 match timeout_at(deadline, receiver).await {
5514 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5515 guard.disarm();
5516 let Some(report) = response.health_report() else {
5517 return Ok(vec![control_error_frame(
5518 &frame,
5519 "invalid_control_body",
5520 "health.check RPC returned a non-health response",
5521 )?]);
5522 };
5523 let HealthReport {
5528 status,
5529 detail,
5530 metrics,
5531 } = report;
5532 let capability_detail = self
5533 .capability_evaluator
5534 .required_problem_detail(&module_id);
5535 let response = ClientControlResponse::SupervisorHealthProbe {
5536 module_id,
5537 status,
5538 detail: append_capability_problem_detail(detail, capability_detail),
5539 metrics,
5540 };
5541 Ok(vec![control_response_body_frame(
5542 &frame,
5543 &response,
5544 "ClientControlResponse::SupervisorHealthProbe",
5545 )?])
5546 }
5547 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
5548 guard.disarm();
5549 Ok(vec![control_error_body_frame(&frame, body)?])
5550 }
5551 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5552 guard.disarm();
5553 Ok(vec![control_error_frame(
5554 &frame,
5555 "target_unavailable",
5556 message,
5557 )?])
5558 }
5559 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5560 guard.disarm();
5561 Ok(vec![control_error_frame(
5562 &frame,
5563 "invalid_control_body",
5564 message,
5565 )?])
5566 }
5567 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5568 guard.disarm();
5569 Ok(vec![control_error_frame(
5570 &frame,
5571 "invalid_control_body",
5572 format!("expected module-control op '{expected}', got '{actual}'"),
5573 )?])
5574 }
5575 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
5576 guard.disarm();
5577 Ok(vec![control_error_frame(
5578 &frame,
5579 "module_timeout",
5580 format!(
5581 "module_id '{module_id}' answered health.check after {:?}",
5582 self.health_probe_timeout
5583 ),
5584 )?])
5585 }
5586 Ok(Err(_)) => Ok(vec![control_error_frame(
5587 &frame,
5588 "target_unavailable",
5589 "health.check waiter was canceled before the module responded",
5590 )?]),
5591 Err(_) => Ok(vec![control_error_frame(
5592 &frame,
5593 "module_timeout",
5594 format!(
5595 "module_id '{module_id}' did not answer health.check within {:?}",
5596 self.health_probe_timeout
5597 ),
5598 )?]),
5599 }
5600 }
5601
5602 fn supervisor_status(
5603 &self,
5604 module_id: &str,
5605 corr: u64,
5606 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
5607 self.supervisor
5608 .get(module_id)
5609 .map(|module| {
5610 let warming = module.is_warming_for_control("status").map_err(|err| {
5611 RouterError::backend(
5612 0,
5613 corr,
5614 format!(
5615 "failed to read supervisor warming state for module_id '{module_id}': {err}"
5616 ),
5617 )
5618 })?;
5619 module.status_for_control("status").map_err(|err| {
5620 RouterError::backend(
5621 0,
5622 corr,
5623 format!(
5624 "failed to read supervisor status for module_id '{module_id}': {err}"
5625 ),
5626 )
5627 }).map(|status| (status, warming))
5628 })
5629 .transpose()
5630 }
5631
5632 fn guard_module_control_op(
5633 &self,
5634 frame: &Frame,
5635 module_id: &str,
5636 op: &str,
5637 ) -> Result<Option<Frame>, RouterError> {
5638 if self.module_grants_op(module_id, op, frame.header.corr)? {
5639 return Ok(None);
5640 }
5641
5642 Ok(Some(control_error_frame(
5643 frame,
5644 "op_not_allowed",
5645 format!("module_id '{module_id}' did not grant control op '{op}'"),
5646 )?))
5647 }
5648
5649 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
5650 let Some(registration) = self
5651 .registry
5652 .get_module(module_id)
5653 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
5654 else {
5655 return Ok(false);
5656 };
5657 Ok(module_registration_grants_op(®istration.control_ops, op))
5658 }
5659
5660 fn handle_status_update(
5661 &self,
5662 endpoint: ModuleEndpointId,
5663 frame: Frame,
5664 ) -> Result<Vec<Frame>, RouterError> {
5665 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
5666 Ok(update) => update,
5667 Err(err) => {
5668 if is_known_module_push_op(&frame.body) {
5673 return Ok(vec![control_error_frame(
5674 &frame,
5675 "invalid_control_body",
5676 format!("malformed module control push body: {err}"),
5677 )?]);
5678 }
5679 return Ok(Vec::new());
5680 }
5681 };
5682
5683 match update {
5684 ModuleControlPush::RouteStatus {
5685 route_channel,
5686 route_epoch,
5687 status,
5688 } => {
5689 self.forwarding
5690 .cache_status(endpoint, route_channel, route_epoch, status)
5691 .map_err(RouterError::Forwarding)?;
5692 }
5693 }
5694 Ok(Vec::new())
5695 }
5696
5697 fn handle_route_poll(
5698 &self,
5699 ctx: &RouteCtx,
5700 frame: Frame,
5701 route_channel: u16,
5702 route_epoch: u32,
5703 kind: PollKind,
5704 ) -> Result<Vec<Frame>, RouterError> {
5705 let snapshot = self
5706 .forwarding
5707 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
5708 .map_err(RouterError::Forwarding)?;
5709 let response = match (kind, snapshot) {
5710 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
5711 ClientControlResponse::RoutePoll {
5712 route_channel,
5713 route_epoch,
5714 status,
5715 live: None,
5716 }
5717 }
5718 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5719 route_channel,
5720 route_epoch,
5721 status: None,
5722 live: None,
5723 },
5724 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
5725 let live = self
5751 .process_liveness
5752 .as_ref()
5753 .and_then(|source| source.process_live(&module_id))
5754 .unwrap_or(true);
5755 ClientControlResponse::RoutePoll {
5756 route_channel,
5757 route_epoch,
5758 status: None,
5759 live: Some(live),
5760 }
5761 }
5762 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5763 route_channel,
5764 route_epoch,
5765 status: None,
5766 live: Some(false),
5767 },
5768 };
5769
5770 Ok(vec![control_response_body_frame(
5771 &frame,
5772 &response,
5773 "ClientControlResponse::RoutePoll",
5774 )?])
5775 }
5776
5777 pub(crate) fn observe_module_control_completion(
5778 &self,
5779 completion: ModuleControlRpcCompletion,
5780 ) -> bool {
5781 match completion {
5782 ModuleControlRpcCompletion::Unknown => false,
5783 ModuleControlRpcCompletion::Settled => true,
5784 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
5785 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
5786 info!(
5787 module_id = %module_id,
5788 latency_ms,
5789 "late health.check answer proves the module is alive"
5790 );
5791 match self
5792 .supervisor
5793 .record_late_health_answer(&module_id, latency_ms)
5794 {
5795 Ok(true) => {}
5796 Ok(false) => debug!(
5797 module_id = %module_id,
5798 latency_ms,
5799 "late health.check answer has no active supervisor snapshot"
5800 ),
5801 Err(err) => warn!(
5802 module_id = %module_id,
5803 latency_ms,
5804 error = %err,
5805 "failed to record late health.check answer"
5806 ),
5807 }
5808 true
5809 }
5810 }
5811 }
5812
5813 fn refuse_to_end_module_connection_for_a_client(
5834 &self,
5835 module_connection_id: ConnectionId,
5836 corr: u64,
5837 err: ForwardingError,
5838 ) -> Result<(), RouterError> {
5839 if let ForwardingError::ConnectionClosing { connection_id } = err {
5840 if connection_id != module_connection_id {
5841 warn!(
5842 module_connection_id = module_connection_id.get(),
5843 client_connection_id = connection_id.get(),
5844 corr,
5845 "dropping a route.bind response for a closing client; the module connection keeps serving"
5846 );
5847 return Ok(());
5848 }
5849 }
5850 Err(RouterError::Forwarding(err))
5851 }
5852
5853 fn handle_module_relay_response(
5854 &self,
5855 connection_id: ConnectionId,
5856 frame: Frame,
5857 ) -> Result<Vec<Frame>, RouterError> {
5858 let mut secondary_error = None;
5859 let outcome = match frame.header.ty {
5860 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
5861 Ok(probe) if probe.op == "route.bind" => {
5862 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
5863 Ok(ModuleControlResponse::RouteBindAck {}) => {
5864 RouteBindRelayOutcome::Accepted
5865 }
5866 Ok(other) => {
5867 let message =
5868 format!("route.bind response carried unexpected body: {other:?}");
5869 secondary_error = Some(control_error_frame(
5870 &frame,
5871 "invalid_control_body",
5872 message.clone(),
5873 )?);
5874 RouteBindRelayOutcome::ModuleGone(message)
5875 }
5876 Err(err) => {
5877 let message = format!("malformed route.bind response body: {err}");
5878 secondary_error = Some(control_error_frame(
5879 &frame,
5880 "invalid_control_body",
5881 message.clone(),
5882 )?);
5883 RouteBindRelayOutcome::ModuleGone(message)
5884 }
5885 }
5886 }
5887 Ok(probe) => {
5888 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
5889 {
5890 Ok(response) => ModuleControlRpcOutcome::Response(response),
5891 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5892 "malformed {} response body: {err}",
5893 probe.op
5894 )),
5895 };
5896 let completion = self
5897 .forwarding
5898 .complete_module_control_rpc(
5899 connection_id,
5900 frame.header.corr,
5901 Some(&probe.op),
5902 outcome,
5903 )
5904 .map_err(RouterError::Forwarding)?;
5905 if !self.observe_module_control_completion(completion) {
5906 debug!(
5907 connection_id = connection_id.get(),
5908 corr = frame.header.corr,
5909 op = %probe.op,
5910 "dropping late or unknown module-control RPC response"
5911 );
5912 }
5913 return Ok(Vec::new());
5914 }
5915 Err(err) => {
5916 if let Some(expected_op) = self
5917 .forwarding
5918 .pending_module_control_op(connection_id, frame.header.corr)
5919 .map_err(RouterError::Forwarding)?
5920 {
5921 let completion = self
5922 .forwarding
5923 .complete_module_control_rpc(
5924 connection_id,
5925 frame.header.corr,
5926 None,
5927 ModuleControlRpcOutcome::MalformedResponse(format!(
5928 "malformed {expected_op} response body: {err}"
5929 )),
5930 )
5931 .map_err(RouterError::Forwarding)?;
5932 if !self.observe_module_control_completion(completion) {
5933 debug!(
5934 connection_id = connection_id.get(),
5935 corr = frame.header.corr,
5936 "dropping late malformed module-control RPC response"
5937 );
5938 }
5939 return Ok(Vec::new());
5940 }
5941 let message = format!("malformed route.bind response body: {err}");
5942 secondary_error = Some(control_error_frame(
5943 &frame,
5944 "invalid_control_body",
5945 message.clone(),
5946 )?);
5947 RouteBindRelayOutcome::ModuleGone(message)
5948 }
5949 },
5950 FrameType::Error => {
5951 if self
5952 .forwarding
5953 .pending_module_control_op(connection_id, frame.header.corr)
5954 .map_err(RouterError::Forwarding)?
5955 .is_some()
5956 {
5957 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
5958 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
5959 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5960 "malformed module-control ERROR body: {err}"
5961 )),
5962 };
5963 let completion = self
5964 .forwarding
5965 .complete_module_control_rpc(
5966 connection_id,
5967 frame.header.corr,
5968 None,
5969 outcome,
5970 )
5971 .map_err(RouterError::Forwarding)?;
5972 if !self.observe_module_control_completion(completion) {
5973 debug!(
5974 connection_id = connection_id.get(),
5975 corr = frame.header.corr,
5976 "dropping late or unknown module-control RPC error"
5977 );
5978 }
5979 return Ok(Vec::new());
5980 }
5981 match serde_json::from_slice::<ErrorBody>(&frame.body) {
5982 Ok(body) => RouteBindRelayOutcome::Rejected(body),
5983 Err(err) => {
5984 let message = format!("malformed route.bind ERROR body: {err}");
5985 secondary_error = Some(control_error_frame(
5986 &frame,
5987 "invalid_control_body",
5988 message.clone(),
5989 )?);
5990 RouteBindRelayOutcome::ModuleGone(message)
5991 }
5992 }
5993 }
5994 ty => {
5995 return Ok(vec![control_error_frame(
5996 &frame,
5997 "unsupported_control_frame",
5998 format!("unsupported module channel-0 frame {ty:?}"),
5999 )?])
6000 }
6001 };
6002
6003 let settled =
6004 self.forwarding
6005 .complete_pending_relay(connection_id, frame.header.corr, outcome);
6006 let completion = match settled {
6007 Ok(completion) => completion,
6008 Err(err) => {
6009 self.refuse_to_end_module_connection_for_a_client(
6010 connection_id,
6011 frame.header.corr,
6012 err,
6013 )?;
6014 return Ok(secondary_error.into_iter().collect());
6015 }
6016 };
6017 if let Some(target) = completion.abandoned.as_ref() {
6018 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6019 }
6020 if !completion.settled {
6021 debug!(
6022 connection_id = connection_id.get(),
6023 corr = frame.header.corr,
6024 frame_type = ?frame.header.ty,
6025 "dropping late or unknown route.bind relay response"
6026 );
6027 }
6028 Ok(secondary_error.into_iter().collect())
6029 }
6030
6031 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6032 debug!(connection_id = connection_id.get(), "handling GOODBYE");
6033 self.cleanup_connection_with_end_reason(
6037 connection_id,
6038 RegistrationEndReason::ExplicitGoodbye,
6039 )
6040 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6041 Ok(Vec::new())
6042 }
6043}
6044
6045impl Default for ControlHandler {
6046 fn default() -> Self {
6047 Self::new(Arc::new(Registry::default()))
6048 }
6049}
6050
6051impl crate::supervise::SwapPromotionObserver for ControlHandler {
6052 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6053 self.apply_registration_capabilities(registration);
6054 }
6055}
6056
6057fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6058 CapabilityRequirementStatus {
6059 consumer: status.consumer,
6060 capability: status.capability,
6061 need: match status.need {
6062 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6063 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6064 },
6065 verdict: status.verdict.as_str().to_string(),
6066 episode_seq: status.episode_seq,
6067 config_satisfiable: status.config_satisfiable,
6068 runtime_available: status.runtime_available,
6069 detail: status.detail,
6070 }
6071}
6072
6073fn append_capability_problem_detail(
6074 detail: Option<String>,
6075 capability_detail: Option<String>,
6076) -> Option<String> {
6077 match (detail, capability_detail) {
6078 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
6079 (Some(detail), None) => Some(detail),
6080 (None, Some(capability_detail)) => Some(capability_detail),
6081 (None, None) => None,
6082 }
6083}
6084
6085fn subc_ops() -> Vec<String> {
6086 SUBC_CONTROL_OPS
6087 .iter()
6088 .map(|op| (*op).to_string())
6089 .collect()
6090}
6091
6092fn module_subc_ops() -> Vec<String> {
6093 SUBC_CONTROL_OPS
6094 .iter()
6095 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
6096 .map(|op| (*op).to_string())
6097 .collect()
6098}
6099
6100#[cfg(test)]
6101fn module_baseline_control_ops() -> Vec<String> {
6102 MODULE_BASELINE_CONTROL_OPS
6103 .iter()
6104 .map(|op| (*op).to_string())
6105 .collect()
6106}
6107
6108fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
6109 let mut seen = HashSet::new();
6110 let mut effective = Vec::new();
6111 for op in MODULE_BASELINE_CONTROL_OPS {
6112 if seen.insert((*op).to_string()) {
6113 effective.push((*op).to_string());
6114 }
6115 }
6116 for op in declared.unwrap_or_default() {
6117 if seen.insert(op.clone()) {
6118 effective.push(op);
6119 }
6120 }
6121 effective
6122}
6123
6124fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
6125 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
6126}
6127
6128fn target_module_id(target: &RouteTarget) -> &str {
6129 match target {
6130 RouteTarget::ToolProvider { module_id }
6131 | RouteTarget::ManagementSurface { module_id }
6132 | RouteTarget::InternalService { module_id, .. } => module_id,
6133 }
6134}
6135
6136fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
6137 roles.iter().any(|role| match (target, role) {
6138 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
6139 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
6140 (
6141 RouteTarget::InternalService { service_id, .. },
6142 ProviderRole::InternalService {
6143 service_id: provided,
6144 ..
6145 },
6146 ) => service_id == provided,
6147 _ => false,
6148 })
6149}
6150
6151fn is_routable_role(role: &ProviderRole) -> bool {
6152 matches!(
6153 role,
6154 ProviderRole::ToolProvider { .. }
6155 | ProviderRole::ManagementSurface { .. }
6156 | ProviderRole::InternalService { .. }
6157 )
6158}
6159
6160#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6161enum ControlRequestBodyError {
6162 UnknownOp,
6163 InvalidBody,
6164}
6165
6166#[derive(Debug, Deserialize)]
6167struct ControlOpProbe {
6168 op: String,
6169}
6170
6171const MODULE_PUSH_OPS: &[&str] = &["route.status"];
6174
6175fn is_known_module_push_op(body: &[u8]) -> bool {
6176 serde_json::from_slice::<ControlOpProbe>(body)
6177 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
6178 .unwrap_or(false)
6179}
6180
6181fn is_known_module_request_op(body: &[u8]) -> bool {
6182 serde_json::from_slice::<ControlOpProbe>(body)
6183 .map(|probe| is_module_to_subc_op(&probe.op))
6184 .unwrap_or(false)
6185}
6186
6187fn is_module_to_subc_op(op: &str) -> bool {
6188 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
6189}
6190
6191fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
6192 debug!(
6193 op = %op,
6194 connection_id = connection_id.get(),
6195 corr,
6196 "control dispatch"
6197 );
6198}
6199
6200fn log_slow_control_dispatch(
6201 dispatch_started_at: Option<StdInstant>,
6202 op: &'static str,
6203 connection_id: ConnectionId,
6204 corr: u64,
6205) {
6206 let Some(dispatch_started_at) = dispatch_started_at else {
6207 return;
6208 };
6209 let elapsed = dispatch_started_at.elapsed();
6210 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
6211 warn!(
6212 op = %op,
6213 connection_id = connection_id.get(),
6214 corr,
6215 elapsed_ms = elapsed.as_millis() as u64,
6216 "slow control dispatch"
6217 );
6218 }
6219}
6220
6221fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
6222 match request {
6223 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
6224 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
6225 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
6226 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
6227 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
6228 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
6229 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
6230 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
6231 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
6232 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
6233 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
6234 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
6235 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
6236 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
6237 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
6238 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
6239 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
6240 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
6241 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
6242 }
6243}
6244
6245fn principal_label(principal: &Principal) -> String {
6246 match principal {
6247 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
6248 Principal::Direct => "direct".to_string(),
6249 other => format!("{other:?}"),
6250 }
6251}
6252
6253fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
6254 match request {
6255 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
6256 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
6257 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
6258 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
6259 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
6260 }
6261}
6262
6263fn parse_client_control_request(
6264 body: &[u8],
6265) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
6266 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
6267 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6268 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
6269 ControlRequestBodyError::InvalidBody
6270 }
6271 Ok(_) => ControlRequestBodyError::UnknownOp,
6272 Err(_) => ControlRequestBodyError::InvalidBody,
6273 };
6274 (err, classification)
6275 })
6276}
6277
6278fn parse_module_control_request_from_module(
6279 body: &[u8],
6280) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
6281 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
6282 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6283 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
6284 Ok(_) => ControlRequestBodyError::UnknownOp,
6285 Err(_) => ControlRequestBodyError::InvalidBody,
6286 };
6287 (err, classification)
6288 })
6289}
6290
6291#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
6292enum ProviderRoleKind {
6293 ToolProvider,
6294 PipelineStage,
6295 ManagementSurface,
6296 InternalService,
6297}
6298
6299fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
6300 match role {
6301 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
6302 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
6303 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
6304 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
6305 }
6306}
6307
6308fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
6309 roles.iter().map(provider_role_kind).collect()
6310}
6311
6312const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
6315
6316#[derive(Debug, Default, PartialEq, Eq)]
6318struct ScopeOutcomeCounts {
6319 created: usize,
6320 replaced: usize,
6321 updated: usize,
6322 unchanged: usize,
6323 refused: usize,
6324}
6325
6326impl ScopeOutcomeCounts {
6327 fn of(results: &[ScopeRecordResult]) -> Self {
6328 let mut counts = Self::default();
6329 for result in results {
6330 let slot = match result.outcome {
6331 ScopeRecordOutcome::Created => &mut counts.created,
6332 ScopeRecordOutcome::Replaced => &mut counts.replaced,
6333 ScopeRecordOutcome::Updated => &mut counts.updated,
6334 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
6335 ScopeRecordOutcome::Refused => &mut counts.refused,
6336 };
6337 *slot += 1;
6338 }
6339 counts
6340 }
6341}
6342
6343#[cfg(test)]
6344mod scope_outcome_count_tests {
6345 use super::*;
6346
6347 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
6348 ScopeRecordResult {
6349 scope_ref: "r".to_string(),
6350 scope_epoch: 1,
6351 outcome,
6352 code: None,
6353 message: None,
6354 version: None,
6355 parent_state: None,
6356 }
6357 }
6358
6359 #[test]
6362 fn every_outcome_is_counted_in_its_own_field() {
6363 let results = [
6364 result(ScopeRecordOutcome::Created),
6365 result(ScopeRecordOutcome::Created),
6366 result(ScopeRecordOutcome::Replaced),
6367 result(ScopeRecordOutcome::Updated),
6368 result(ScopeRecordOutcome::Unchanged),
6369 result(ScopeRecordOutcome::Refused),
6370 result(ScopeRecordOutcome::Refused),
6371 result(ScopeRecordOutcome::Refused),
6372 ];
6373 assert_eq!(
6374 ScopeOutcomeCounts::of(&results),
6375 ScopeOutcomeCounts {
6376 created: 2,
6377 replaced: 1,
6378 updated: 1,
6379 unchanged: 1,
6380 refused: 3,
6381 }
6382 );
6383 }
6384}
6385
6386fn capability_census_trigger(
6390 old: Option<&CapabilityDeclarations>,
6391 new: Option<&CapabilityDeclarations>,
6392) -> bool {
6393 let old_provides = old
6394 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
6395 .unwrap_or_default();
6396 let old_denies = old
6397 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
6398 .unwrap_or_default();
6399 let new = new.cloned().unwrap_or(CapabilityDeclarations {
6400 provides: Vec::new(),
6401 requires: Vec::new(),
6402 must_never_reach: Vec::new(),
6403 });
6404
6405 new.provides
6406 .iter()
6407 .any(|capability| !old_provides.contains(capability))
6408 || new
6409 .must_never_reach
6410 .iter()
6411 .any(|capability| !old_denies.contains(capability))
6412}
6413
6414fn denied_capability<'a>(
6417 opening_manifest: &'a ModuleManifest,
6418 target_manifest: &ModuleManifest,
6419) -> Option<&'a str> {
6420 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
6421 let target_capabilities = target_manifest.capabilities.as_ref()?;
6422 opening_capabilities
6423 .must_never_reach
6424 .iter()
6425 .find(|denied| {
6426 target_capabilities
6427 .provides
6428 .iter()
6429 .any(|provided| provided == *denied)
6430 })
6431 .map(String::as_str)
6432}
6433
6434fn catalog_update_frozen_field_message(
6435 registered: &ModuleManifest,
6436 provides: &[ProviderRole],
6437) -> Option<String> {
6438 let old_has_provides = !registered.provides.is_empty();
6439 let new_has_provides = !provides.is_empty();
6440 if old_has_provides != new_has_provides {
6441 return Some(format!(
6442 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
6443 registered.module_id
6444 ));
6445 }
6446
6447 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
6448 return Some(format!(
6449 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
6450 registered.module_id
6451 ));
6452 }
6453
6454 let registered_concurrency = manifest_concurrency(registered);
6455 let mut candidate = registered.clone();
6456 candidate.provides = provides.to_vec();
6457 let candidate_concurrency = manifest_concurrency(&candidate);
6458 if candidate_concurrency != registered_concurrency {
6459 return Some(format!(
6460 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
6461 registered.module_id, registered_concurrency, candidate_concurrency
6462 ));
6463 }
6464
6465 None
6468}
6469
6470fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
6471 manifest.provides.iter().any(is_routable_role)
6472}
6473
6474fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
6480 manifest
6481 .provides
6482 .iter()
6483 .find_map(|provider| match provider {
6484 ProviderRole::ToolProvider { concurrency, .. }
6485 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
6486 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
6487 })
6488 .unwrap_or(Concurrency::ModuleManaged)
6489}
6490
6491fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
6498 let has_management_surface = manifest
6499 .provides
6500 .iter()
6501 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
6502 if !has_management_surface {
6503 return false;
6504 }
6505 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
6506 return false;
6507 };
6508 let Some(provides) = raw
6509 .get("manifest")
6510 .and_then(|manifest| manifest.get("provides"))
6511 .and_then(serde_json::Value::as_array)
6512 else {
6513 return false;
6514 };
6515 provides.iter().any(|role| {
6520 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
6521 && role.get("concurrency").is_none()
6522 })
6523}
6524
6525fn negotiate_version(peer_version: u8) -> Result<u8, String> {
6526 if peer_version != PROTOCOL_VERSION {
6527 return Err(format!(
6528 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
6529 ));
6530 }
6531 Ok(PROTOCOL_VERSION)
6532}
6533
6534fn pong(frame: &Frame) -> Result<Frame, RouterError> {
6535 Frame::build_with_version(
6536 response_version(frame),
6537 FrameType::Pong,
6538 frame.header.flags,
6539 0,
6540 0,
6541 frame.header.corr,
6542 Vec::new(),
6543 )
6544 .map_err(RouterError::FrameBuild)
6545}
6546
6547pub(crate) fn control_error_frame(
6548 frame: &Frame,
6549 code: &'static str,
6550 message: impl Into<String>,
6551) -> Result<Frame, RouterError> {
6552 control_error_body_frame(
6553 frame,
6554 ErrorBody {
6555 code: code.to_string(),
6556 message: message.into(),
6557 detail: None,
6558 },
6559 )
6560}
6561
6562fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
6563 let body = serde_json::to_vec(&error).map_err(|err| {
6564 RouterError::backend(
6565 0,
6566 frame.header.corr,
6567 format!("failed to encode control ERROR: {err}"),
6568 )
6569 })?;
6570
6571 Frame::build_with_version(
6572 response_version(frame),
6573 FrameType::Error,
6574 control_flags(),
6575 0,
6576 0,
6577 frame.header.corr,
6578 body,
6579 )
6580 .map_err(RouterError::FrameBuild)
6581}
6582
6583fn control_response_body_frame<T: Serialize>(
6584 frame: &Frame,
6585 reply: &T,
6586 label: &'static str,
6587) -> Result<Frame, RouterError> {
6588 let body = serde_json::to_vec(reply).map_err(|err| {
6589 RouterError::backend(
6590 0,
6591 frame.header.corr,
6592 format!("failed to encode {label}: {err}"),
6593 )
6594 })?;
6595
6596 Frame::build_with_version(
6597 response_version(frame),
6598 FrameType::Response,
6599 control_flags(),
6600 0,
6601 0,
6602 frame.header.corr,
6603 body,
6604 )
6605 .map_err(RouterError::FrameBuild)
6606}
6607
6608fn forwarding_error_code(err: &ForwardingError) -> &'static str {
6638 match err {
6639 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
6640 ForwardingError::NoModuleConnection => "target_unavailable",
6641 ForwardingError::ModuleReloading { .. } => "module_reloading",
6642 ForwardingError::ClientRouteChannelExhausted { .. }
6643 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
6644 ForwardingError::StaleModuleEndpoint
6645 | ForwardingError::UnknownReservation { .. }
6646 | ForwardingError::ConnectionClosing { .. }
6647 | ForwardingError::ClientEgressClosed { .. }
6648 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
6649 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
6652 ForwardingError::RelayCorrelationExhausted
6653 | ForwardingError::RouteOpenBuild(_)
6654 | ForwardingError::Poisoned => "forwarding_error",
6655 }
6656}
6657
6658fn response_version(frame: &Frame) -> u8 {
6659 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
6660 frame.header.ver
6661 } else {
6662 PROTOCOL_VERSION
6663 }
6664}
6665
6666fn control_flags() -> Flags {
6667 Flags::new(false, Priority::Passive, false)
6668}
6669
6670fn send_goodbye_target_best_effort(
6675 counters: &DaemonCounters,
6676 target: &GoodbyeTarget,
6677 context: &'static str,
6678) {
6679 let Ok(frame) = Frame::build_with_version(
6680 target.negotiated_ver,
6681 FrameType::Goodbye,
6682 control_flags(),
6683 target.channel,
6684 target.epoch,
6685 0,
6686 Vec::new(),
6687 ) else {
6688 return;
6689 };
6690 crate::forwarding::send_module_route_goodbye(
6691 counters,
6692 &target.sink,
6693 frame,
6694 target.module_id.as_deref(),
6695 context,
6696 );
6697}
6698
6699pub(crate) fn send_route_control_pushes(
6700 forwarding: &ForwardingTable,
6701 routes: Vec<EndpointRoute>,
6702 push: ClientControlPush,
6703) {
6704 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
6705 for route in routes {
6706 let target = route.goodbye_target;
6707 if let Some((existing, channels)) = targets
6708 .iter_mut()
6709 .find(|(existing, _)| existing.connection_id == target.connection_id)
6710 {
6711 debug_assert_eq!(
6712 existing.negotiated_ver, target.negotiated_ver,
6713 "one connection cannot negotiate multiple frame versions"
6714 );
6715 if !channels.contains(&target.channel) {
6716 channels.push(target.channel);
6717 }
6718 continue;
6719 }
6720 let channel = target.channel;
6721 targets.push((target, vec![channel]));
6722 }
6723 for (target, mut channels) in targets {
6724 channels.sort_unstable();
6725 let mut push = push.clone();
6726 match &mut push {
6727 ClientControlPush::RouteClosing {
6728 channels: covered, ..
6729 }
6730 | ClientControlPush::RouteClosed {
6731 channels: covered, ..
6732 } => *covered = channels,
6733 }
6734 let body = match serde_json::to_vec(&push) {
6735 Ok(body) => body,
6736 Err(err) => {
6737 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
6738 continue;
6739 }
6740 };
6741 let frame = match Frame::build_with_version(
6742 target.negotiated_ver,
6743 FrameType::Push,
6744 control_flags(),
6745 0,
6746 0,
6747 0,
6748 body.clone(),
6749 ) {
6750 Ok(frame) => frame,
6751 Err(err) => {
6752 warn!(
6753 route_channel = target.channel,
6754 error = %err,
6755 "failed to build route lifecycle control PUSH frame"
6756 );
6757 continue;
6758 }
6759 };
6760 if let Err(err) = target.sink.try_send(frame) {
6761 if target.close_on_delivery_failure() {
6762 warn!(
6763 target_connection_id = target.connection_id.get(),
6764 route_channel = target.channel,
6765 error = %err,
6766 "route lifecycle control PUSH was not delivered to client; closing target connection"
6767 );
6768 let _ = forwarding.escalate_client_delivery_failure(
6769 target.connection_id,
6770 target.channel,
6771 target.epoch,
6772 CloseReason::new(
6773 "route_lifecycle_push_delivery_failed",
6774 format!(
6775 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
6776 target.channel
6777 ),
6778 ),
6779 crate::forwarding::UndeliveredFrame {
6780 module_id: target.module_id.as_deref(),
6781 sink: &target.sink,
6782 },
6783 );
6784 }
6785 }
6786 }
6787}
6788
6789#[cfg(test)]
6790mod tests {
6791 #[cfg(unix)]
6792 #[tokio::test]
6793 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
6794 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
6795 let path = dir.join("subc.jsonc");
6796 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
6797 "program":"/bin/sleep","args":["60"],"protocol":"none",
6798 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
6799 }}}).to_string()).unwrap();
6800 let mut configured = crate::daemon_config::load(&path)
6801 .unwrap()
6802 .unwrap()
6803 .modules
6804 .pop()
6805 .unwrap();
6806 let registry = std::sync::Arc::new(crate::Registry::default());
6807 let handle = crate::SupervisorHandle::new();
6808 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
6809 .with_handle(handle.clone());
6810 let module = supervisor
6811 .supervise_configured_with_health(
6812 configured.module_spec(),
6813 true,
6814 configured.health.clone(),
6815 None,
6816 configured.restart,
6817 )
6818 .unwrap();
6819 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
6820 let before = module.status().unwrap().pid;
6821 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
6822 configured.health.cadence = std::time::Duration::from_secs(3600);
6823 let health_only = handler
6824 .reconcile_supervised_modules(&supervisor, vec![configured.clone()], false)
6825 .await
6826 .unwrap();
6827 assert!(
6828 health_only.changed_pending_reload.is_empty(),
6829 "health policy is already applied live"
6830 );
6831 assert_eq!(module.status().unwrap().pid, before);
6832 assert_eq!(
6833 module.configuration().unwrap().1.http,
6834 configured.health.http
6835 );
6836 configured.args = vec!["61".into()];
6837 let launch = handler
6838 .reconcile_supervised_modules(&supervisor, vec![configured], false)
6839 .await
6840 .unwrap();
6841 assert_eq!(launch.changed_pending_reload, ["stock"]);
6842 assert_eq!(
6843 module.status().unwrap().pid,
6844 before,
6845 "a launch edit is stored until reload"
6846 );
6847 module.drain().await.unwrap();
6848 }
6849 use cortexkit_test_support::ScratchDir;
6850 use std::{
6851 collections::BTreeMap,
6852 fmt,
6853 path::PathBuf,
6854 sync::{Arc, Mutex},
6855 time::Duration,
6856 };
6857
6858 use serde_json::{json, Value};
6859 use subc_protocol::{
6860 manifest::{
6861 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
6862 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
6863 },
6864 session::HealthStatus,
6865 FrameType,
6866 };
6867
6868 use super::*;
6869 use crate::{
6870 forwarding::{DataRoute, DataRouteState},
6871 registry::ChannelState,
6872 router::FrameSink,
6873 stderr_tail::DEFAULT_MAX_LINE_BYTES,
6874 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
6875 RouteCtx, Router,
6876 };
6877 use tokio::{
6878 sync::mpsc,
6879 time::{sleep, Instant},
6880 };
6881 use tracing::{
6882 field::{Field, Visit},
6883 Event, Subscriber,
6884 };
6885 use tracing_subscriber::{layer::Context, prelude::*, Layer};
6886
6887 fn fake_aft_stub_path() -> PathBuf {
6906 let mut path = std::env::current_exe().expect("current_exe available in tests");
6907 path.pop(); path.pop(); path.push(if cfg!(windows) {
6910 "fake-aft-stub.exe"
6911 } else {
6912 "fake-aft-stub"
6913 });
6914 assert!(
6915 path.exists(),
6916 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
6917 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
6918 path.display()
6919 );
6920 path
6921 }
6922
6923 fn client_retries(code: &str) -> bool {
6928 subc_protocol::error_codes::is_retryable_route_open(code)
6929 }
6930
6931 #[test]
6942 fn retryability_of_forwarding_codes_matches_the_failure() {
6943 let transient = [
6946 ForwardingError::NoModuleConnection,
6947 ForwardingError::ModuleReloading {
6948 module_id: "m".into(),
6949 },
6950 ForwardingError::StaleModuleEndpoint,
6951 ForwardingError::UnknownReservation {
6952 client_channel: 1,
6953 module_channel: 1,
6954 },
6955 ForwardingError::ConnectionClosing {
6956 connection_id: ConnectionId::new(1),
6957 },
6958 ForwardingError::ClientEgressClosed {
6959 connection_id: ConnectionId::new(1),
6960 },
6961 ForwardingError::ModuleEgressUnavailable {
6962 connection_id: ConnectionId::new(1),
6963 },
6964 ];
6965 for err in transient {
6966 let code = forwarding_error_code(&err);
6967 assert!(
6968 client_retries(code),
6969 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
6970 );
6971 }
6972
6973 let permanent = [
6978 ForwardingError::ConnectionRoleConflict {
6979 connection_id: ConnectionId::new(1),
6980 },
6981 ForwardingError::ClientRouteChannelExhausted {
6982 connection_id: ConnectionId::new(1),
6983 },
6984 ForwardingError::ModuleRouteChannelExhausted {
6985 endpoint: ModuleEndpointId {
6986 connection_id: ConnectionId::new(1),
6987 generation: 1,
6988 },
6989 },
6990 ForwardingError::RelayCorrelationExhausted,
6991 ForwardingError::RouteOpenBuild("x".into()),
6992 ForwardingError::Poisoned,
6993 ];
6994 for err in permanent {
6995 let code = forwarding_error_code(&err);
6996 assert!(
6997 !client_retries(code),
6998 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
6999 );
7000 }
7001 }
7002
7003 #[tokio::test]
7031 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
7032 let handler = ControlHandler::default();
7033 let frame =
7034 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
7035
7036 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
7040 assert_eq!(
7041 stamped,
7042 Principal::Direct,
7043 "a caller that proved nothing must not be stamped as a supervised module"
7044 );
7045
7046 let forged = handler
7051 .route_open_principal(
7052 &frame,
7053 Some(ConsumerIdentity {
7054 module_id: "aft".to_string(),
7055 launch_nonce: "not-a-real-nonce".to_string(),
7056 }),
7057 )
7058 .unwrap();
7059 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
7060 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
7061 }
7062
7063 #[test]
7080 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
7081 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
7082 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
7083 let ClientControlRequest::RouteOpen {
7084 consumer_identity, ..
7085 } = parsed
7086 else {
7087 panic!("route.open body must parse as RouteOpen");
7088 };
7089 assert_eq!(
7090 consumer_identity,
7091 Some(ConsumerIdentity {
7092 module_id: "aft".to_string(),
7093 launch_nonce: "n".to_string(),
7094 }),
7095 "the wire field name must reach the value route_open_principal reads"
7096 );
7097 }
7098
7099 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
7100 ModuleManifest::builder(module_id, "0.1.0")
7101 .protocol_ver(protocol_ver)
7102 .provides(vec![ProviderRole::ToolProvider {
7103 tools: vec![Tool {
7104 name: "read".to_string(),
7105 description: None,
7106 execution_mode: ExecutionMode::Pure,
7107 schema: json!({"type": "object"}),
7108 }],
7109 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
7110 concurrency: Concurrency::ModuleManaged,
7111 emits_push: true,
7112 sub_supervises: true,
7113 }])
7114 .build()
7115 }
7116
7117 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
7118 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
7119 }
7120
7121 fn hello_frame_with_control_ops(
7122 module_id: &str,
7123 protocol_ver: u8,
7124 corr: u64,
7125 control_ops: Option<Vec<String>>,
7126 ) -> Frame {
7127 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
7128 }
7129
7130 fn hello_frame_with_nonce(
7131 module_id: &str,
7132 protocol_ver: u8,
7133 corr: u64,
7134 launch_nonce: Option<&str>,
7135 ) -> Frame {
7136 hello_frame_full(
7137 module_id,
7138 protocol_ver,
7139 corr,
7140 None,
7141 launch_nonce.map(ToOwned::to_owned),
7142 )
7143 }
7144
7145 fn hello_frame_full(
7146 module_id: &str,
7147 protocol_ver: u8,
7148 corr: u64,
7149 control_ops: Option<Vec<String>>,
7150 launch_nonce: Option<String>,
7151 ) -> Frame {
7152 let body = serde_json::to_vec(&ModuleHelloBody {
7153 manifest: manifest(module_id, protocol_ver),
7154 protocol_ver,
7155 control_ops,
7156 launch_nonce,
7157 })
7158 .unwrap();
7159 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7160 }
7161
7162 fn non_routable_hello_frame_with_control_ops(
7163 module_id: &str,
7164 corr: u64,
7165 control_ops: Option<Vec<String>>,
7166 ) -> Frame {
7167 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
7168 manifest.provides.clear();
7169 let body = serde_json::to_vec(&ModuleHelloBody {
7170 manifest,
7171 protocol_ver: PROTOCOL_VERSION,
7172 control_ops,
7173 launch_nonce: None,
7174 })
7175 .unwrap();
7176 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7177 }
7178
7179 fn capability_grammar_hello_frame(
7180 capabilities: Value,
7181 runtime_computed: Option<Value>,
7182 corr: u64,
7183 ) -> Frame {
7184 let mut body = serde_json::to_value(ModuleHelloBody {
7185 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
7186 protocol_ver: PROTOCOL_VERSION,
7187 control_ops: None,
7188 launch_nonce: None,
7189 })
7190 .expect("HELLO body serializes");
7191 body["manifest"]["capabilities"] = capabilities;
7192 if let Some(runtime_computed) = runtime_computed {
7193 body["runtime_computed"] = runtime_computed;
7194 }
7195 Frame::build(
7196 FrameType::Hello,
7197 control_flags(),
7198 0,
7199 0,
7200 corr,
7201 serde_json::to_vec(&body).expect("HELLO body reserializes"),
7202 )
7203 .expect("HELLO frame builds")
7204 }
7205
7206 fn channel_request(channel: u16, corr: u64) -> Frame {
7207 Frame::build(
7208 FrameType::Request,
7209 Flags::new(true, Priority::Interactive, false),
7210 channel,
7211 0,
7212 corr,
7213 b"opaque".to_vec(),
7214 )
7215 .unwrap()
7216 }
7217
7218 fn route_ctx(
7219 connection_id: ConnectionId,
7220 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
7221 let (tx, rx) = mpsc::channel(8);
7222 (
7223 RouteCtx {
7224 connection_id,
7225 egress: FrameSink::new(tx),
7226 },
7227 rx,
7228 )
7229 }
7230
7231 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
7232 serde_json::from_slice(&frame.body).unwrap()
7233 }
7234
7235 async fn hello_via_sink(
7240 handler: &ControlHandler,
7241 ctx: &RouteCtx,
7242 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7243 hello: Frame,
7244 ) -> Frame {
7245 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
7246 assert!(
7247 replies.is_empty(),
7248 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
7249 );
7250 let ack = rx
7251 .try_recv()
7252 .expect("HELLO_ACK is queued on the module sink")
7253 .frame;
7254 assert_eq!(ack.header.ty, FrameType::HelloAck);
7255 ack
7256 }
7257
7258 fn parse_error(frame: &Frame) -> Value {
7259 serde_json::from_slice(&frame.body).unwrap()
7260 }
7261
7262 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
7263 serde_json::from_slice(&frame.body).unwrap()
7264 }
7265
7266 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
7267 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
7268 route_channel,
7269 route_epoch: 0,
7270 kind,
7271 })
7272 .unwrap();
7273 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7274 }
7275
7276 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
7277 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
7278 module_id: module_id.to_string(),
7279 })
7280 .unwrap();
7281 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7282 }
7283
7284 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
7285 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
7286 }
7287
7288 fn route_open_frame_with_consumer_capabilities(
7289 corr: u64,
7290 module_id: &str,
7291 project_root: ScratchDir,
7292 consumer_capabilities: Option<Vec<String>>,
7293 ) -> Frame {
7294 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7295 target: RouteTarget::ToolProvider {
7296 module_id: module_id.to_string(),
7297 },
7298 identity: BindIdentity::new(
7299 project_root.path().to_path_buf(),
7300 "unit".to_string(),
7301 "session".to_string(),
7302 ),
7303 consumer_identity: None,
7304 consumer_capabilities,
7305 role_versions: None,
7306 admission_facts: None,
7307 scope: None,
7308 })
7309 .unwrap();
7310 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7311 }
7312
7313 fn route_open_frame_with_role_versions(
7314 corr: u64,
7315 module_id: &str,
7316 project_root: ScratchDir,
7317 role_versions: Option<BTreeMap<String, String>>,
7318 ) -> Frame {
7319 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7320 target: RouteTarget::ToolProvider {
7321 module_id: module_id.to_string(),
7322 },
7323 identity: BindIdentity::new(
7324 project_root.path().to_path_buf(),
7325 "unit".to_string(),
7326 format!("session-{corr}"),
7327 ),
7328 consumer_identity: None,
7329 consumer_capabilities: None,
7330 role_versions,
7331 admission_facts: None,
7332 scope: None,
7333 })
7334 .unwrap();
7335 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7336 }
7337
7338 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
7339 entries
7340 .iter()
7341 .map(|(role, version)| (role.to_string(), version.to_string()))
7342 .collect()
7343 }
7344
7345 fn route_open_frame_with_admission_facts(
7346 corr: u64,
7347 module_id: &str,
7348 project_root: ScratchDir,
7349 consumer_identity: Option<subc_control::ConsumerIdentity>,
7350 facts: Option<Value>,
7351 ) -> Frame {
7352 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7353 target: RouteTarget::ToolProvider {
7354 module_id: module_id.to_string(),
7355 },
7356 identity: BindIdentity::new(
7357 project_root.path().to_path_buf(),
7358 "unit".to_string(),
7359 format!("session-{corr}"),
7360 ),
7361 consumer_identity,
7362 consumer_capabilities: None,
7363 role_versions: None,
7364 admission_facts: facts,
7365 scope: None,
7366 })
7367 .unwrap();
7368 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7369 }
7370
7371 #[derive(Clone, Default)]
7372 struct EventCapture {
7373 events: Arc<Mutex<Vec<CapturedEvent>>>,
7374 }
7375
7376 #[derive(Clone, Debug)]
7377 struct CapturedEvent {
7378 target: String,
7379 level: tracing::Level,
7380 fields: BTreeMap<String, String>,
7381 }
7382
7383 impl EventCapture {
7384 fn events(&self) -> Vec<CapturedEvent> {
7385 self.events.lock().unwrap().clone()
7386 }
7387 }
7388
7389 impl<S> Layer<S> for EventCapture
7390 where
7391 S: Subscriber,
7392 {
7393 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
7394 let mut visitor = EventFieldVisitor::default();
7395 event.record(&mut visitor);
7396 self.events.lock().unwrap().push(CapturedEvent {
7397 target: event.metadata().target().to_string(),
7398 level: *event.metadata().level(),
7399 fields: visitor.fields,
7400 });
7401 }
7402 }
7403
7404 #[derive(Default)]
7405 struct EventFieldVisitor {
7406 fields: BTreeMap<String, String>,
7407 }
7408
7409 impl Visit for EventFieldVisitor {
7410 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
7411 self.fields
7412 .insert(field.name().to_string(), format!("{value:?}"));
7413 }
7414 }
7415
7416 fn health_response(corr: u64, status: HealthStatus) -> Frame {
7417 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
7418 status,
7419 detail: Some("warming".to_string()),
7420 metrics: Some(json!({"queue_depth": 3})),
7421 })
7422 .unwrap();
7423 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7424 }
7425
7426 fn route_bind_ack(corr: u64) -> Frame {
7427 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
7428 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7429 }
7430
7431 fn unique_project_root(label: &str) -> ScratchDir {
7432 ScratchDir::new(label)
7433 }
7434
7435 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
7436 match parse_route_poll(frame) {
7437 ClientControlResponse::RoutePoll {
7438 status: None,
7439 live: Some(live),
7440 ..
7441 } => assert_eq!(live, expected_live),
7442 other => panic!("unexpected route.poll response: {other:?}"),
7443 }
7444 }
7445
7446 fn bind_liveness_route(
7447 registry: &Registry,
7448 forwarding: &ForwardingTable,
7449 module_id: &str,
7450 ) -> (RouteCtx, u16, u32) {
7451 let module_connection = ConnectionId::new(101);
7452 let client_connection = ConnectionId::new(202);
7453 let registration = registry
7454 .register_with_control_ops(
7455 manifest(module_id, PROTOCOL_VERSION),
7456 PROTOCOL_VERSION,
7457 module_connection,
7458 module_baseline_control_ops(),
7459 )
7460 .unwrap();
7461 let (module_tx, _module_rx) = mpsc::channel(8);
7462 let endpoint = forwarding
7463 .register_module_connection(
7464 module_connection,
7465 module_id.to_string(),
7466 PROTOCOL_VERSION,
7467 manifest_concurrency(®istration.manifest),
7468 FrameSink::new(module_tx),
7469 )
7470 .unwrap();
7471 let (client_ctx, _client_rx) = route_ctx(client_connection);
7472 let pending = forwarding
7473 .begin_route_bind_relay_for_test(
7474 client_connection,
7475 client_ctx.egress.clone(),
7476 1,
7477 module_id,
7478 )
7479 .unwrap();
7480 assert_eq!(pending.endpoint, endpoint);
7481 let route_channel = pending.client_channel;
7482 let route_epoch = pending.client_epoch;
7483 forwarding
7484 .complete_pending_relay(
7485 module_connection,
7486 pending.corr,
7487 RouteBindRelayOutcome::Accepted,
7488 )
7489 .unwrap();
7490 (client_ctx, route_channel, route_epoch)
7491 }
7492
7493 struct FakeProcessLiveness {
7494 live: Option<bool>,
7495 }
7496
7497 impl ModuleProcessLiveness for FakeProcessLiveness {
7498 fn process_live(&self, _module_id: &str) -> Option<bool> {
7499 self.live
7500 }
7501 }
7502
7503 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7504 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
7505 {
7506 let registry = Arc::new(Registry::default());
7507 let supervisor_handle = SupervisorHandle::new();
7508 let supervisor = Supervisor::new_for_test(
7509 Arc::clone(®istry),
7510 RestartPolicy::new(1, Duration::from_millis(10)),
7511 )
7512 .with_handle(supervisor_handle.clone());
7513 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
7514 let module = supervisor
7515 .spawn(ModuleSpec {
7516 module_id: "stderr-tail-wire".to_string(),
7517 program: fake_aft_stub_path(),
7518 args: Vec::new(),
7519 env: vec![
7520 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
7521 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
7522 ],
7523 reserved: false,
7524 reserved_prefixes: Vec::new(),
7525 protocol: ModuleProtocol::Subc,
7526 overlap: Default::default(),
7527 })
7528 .unwrap();
7529
7530 let deadline = Instant::now() + Duration::from_secs(5);
7531 loop {
7532 let tail = module.stderr_tail(None, None);
7533 if tail
7534 .entries
7535 .iter()
7536 .any(|entry| matches!(entry, TailEntry::ProcessStart))
7537 && tail.entries.iter().any(|entry| {
7538 matches!(
7539 entry,
7540 TailEntry::Line {
7541 truncated: true,
7542 ..
7543 }
7544 )
7545 })
7546 {
7547 break;
7548 }
7549 assert!(
7550 Instant::now() < deadline,
7551 "module did not produce a truncated line and restart boundary: {tail:?}"
7552 );
7553 sleep(Duration::from_millis(10)).await;
7554 }
7555
7556 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
7557 let request = ClientControlRequest::SupervisorStderrTail {
7558 module_id: "stderr-tail-wire".to_string(),
7559 max_lines: None,
7560 max_bytes: None,
7561 };
7562 let frame = Frame::build(
7563 FrameType::Request,
7564 control_flags(),
7565 0,
7566 0,
7567 1,
7568 serde_json::to_vec(&request).unwrap(),
7569 )
7570 .unwrap();
7571 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7572 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7573 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
7574 serde_json::from_slice(&responses[0].body).unwrap()
7575 else {
7576 panic!("expected supervisor.stderr_tail response");
7577 };
7578
7579 assert!(
7580 tail.entries
7581 .iter()
7582 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
7583 "the control response lost the restart boundary"
7584 );
7585 let Some(StderrTailEntry::Line {
7586 text,
7587 truncated,
7588 at_ms,
7589 }) = tail.entries.iter().find(|entry| {
7590 matches!(
7591 entry,
7592 StderrTailEntry::Line {
7593 truncated: true,
7594 ..
7595 }
7596 )
7597 })
7598 else {
7599 panic!("the control response lost the truncated line");
7600 };
7601 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
7602 assert!(*truncated);
7603 assert!(
7604 at_ms.is_some(),
7605 "the control response lost the line's capture time"
7606 );
7607 }
7608
7609 #[tokio::test(flavor = "current_thread")]
7614 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
7615 let dir = ScratchDir::new("terminals-off-worker");
7616 let journal_path = dir.join("terminals.jsonl");
7617 let registry = Arc::new(Registry::default());
7618 let supervisor_handle = SupervisorHandle::new();
7619 let supervisor =
7620 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7621 .with_handle(supervisor_handle.clone())
7622 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
7623 let module = supervisor
7624 .spawn(ModuleSpec {
7625 module_id: "terminal-off-worker".to_string(),
7626 program: fake_aft_stub_path(),
7627 args: Vec::new(),
7628 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7629 reserved: false,
7630 reserved_prefixes: Vec::new(),
7631 protocol: ModuleProtocol::Subc,
7632 overlap: Default::default(),
7633 })
7634 .unwrap();
7635 let deadline = Instant::now() + Duration::from_secs(5);
7636 while module.terminal_history().entries.len() != 2 {
7637 assert!(Instant::now() < deadline, "module did not record two exits");
7638 sleep(Duration::from_millis(10)).await;
7639 }
7640
7641 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
7642 let handler =
7643 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
7644 let frame = Frame::build(
7645 FrameType::Request,
7646 control_flags(),
7647 0,
7648 0,
7649 1,
7650 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
7651 module_id: "terminal-off-worker".to_string(),
7652 })
7653 .unwrap(),
7654 )
7655 .unwrap();
7656 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7657 let spawned_at = std::time::Instant::now();
7658 let read = tokio::spawn({
7659 let handler = Arc::clone(&handler);
7660 async move { handler.handle_control_frame(&ctx, frame).await }
7661 });
7662 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
7665 .await
7666 .unwrap()
7667 .expect("the history read reached its pause");
7668 let elapsed = spawned_at.elapsed();
7669 assert!(
7670 elapsed < Duration::from_secs(2) && !read.is_finished(),
7671 "this task could not run while the history read was paused \
7672 (resumed after {elapsed:?}, read finished: {})",
7673 read.is_finished()
7674 );
7675
7676 drop(release);
7677 let responses = read.await.unwrap().unwrap();
7678 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7679 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
7680 panic!("expected supervisor.terminals response");
7681 };
7682 assert_eq!(terminals.entries.len(), 2);
7683 assert_eq!(terminals.journal_skipped_lines, 0);
7684 assert_eq!(terminals.journal_read_errors, 0);
7685 }
7686
7687 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7688 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
7689 let registry = Arc::new(Registry::default());
7690 let supervisor_handle = SupervisorHandle::new();
7691 let supervisor =
7692 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7693 .with_handle(supervisor_handle.clone());
7694 let module = supervisor
7695 .spawn(ModuleSpec {
7696 module_id: "terminal-golden".to_string(),
7697 program: fake_aft_stub_path(),
7698 args: Vec::new(),
7699 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7700 reserved: false,
7701 reserved_prefixes: Vec::new(),
7702 protocol: ModuleProtocol::Subc,
7703 overlap: Default::default(),
7704 })
7705 .unwrap();
7706
7707 let deadline = Instant::now() + Duration::from_secs(5);
7708 while module.terminal_history().entries.len() != 2 {
7709 assert!(
7710 Instant::now() < deadline,
7711 "module did not retain two terminal exits: {:?}",
7712 module.terminal_history()
7713 );
7714 sleep(Duration::from_millis(10)).await;
7715 }
7716
7717 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
7718 let request = ClientControlRequest::SupervisorTerminals {
7719 module_id: "terminal-golden".to_string(),
7720 };
7721 let frame = Frame::build(
7722 FrameType::Request,
7723 control_flags(),
7724 0,
7725 0,
7726 1,
7727 serde_json::to_vec(&request).unwrap(),
7728 )
7729 .unwrap();
7730 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7731 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7732 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7733 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
7734 panic!("expected supervisor.terminals response");
7735 };
7736 assert_eq!(terminals.entries.len(), 2);
7737 assert_eq!(terminals.dropped, 0);
7738
7739 let mut rendered = serde_json::to_value(response).unwrap();
7740 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
7743 for (index, entry) in rendered["entries"]
7744 .as_array_mut()
7745 .expect("terminal response entries array")
7746 .iter_mut()
7747 .enumerate()
7748 {
7749 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
7750 }
7751
7752 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7753 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
7754 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
7755 if std::env::var_os("UPDATE_GOLDEN").is_some() {
7756 std::fs::write(&golden_path, &serialized).unwrap();
7757 }
7758 let expected: Value =
7759 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
7760 assert_eq!(rendered, expected);
7761 }
7762
7763 #[test]
7764 fn hello_registers_manifest_and_returns_ack() {
7765 let registry = Arc::new(Registry::default());
7766 let handler = ControlHandler::new(Arc::clone(®istry));
7767 let conn = ConnectionId::new(1);
7768
7769 let responses = handler
7770 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
7771 .unwrap();
7772
7773 assert_eq!(responses.len(), 1);
7774 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7775 assert_eq!(responses[0].header.channel, 0);
7776 assert_eq!(responses[0].header.corr, 7);
7777 let ack = parse_ack(&responses[0]);
7778 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
7779 assert!(ack
7780 .subc_capabilities
7781 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
7782 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
7783 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
7784 assert!(ack
7785 .subc_ops
7786 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
7787 assert!(ack
7788 .subc_ops
7789 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
7790
7791 let registration = registry.get_module("aft").unwrap().unwrap();
7792 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
7793 assert_eq!(registration.state, ChannelState::Active);
7794 assert_eq!(registration.connection_id, conn);
7795 assert_eq!(registration.control_ops, module_baseline_control_ops());
7796 }
7797
7798 #[test]
7799 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
7800 let invalid_identifiers = [
7801 ("case_change", "credentials-Provider/v1"),
7802 ("leading_zero", "credentials-provider/v01"),
7803 ("trailing_hyphen", "credentials-provider-/v1"),
7804 ("consecutive_hyphens", "credentials--provider/v1"),
7805 ("uppercase", "Credentials-provider/v1"),
7806 ("missing_v", "credentials-provider/1"),
7807 ("whitespace", "credentials provider/v1"),
7808 ("zero_version", "credentials-provider/v0"),
7809 ("out_of_range_version", "credentials-provider/v4294967296"),
7810 (
7811 "overlength_name",
7812 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
7813 ),
7814 ];
7815 let mut cases = invalid_identifiers
7816 .into_iter()
7817 .map(|(name, identifier)| {
7818 (
7819 format!("identifier_{name}"),
7820 "capabilities.provides[0]".to_string(),
7821 identifier.to_string(),
7822 json!({ "provides": [identifier] }),
7823 None,
7824 )
7825 })
7826 .collect::<Vec<_>>();
7827 cases.extend([
7828 (
7829 "unknown_need".to_string(),
7830 "capabilities.requires[0].need".to_string(),
7831 "deferred".to_string(),
7832 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
7833 None,
7834 ),
7835 (
7836 "duplicate_provides".to_string(),
7837 "capabilities.provides[1]".to_string(),
7838 "credentials-provider/v1".to_string(),
7839 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
7840 None,
7841 ),
7842 (
7843 "duplicate_must_never_reach".to_string(),
7844 "capabilities.must_never_reach[1]".to_string(),
7845 "credentials-provider/v1".to_string(),
7846 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
7847 None,
7848 ),
7849 (
7850 "duplicate_requires_same_need".to_string(),
7851 "capabilities.requires[1]".to_string(),
7852 "credentials-provider/v1".to_string(),
7853 json!({ "requires": [
7854 { "capability": "credentials-provider/v1", "need": "required" },
7855 { "capability": "credentials-provider/v1", "need": "required" }
7856 ] }),
7857 None,
7858 ),
7859 (
7860 "duplicate_requires_conflicting_need".to_string(),
7861 "capabilities.requires[1]".to_string(),
7862 "credentials-provider/v1".to_string(),
7863 json!({ "requires": [
7864 { "capability": "credentials-provider/v1", "need": "required" },
7865 { "capability": "credentials-provider/v1", "need": "optional" }
7866 ] }),
7867 None,
7868 ),
7869 (
7870 "capabilities_root_pointer".to_string(),
7871 "runtime_computed[0]".to_string(),
7872 "/capabilities".to_string(),
7873 json!({}),
7874 Some(json!(["/capabilities"])),
7875 ),
7876 (
7877 "capabilities_descendant_pointer".to_string(),
7878 "runtime_computed[0]".to_string(),
7879 "/capabilities/provides".to_string(),
7880 json!({}),
7881 Some(json!(["/capabilities/provides"])),
7882 ),
7883 (
7884 "malformed_pointer_without_leading_slash".to_string(),
7885 "runtime_computed[0]".to_string(),
7886 "capabilities".to_string(),
7887 json!({}),
7888 Some(json!(["capabilities"])),
7889 ),
7890 (
7891 "malformed_pointer_escape".to_string(),
7892 "runtime_computed[0]".to_string(),
7893 "/roles/~2/tools".to_string(),
7894 json!({}),
7895 Some(json!(["/roles/~2/tools"])),
7896 ),
7897 (
7898 "unknown_capabilities_field".to_string(),
7899 "capabilities.future".to_string(),
7900 "<array>".to_string(),
7901 json!({ "future": [] }),
7902 None,
7903 ),
7904 ]);
7905
7906 for (index, (name, field, value, capabilities, runtime_computed)) in
7907 cases.into_iter().enumerate()
7908 {
7909 let registry = Arc::new(Registry::default());
7910 let handler = ControlHandler::new(Arc::clone(®istry));
7911 let response = handler
7912 .handle_control(
7913 ConnectionId::new((index + 1) as u64),
7914 capability_grammar_hello_frame(
7915 capabilities,
7916 runtime_computed,
7917 index as u64 + 1,
7918 ),
7919 )
7920 .expect("invalid HELLO returns a refusal");
7921
7922 assert_eq!(response.len(), 1, "{name} must emit one refusal");
7923 let error = parse_error(&response[0]);
7924 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
7925 let message = error["message"]
7926 .as_str()
7927 .expect("error message is a string");
7928 assert!(
7929 message.contains(&field),
7930 "{name}: field missing from {message}"
7931 );
7932 assert!(
7933 message.contains(&value),
7934 "{name}: value missing from {message}"
7935 );
7936 assert_eq!(
7937 registry
7938 .active_registration_count()
7939 .expect("registry reads"),
7940 0,
7941 "{name}: refused HELLO must not create a catalog entry"
7942 );
7943 }
7944 }
7945
7946 #[test]
7947 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
7948 let registry = Arc::new(Registry::default());
7949 let handler = ControlHandler::new(Arc::clone(®istry));
7950 let capabilities = json!({
7951 "provides": ["credentials-provider/v1"],
7952 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
7953 "must_never_reach": ["federation-transport/v1"]
7954 });
7955 let response = handler
7956 .handle_control(
7957 ConnectionId::new(99),
7958 capability_grammar_hello_frame(
7959 capabilities.clone(),
7960 Some(json!(["/roles/0/tools"])),
7961 99,
7962 ),
7963 )
7964 .expect("valid HELLO registers");
7965 assert_eq!(response[0].header.ty, FrameType::HelloAck);
7966
7967 let request = Frame::build(
7968 FrameType::Request,
7969 control_flags(),
7970 0,
7971 0,
7972 100,
7973 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7974 .expect("catalog request serializes"),
7975 )
7976 .expect("catalog request frame builds");
7977 let response = handler
7978 .handle_catalog_list(request, None)
7979 .expect("catalog list succeeds");
7980 let ClientControlResponse::CatalogList { modules, .. } =
7981 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
7982 else {
7983 panic!("catalog request must return catalog.list");
7984 };
7985 assert_eq!(modules.len(), 1);
7986 assert_eq!(
7987 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
7988 capabilities
7989 );
7990 }
7991
7992 #[test]
7993 fn catalog_list_mirrors_management_operation_description() {
7994 let registry = Arc::new(Registry::default());
7995 let handler = ControlHandler::new(Arc::clone(®istry));
7996 let description = "List managed records and return their identifiers and metadata.";
7997 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
7998 manifest.provides = vec![ProviderRole::ManagementSurface {
7999 operations: vec![ManagementOperation {
8000 name: "records.list".to_string(),
8001 kind: ManagementOperationKind::Query,
8002 description: Some(description.to_string()),
8003 }],
8004 config_schema: json!({"type": "object"}),
8005 observability: vec![ObservabilitySurface {
8006 name: "records.stats".to_string(),
8007 kind: ObservabilityKind::Snapshot,
8008 }],
8009 identity_scope: vec![IdentityScope::Project],
8010 concurrency: Concurrency::ModuleManaged,
8011 }];
8012 registry
8013 .register_with_control_ops(
8014 manifest,
8015 PROTOCOL_VERSION,
8016 ConnectionId::new(99),
8017 Vec::new(),
8018 )
8019 .expect("described management manifest registers");
8020
8021 let request = Frame::build(
8022 FrameType::Request,
8023 control_flags(),
8024 0,
8025 0,
8026 100,
8027 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8028 .expect("catalog request serializes"),
8029 )
8030 .expect("catalog request frame builds");
8031 let response = handler
8032 .handle_catalog_list(request, None)
8033 .expect("catalog list succeeds");
8034 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8035 assert_eq!(
8036 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
8037 "catalog.list must preserve the declared operation description verbatim"
8038 );
8039 }
8040
8041 #[test]
8042 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
8043 let registry = Arc::new(Registry::default());
8044 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8045 [("vault".to_string(), true), ("squatter".to_string(), true)],
8046 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8047 );
8048 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
8049 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8050 provides: vec!["credentials-provider/v1".to_string()],
8051 requires: Vec::new(),
8052 must_never_reach: Vec::new(),
8053 });
8054 let frame = Frame::build(
8055 FrameType::Hello,
8056 control_flags(),
8057 0,
8058 0,
8059 77,
8060 serde_json::to_vec(&ModuleHelloBody {
8061 manifest: squatter,
8062 protocol_ver: PROTOCOL_VERSION,
8063 control_ops: None,
8064 launch_nonce: None,
8065 })
8066 .expect("HELLO serializes"),
8067 )
8068 .expect("HELLO frame builds");
8069 let response = handler
8070 .handle_control(ConnectionId::new(77), frame)
8071 .expect("reserved claim receives a typed refusal");
8072 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8073 assert_eq!(
8074 registry
8075 .active_registration_count()
8076 .expect("registry reads"),
8077 0,
8078 "a reserved capability refusal must not leave a catalog entry"
8079 );
8080 }
8081
8082 #[test]
8083 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
8084 for settlement in ["timeout", "inconclusive", "drop"] {
8085 let breakers = RouteBindBreakers::default();
8086 let RouteBindAdmission::Admitted {
8087 guard: mut old,
8088 probe: false,
8089 } = breakers.admit("prov")
8090 else {
8091 panic!("ordinary relay admitted")
8092 };
8093 let RouteBindAdmission::Admitted {
8094 guard: mut opener, ..
8095 } = breakers.admit("prov")
8096 else {
8097 panic!("second relay admitted")
8098 };
8099 assert!(
8100 !opener
8101 .record_timeout(1, Duration::ZERO)
8102 .unwrap()
8103 .reopened_after_probe
8104 );
8105 let RouteBindAdmission::Admitted {
8106 guard: mut probe,
8107 probe: true,
8108 } = breakers.admit("prov")
8109 else {
8110 panic!("one cooldown probe admitted")
8111 };
8112 match settlement {
8113 "timeout" => assert!(
8114 !old.record_timeout(1, Duration::ZERO)
8115 .unwrap()
8116 .reopened_after_probe
8117 ),
8118 "inconclusive" => old.record_inconclusive(),
8119 "drop" => drop(old),
8120 _ => unreachable!(),
8121 }
8122 assert!(
8123 matches!(
8124 breakers.admit("prov"),
8125 RouteBindAdmission::Refused {
8126 probe_in_flight: true,
8127 ..
8128 }
8129 ),
8130 "{settlement} of a pre-open relay cannot release the real probe"
8131 );
8132 assert!(
8133 probe
8134 .record_timeout(1, Duration::ZERO)
8135 .unwrap()
8136 .reopened_after_probe
8137 );
8138 assert!(matches!(
8139 breakers.admit("prov"),
8140 RouteBindAdmission::Admitted { probe: true, .. }
8141 ));
8142 }
8143 let breakers = RouteBindBreakers::default();
8144 let admit = || match breakers.admit("prov") {
8145 RouteBindAdmission::Admitted { guard, .. } => guard,
8146 _ => panic!("relay admitted"),
8147 };
8148 admit().record_timeout(1, Duration::ZERO);
8149 let mut old_probe = admit();
8150 breakers.reset_for_new_module_connection("prov");
8151 admit().record_timeout(1, Duration::ZERO);
8152 let _new_probe = admit();
8153 old_probe.record_inconclusive();
8154 assert!(matches!(
8155 breakers.admit("prov"),
8156 RouteBindAdmission::Refused {
8157 probe_in_flight: true,
8158 ..
8159 }
8160 ));
8161 }
8162
8163 #[tokio::test]
8164 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
8165 for candidate in [false, true] {
8166 let registry = Arc::new(Registry::default());
8167 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8168 [("vault".to_string(), true), ("squatter".to_string(), true)],
8169 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8170 );
8171 let conn = ConnectionId::new(77);
8172 let (ctx, mut rx) = route_ctx(conn);
8173 let initial = capability_manifest("squatter", &[], &[]);
8174 if candidate {
8175 registry
8176 .register_candidate_with_control_ops(
8177 initial.clone(),
8178 PROTOCOL_VERSION,
8179 conn,
8180 module_baseline_control_ops(),
8181 )
8182 .unwrap();
8183 handler
8184 .forwarding
8185 .register_candidate_module_connection(
8186 conn,
8187 "squatter".to_string(),
8188 PROTOCOL_VERSION,
8189 manifest_concurrency(&initial),
8190 ctx.egress.clone(),
8191 )
8192 .unwrap();
8193 } else {
8194 hello_via_sink(
8195 &handler,
8196 &ctx,
8197 &mut rx,
8198 hello_frame_with_manifest(initial.clone(), 1),
8199 )
8200 .await;
8201 }
8202 let response = handler
8203 .handle_control_frame(
8204 &ctx,
8205 catalog_update_with_capabilities_frame(
8206 2,
8207 capability_manifest("squatter", &["credentials-provider/v1"], &[])
8208 .capabilities
8209 .unwrap(),
8210 ),
8211 )
8212 .await
8213 .unwrap();
8214 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8215 assert_eq!(
8216 registry
8217 .get_module_by_connection(conn)
8218 .unwrap()
8219 .unwrap()
8220 .manifest,
8221 initial
8222 );
8223 }
8224 }
8225
8226 #[test]
8227 fn server_describe_surfaces_required_capability_verdict_fields() {
8228 let registry = Arc::new(Registry::default());
8229 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8230 [
8231 ("consumer".to_string(), true),
8232 ("provider".to_string(), false),
8233 ],
8234 BTreeMap::new(),
8235 );
8236 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
8237 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8238 provides: Vec::new(),
8239 requires: vec![subc_protocol::manifest::CapabilityRequirement {
8240 capability: "credentials-provider/v1".to_string(),
8241 need: subc_protocol::manifest::CapabilityNeed::Required,
8242 }],
8243 must_never_reach: Vec::new(),
8244 });
8245 let hello = Frame::build(
8246 FrameType::Hello,
8247 control_flags(),
8248 0,
8249 0,
8250 78,
8251 serde_json::to_vec(&ModuleHelloBody {
8252 manifest: consumer,
8253 protocol_ver: PROTOCOL_VERSION,
8254 control_ops: None,
8255 launch_nonce: None,
8256 })
8257 .expect("HELLO serializes"),
8258 )
8259 .expect("HELLO frame builds");
8260 handler
8261 .handle_control(ConnectionId::new(78), hello)
8262 .expect("consumer registers");
8263 let describe = Frame::build(
8264 FrameType::Request,
8265 control_flags(),
8266 0,
8267 0,
8268 79,
8269 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
8270 .expect("request serializes"),
8271 )
8272 .expect("describe frame builds");
8273 let response = handler
8274 .handle_server_describe(describe)
8275 .expect("server.describe succeeds");
8276 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
8277 let requirement = &rendered["capability_requirements"][0];
8278 assert_eq!(requirement["consumer"], "consumer");
8279 assert_eq!(requirement["verdict"], "never_provided");
8280 assert_eq!(requirement["episode_seq"], 1);
8281 assert_eq!(requirement["config_satisfiable"], false);
8282 assert_eq!(requirement["runtime_available"], false);
8283 assert!(requirement["detail"]
8284 .as_str()
8285 .expect("detail string")
8286 .contains("credentials-provider/v1"));
8287 }
8288
8289 #[test]
8290 fn catalog_list_omits_capabilities_for_legacy_manifest() {
8291 let registry = Arc::new(Registry::default());
8292 let handler = ControlHandler::new(Arc::clone(®istry));
8293 let hello = handler
8294 .handle_control(
8295 ConnectionId::new(101),
8296 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
8297 )
8298 .expect("legacy HELLO registers");
8299 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
8300
8301 let request = Frame::build(
8302 FrameType::Request,
8303 control_flags(),
8304 0,
8305 0,
8306 102,
8307 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8308 .expect("catalog request serializes"),
8309 )
8310 .expect("catalog request frame builds");
8311 let response = handler
8312 .handle_catalog_list(request, None)
8313 .expect("catalog list succeeds");
8314 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8315 assert!(
8316 body["modules"][0].get("capabilities").is_none(),
8317 "legacy manifest must retain an absent capabilities field on catalog.list"
8318 );
8319 }
8320
8321 #[test]
8322 fn hello_ack_omits_storage_when_no_storage_config() {
8323 let registry = Arc::new(Registry::default());
8324 let handler = ControlHandler::new(Arc::clone(®istry));
8325 let responses = handler
8326 .handle_control(
8327 ConnectionId::new(1),
8328 hello_frame("aft", PROTOCOL_VERSION, 7),
8329 )
8330 .unwrap();
8331 let ack = parse_ack(&responses[0]);
8332 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
8333 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
8334 }
8335
8336 #[tokio::test]
8337 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
8338 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
8339 let registry = Arc::new(Registry::default());
8340 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
8341 let responses = handler
8342 .handle_control(
8343 ConnectionId::new(1),
8344 hello_frame("aft", PROTOCOL_VERSION, 7),
8345 )
8346 .unwrap();
8347 let ack = parse_ack(&responses[0]);
8348 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
8349
8350 let described = handler
8351 .handle_control_frame(
8352 &route_ctx(ConnectionId::new(2)).0,
8353 Frame::build(
8354 FrameType::Request,
8355 control_flags(),
8356 0,
8357 0,
8358 9,
8359 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8360 )
8361 .unwrap(),
8362 )
8363 .await
8364 .unwrap();
8365 let ClientControlResponse::ServerDescribe { machine_id, .. } =
8366 serde_json::from_slice(&described[0].body).unwrap()
8367 else {
8368 panic!("server.describe answered with another shape");
8369 };
8370 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
8371 }
8372
8373 #[test]
8374 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
8375 let registry = Arc::new(Registry::default());
8378 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
8379 crate::daemon_config::StorageConfig::Sqlite {
8380 data_home: std::path::PathBuf::from("/data"),
8381 },
8382 ));
8383
8384 let responses = handler
8385 .handle_control(
8386 ConnectionId::new(1),
8387 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
8388 )
8389 .unwrap();
8390 let ack = parse_ack(&responses[0]);
8391 assert_eq!(
8392 ack.storage,
8393 Some(serde_json::json!({
8394 "module_id": "alfonso-routing",
8395 "storage_namespace": "default",
8396 "isolation": { "kind": "module" },
8397 "backend": {
8398 "backend": "sqlite",
8399 "path": "/data/cortexkit/alfonso-routing/store.db"
8400 }
8401 })),
8402 "the delivered descriptor is the module's own sqlite store path"
8403 );
8404 }
8405
8406 #[test]
8407 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
8408 let registry = Arc::new(Registry::default());
8409 let handler = ControlHandler::new(Arc::clone(®istry));
8410 let conn = ConnectionId::new(1);
8411 let responses = handler
8412 .handle_control(
8413 conn,
8414 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8415 )
8416 .unwrap();
8417 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8418 let registration = registry.get_module("aft").unwrap().unwrap();
8419 assert_eq!(registration.control_ops, module_baseline_control_ops());
8420
8421 let frame =
8422 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
8423 assert!(handler
8424 .guard_module_control_op(&frame, "aft", "route.bind")
8425 .unwrap()
8426 .is_none());
8427 let error = handler
8428 .guard_module_control_op(&frame, "aft", "test.synthetic")
8429 .unwrap()
8430 .expect("synthetic ungranted op should be rejected");
8431 assert_eq!(error.header.ty, FrameType::Error);
8432 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
8433 }
8434
8435 #[test]
8436 fn hello_control_ops_some_adds_optional_grants() {
8437 let registry = Arc::new(Registry::default());
8438 let handler = ControlHandler::new(Arc::clone(®istry));
8439 handler
8440 .handle_control(
8441 ConnectionId::new(1),
8442 hello_frame_with_control_ops(
8443 "aft",
8444 PROTOCOL_VERSION,
8445 7,
8446 Some(vec![
8447 "future.synthetic".to_string(),
8448 "route.bind".to_string(),
8449 ]),
8450 ),
8451 )
8452 .unwrap();
8453 let registration = registry.get_module("aft").unwrap().unwrap();
8454 assert_eq!(
8455 registration.control_ops,
8456 vec![
8457 "route.bind".to_string(),
8458 "route.status".to_string(),
8459 "future.synthetic".to_string(),
8460 ]
8461 );
8462 let frame =
8463 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
8464 assert!(handler
8465 .guard_module_control_op(&frame, "aft", "future.synthetic")
8466 .unwrap()
8467 .is_none());
8468 }
8469
8470 #[tokio::test]
8471 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
8472 let registry = Arc::new(Registry::default());
8473 let forwarding = Arc::new(ForwardingTable::default());
8474 let handler =
8475 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8476 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
8477 hello_via_sink(
8478 &handler,
8479 &module_ctx,
8480 &mut module_rx,
8481 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8482 )
8483 .await;
8484
8485 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
8486 let responses = handler
8487 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
8488 .await
8489 .unwrap();
8490 assert_eq!(responses.len(), 1);
8491 assert_eq!(responses[0].header.ty, FrameType::Error);
8492 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
8493 assert!(module_rx.try_recv().is_err());
8494 }
8495
8496 #[tokio::test]
8497 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
8498 let registry = Arc::new(Registry::default());
8499 let forwarding = Arc::new(ForwardingTable::default());
8500 let handler =
8501 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8502 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
8503 hello_via_sink(
8504 &handler,
8505 &module_ctx,
8506 &mut module_rx,
8507 hello_frame_with_control_ops(
8508 "aft",
8509 PROTOCOL_VERSION,
8510 7,
8511 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8512 ),
8513 )
8514 .await;
8515
8516 let project_root = unique_project_root("demux");
8517 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
8518 let route_handler = handler.clone();
8519 let route_task = tokio::spawn(async move {
8520 route_handler
8521 .handle_control_frame(
8522 &route_client_ctx,
8523 route_open_frame(100, "aft", project_root),
8524 )
8525 .await
8526 .unwrap()
8527 });
8528 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8529 .await
8530 .unwrap()
8531 .unwrap();
8532 assert!(matches!(
8533 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
8534 ModuleControlRequest::RouteBind { .. }
8535 ));
8536
8537 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
8538 let health_handler = handler.clone();
8539 let health_task = tokio::spawn(async move {
8540 health_handler
8541 .handle_control_frame(
8542 &health_client_ctx,
8543 supervisor_health_probe_frame(101, "aft"),
8544 )
8545 .await
8546 .unwrap()
8547 });
8548 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8549 .await
8550 .unwrap()
8551 .unwrap();
8552 assert_eq!(
8553 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8554 ModuleControlRequest::HealthCheck {}
8555 );
8556
8557 handler
8558 .handle_control_frame(
8559 &module_ctx,
8560 health_response(health_frame.header.corr, HealthStatus::Degraded),
8561 )
8562 .await
8563 .unwrap();
8564 let health_response = health_task.await.unwrap();
8565 assert_eq!(health_response.len(), 1);
8566 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8567 ClientControlResponse::SupervisorHealthProbe {
8568 module_id,
8569 status,
8570 detail,
8571 metrics,
8572 } => {
8573 assert_eq!(module_id, "aft");
8574 assert_eq!(status, HealthStatus::Degraded);
8575 assert_eq!(detail.as_deref(), Some("warming"));
8576 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
8577 }
8578 other => panic!("unexpected health response: {other:?}"),
8579 }
8580
8581 handler
8582 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8583 .await
8584 .unwrap();
8585 let route_response = route_task.await.unwrap();
8586 assert!(route_response.is_empty());
8587 let published = route_client_rx.recv().await.unwrap();
8588 assert!(matches!(
8589 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8590 ClientControlResponse::RouteOpen { .. }
8591 ));
8592 }
8593
8594 async fn relay_route_open(
8599 handler: &ControlHandler,
8600 client_connection: ConnectionId,
8601 client_egress: &FrameSink,
8602 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8603 corr: u64,
8604 module_id: &str,
8605 project_root_label: &str,
8606 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
8607 let ctx = RouteCtx {
8608 connection_id: client_connection,
8609 egress: client_egress.clone(),
8610 };
8611 let handler = handler.clone();
8612 let project_root = unique_project_root(project_root_label);
8613 let module_id = module_id.to_string();
8614 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
8615 let task = tokio::spawn(async move {
8616 let _guard = tracing::dispatcher::set_default(&dispatch);
8617 handler
8618 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
8619 .await
8620 .unwrap()
8621 });
8622 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
8623 .await
8624 .expect("module receives the relayed route.bind")
8625 .expect("module egress is open");
8626 (task, bind.frame)
8627 }
8628
8629 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
8630 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
8631 ModuleControlRequest::RouteBind {
8632 route_channel,
8633 epoch,
8634 ..
8635 } => (route_channel, epoch),
8636 other => panic!("expected a route.bind request, got {other:?}"),
8637 }
8638 }
8639
8640 fn published_route(frame: &Frame) -> (u16, u32) {
8641 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
8642 ClientControlResponse::RouteOpen {
8643 route_channel,
8644 route_epoch,
8645 } => (route_channel, route_epoch),
8646 other => panic!("expected a route.open response, got {other:?}"),
8647 }
8648 }
8649
8650 #[tokio::test]
8670 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
8671 let registry = Arc::new(Registry::default());
8672 let forwarding = Arc::new(ForwardingTable::default());
8673 let handler =
8674 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8675
8676 let module_connection = ConnectionId::new(30);
8677 let (module_ctx, mut module_rx) = route_ctx(module_connection);
8678 hello_via_sink(
8679 &handler,
8680 &module_ctx,
8681 &mut module_rx,
8682 hello_frame("aft", PROTOCOL_VERSION, 7),
8683 )
8684 .await;
8685
8686 let dying_client = ConnectionId::new(31);
8687 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
8688
8689 let (first_task, first_bind) = relay_route_open(
8692 &handler,
8693 dying_client,
8694 &dying_ctx.egress,
8695 &mut module_rx,
8696 100,
8697 "aft",
8698 "closing-first",
8699 )
8700 .await;
8701 handler
8702 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
8703 .await
8704 .unwrap();
8705 assert!(first_task.await.unwrap().is_empty());
8706 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
8707
8708 let (second_task, second_bind) = relay_route_open(
8710 &handler,
8711 dying_client,
8712 &dying_ctx.egress,
8713 &mut module_rx,
8714 101,
8715 "aft",
8716 "closing-second",
8717 )
8718 .await;
8719 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
8720
8721 assert!(forwarding
8724 .escalate_client_delivery_failure(
8725 dying_client,
8726 first_channel,
8727 first_epoch,
8728 CloseReason::new(
8729 "module_to_client_delivery_failed",
8730 "client egress refused a module frame",
8731 ),
8732 crate::forwarding::UndeliveredFrame {
8733 module_id: None,
8734 sink: &dying_ctx.egress,
8735 },
8736 )
8737 .unwrap());
8738 assert!(!dying_ctx.egress.is_closed());
8739
8740 let ack = handler
8742 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
8743 .await;
8744 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
8745 if module_loop_error.is_some() {
8746 handler.cleanup_connection(module_connection).unwrap();
8750 }
8751 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8755 .await
8756 .ok()
8757 .flatten();
8758
8759 assert!(
8761 registry
8762 .get_module_by_connection(module_connection)
8763 .unwrap()
8764 .is_some(),
8765 "one client's closing connection ended the shared module connection: \
8766 {module_loop_error:?}"
8767 );
8768 let cotenant = ConnectionId::new(32);
8770 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
8771 let (cotenant_task, cotenant_bind) = relay_route_open(
8772 &handler,
8773 cotenant,
8774 &cotenant_ctx.egress,
8775 &mut module_rx,
8776 102,
8777 "aft",
8778 "closing-cotenant",
8779 )
8780 .await;
8781 handler
8782 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
8783 .await
8784 .unwrap();
8785 assert!(cotenant_task.await.unwrap().is_empty());
8786 let (cotenant_channel, cotenant_epoch) =
8787 published_route(&cotenant_rx.recv().await.unwrap());
8788 assert!(matches!(
8789 forwarding
8790 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
8791 .unwrap(),
8792 DataRoute::Client(DataRouteState::Bound(_))
8793 ));
8794
8795 let goodbye = post_ack_module_frame
8798 .expect("module receives a GOODBYE for the abandoned route channel");
8799 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8800 assert_eq!(goodbye.header.channel, abandoned_channel);
8801 assert_eq!(goodbye.header.epoch, abandoned_epoch);
8802
8803 assert!(dying_rx.try_recv().is_err());
8807 let second_response = second_task.await.unwrap();
8808 assert_eq!(second_response.len(), 1);
8809 assert_eq!(
8810 parse_error(&second_response[0])["code"],
8811 "target_unavailable"
8812 );
8813 }
8814
8815 async fn route_open_held_after_bind_send() -> (
8819 ControlHandler,
8820 Arc<tokio::sync::Barrier>,
8821 RouteCtx,
8822 mpsc::Receiver<crate::router::OutboundFrame>,
8823 ) {
8824 let hold = Arc::new(tokio::sync::Barrier::new(2));
8825 let handler = ControlHandler::with_forwarding(
8826 Arc::new(Registry::default()),
8827 Arc::new(ForwardingTable::default()),
8828 )
8829 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
8830 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
8831 hello_via_sink(
8832 &handler,
8833 &module_ctx,
8834 &mut module_rx,
8835 hello_frame("aft", PROTOCOL_VERSION, 7),
8836 )
8837 .await;
8838 (handler, hold, module_ctx, module_rx)
8839 }
8840
8841 #[tokio::test]
8848 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
8849 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
8850 let client = ConnectionId::new(41);
8851 let (client_ctx, mut client_rx) = route_ctx(client);
8852
8853 let (open_task, bind) = relay_route_open(
8854 &handler,
8855 client,
8856 &client_ctx.egress,
8857 &mut module_rx,
8858 200,
8859 "aft",
8860 "acked-before-mark",
8861 )
8862 .await;
8863 let (module_channel, module_epoch) = route_bind_channel(&bind);
8864 hold.wait().await;
8866
8867 assert!(handler
8869 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
8870 .await
8871 .unwrap()
8872 .is_empty());
8873 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
8874
8875 hold.wait().await;
8876 assert!(
8877 open_task.await.unwrap().is_empty(),
8878 "route.open answers accepted"
8879 );
8880
8881 match module_rx.try_recv() {
8884 Err(mpsc::error::TryRecvError::Empty) => {}
8885 Ok(outbound) => panic!(
8886 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
8887 outbound.frame.header.ty,
8888 outbound.frame.header.channel,
8889 outbound.frame.header.epoch
8890 ),
8891 Err(err) => panic!("module egress closed: {err}"),
8892 }
8893
8894 let router = Router::with_control_handler(Arc::new(handler.clone()));
8897 let request = Frame::build(
8898 FrameType::Request,
8899 Flags::new(true, Priority::Interactive, false),
8900 client_channel,
8901 client_epoch,
8902 201,
8903 b"first request".to_vec(),
8904 )
8905 .unwrap();
8906 router
8907 .route_for_connection(&client_ctx, request)
8908 .await
8909 .unwrap();
8910 let delivered = module_rx.try_recv().unwrap().frame;
8911 assert_eq!(delivered.header.ty, FrameType::Request);
8912 assert_eq!(
8913 (delivered.header.channel, delivered.header.epoch),
8914 (module_channel, module_epoch)
8915 );
8916 assert_eq!(delivered.body, b"first request");
8917 }
8918
8919 #[tokio::test]
8925 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
8926 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
8927 let client = ConnectionId::new(42);
8928 let (client_ctx, _client_rx) = route_ctx(client);
8929
8930 let (open_task, bind) = relay_route_open(
8931 &handler,
8932 client,
8933 &client_ctx.egress,
8934 &mut module_rx,
8935 300,
8936 "aft",
8937 "released-before-mark",
8938 )
8939 .await;
8940 let (module_channel, module_epoch) = route_bind_channel(&bind);
8941 hold.wait().await;
8942
8943 handler.cleanup_connection(client).unwrap();
8944 assert!(
8945 module_rx.try_recv().is_err(),
8946 "client teardown must not GOODBYE a bind route.open is still queueing"
8947 );
8948
8949 hold.wait().await;
8950 let response = open_task.await.unwrap();
8951 assert_eq!(response.len(), 1);
8952 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8953
8954 let goodbye = module_rx
8955 .try_recv()
8956 .expect("module is told to drop the binding the daemon abandoned")
8957 .frame;
8958 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8959 assert_eq!(
8960 (goodbye.header.channel, goodbye.header.epoch),
8961 (module_channel, module_epoch)
8962 );
8963 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
8964 }
8965
8966 #[tokio::test]
8971 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
8972 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
8973 let client = ConnectionId::new(43);
8974 let (client_ctx, client_rx) = route_ctx(client);
8975
8976 let (open_task, bind) = relay_route_open(
8977 &handler,
8978 client,
8979 &client_ctx.egress,
8980 &mut module_rx,
8981 400,
8982 "aft",
8983 "gone-client-before-mark",
8984 )
8985 .await;
8986 let (module_channel, module_epoch) = route_bind_channel(&bind);
8987 hold.wait().await;
8988
8989 drop(client_rx);
8990 assert!(client_ctx.egress.is_closed());
8991 handler
8992 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
8993 .await
8994 .unwrap();
8995
8996 hold.wait().await;
8997 let response = open_task.await.unwrap();
8998 assert_eq!(response.len(), 1);
8999 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9000
9001 let goodbye = module_rx
9002 .try_recv()
9003 .expect("module is told to drop the binding it acked")
9004 .frame;
9005 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9006 assert_eq!(
9007 (goodbye.header.channel, goodbye.header.epoch),
9008 (module_channel, module_epoch)
9009 );
9010 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9011 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
9012 }
9013
9014 #[test]
9021 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
9022 let handler = ControlHandler::default();
9023 let module_connection = ConnectionId::new(30);
9024 let client_connection = ConnectionId::new(31);
9025
9026 handler
9027 .refuse_to_end_module_connection_for_a_client(
9028 module_connection,
9029 77,
9030 ForwardingError::ConnectionClosing {
9031 connection_id: client_connection,
9032 },
9033 )
9034 .expect("a closing client must never end the module connection");
9035
9036 assert!(matches!(
9037 handler.refuse_to_end_module_connection_for_a_client(
9038 module_connection,
9039 78,
9040 ForwardingError::ConnectionClosing {
9041 connection_id: module_connection,
9042 },
9043 ),
9044 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
9045 connection_id
9046 })) if connection_id == module_connection
9047 ));
9048 assert!(matches!(
9049 handler.refuse_to_end_module_connection_for_a_client(
9050 module_connection,
9051 79,
9052 ForwardingError::Poisoned,
9053 ),
9054 Err(RouterError::Forwarding(ForwardingError::Poisoned))
9055 ));
9056 assert!(matches!(
9057 handler.refuse_to_end_module_connection_for_a_client(
9058 module_connection,
9059 80,
9060 ForwardingError::StaleModuleEndpoint,
9061 ),
9062 Err(RouterError::Forwarding(
9063 ForwardingError::StaleModuleEndpoint
9064 ))
9065 ));
9066 }
9067
9068 #[tokio::test]
9081 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
9082 let registry = Arc::new(Registry::default());
9083 let forwarding = Arc::new(ForwardingTable::default());
9084 let supervisor = SupervisorHandle::new();
9085 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9086 let handler =
9087 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9088 .with_supervisor(supervisor);
9089
9090 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9091 hello_via_sink(
9092 &handler,
9093 &target_ctx,
9094 &mut target_rx,
9095 hello_frame("target", PROTOCOL_VERSION, 1),
9096 )
9097 .await;
9098
9099 let wrong_nonce = handler
9103 .handle_control_frame(
9104 &route_ctx(ConnectionId::new(91)).0,
9105 route_open_frame_with_admission_facts(
9106 20,
9107 "target",
9108 unique_project_root("admission-facts"),
9109 Some(subc_control::ConsumerIdentity {
9110 module_id: "fed".to_string(),
9111 launch_nonce: "not-the-real-nonce".to_string(),
9112 }),
9113 None,
9114 ),
9115 )
9116 .await
9117 .unwrap();
9118 assert_eq!(
9119 parse_error(&wrong_nonce[0])["code"],
9120 "bad_consumer_identity",
9121 "a mismatched launch nonce must be refused, not stamped Reserved"
9122 );
9123
9124 let never_spawned = handler
9128 .handle_control_frame(
9129 &route_ctx(ConnectionId::new(92)).0,
9130 route_open_frame_with_admission_facts(
9131 21,
9132 "target",
9133 unique_project_root("admission-facts"),
9134 Some(subc_control::ConsumerIdentity {
9135 module_id: "never-spawned".to_string(),
9136 launch_nonce: "any-nonce".to_string(),
9137 }),
9138 None,
9139 ),
9140 )
9141 .await
9142 .unwrap();
9143 assert_eq!(
9144 parse_error(&never_spawned[0])["code"],
9145 "bad_consumer_identity",
9146 "an unspawned module_id must be refused rather than accepted for lack of a record"
9147 );
9148 }
9149
9150 #[tokio::test]
9173 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
9174 let registry = Arc::new(Registry::default());
9175 let forwarding = Arc::new(ForwardingTable::default());
9176 let supervisor = SupervisorHandle::new();
9177 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9178 let handler =
9179 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9180 .with_supervisor(supervisor);
9181
9182 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
9183 hello_via_sink(
9184 &handler,
9185 &target_ctx,
9186 &mut target_rx,
9187 hello_frame("target", PROTOCOL_VERSION, 1),
9188 )
9189 .await;
9190
9191 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
9192 let route_handler = handler.clone();
9193 let route_task = tokio::spawn(async move {
9194 route_handler
9195 .handle_control_frame(
9196 &client_ctx,
9197 route_open_frame_with_admission_facts(
9198 30,
9199 "target",
9200 unique_project_root("admission-facts"),
9201 Some(subc_control::ConsumerIdentity {
9202 module_id: "fed".to_string(),
9203 launch_nonce: "fed-nonce".to_string(),
9204 }),
9205 None,
9206 ),
9207 )
9208 .await
9209 .unwrap()
9210 });
9211
9212 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9219 .await
9220 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
9221 .expect("module control channel closed before route.bind");
9222 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9223 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
9224 panic!("expected route.bind")
9225 };
9226 assert_eq!(
9227 principal,
9228 Some(Principal::Reserved {
9229 module_id: "fed".to_string()
9230 }),
9231 "a correctly attested consumer must be stamped Reserved for its own id"
9232 );
9233
9234 handler
9235 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9236 .await
9237 .unwrap();
9238 assert!(route_task.await.unwrap().is_empty());
9239 assert!(
9240 matches!(
9241 serde_json::from_slice::<ClientControlResponse>(
9242 &client_rx.recv().await.unwrap().body
9243 )
9244 .unwrap(),
9245 ClientControlResponse::RouteOpen { .. }
9246 ),
9247 "the route must actually open, not merely avoid an error"
9248 );
9249 }
9250
9251 #[tokio::test(start_paused = true)]
9252 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
9253 let registry = Arc::new(Registry::default());
9254 let forwarding = Arc::new(ForwardingTable::default());
9255 let supervisor = SupervisorHandle::new();
9256 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9257 let handler =
9258 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9259 .with_supervisor(supervisor);
9260
9261 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
9262 hello_via_sink(
9263 &handler,
9264 &target_ctx,
9265 &mut target_rx,
9266 hello_frame("target", PROTOCOL_VERSION, 1),
9267 )
9268 .await;
9269
9270 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
9271 let direct_handler = handler.clone();
9272 let direct_open = tokio::spawn(async move {
9273 direct_handler
9274 .handle_control_frame(
9275 &direct_ctx,
9276 route_open_frame(2, "target", unique_project_root("route-census-direct")),
9277 )
9278 .await
9279 .unwrap()
9280 });
9281 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9282 .await
9283 .expect("no direct route.bind within 5s")
9284 .expect("target control channel closed before direct route.bind");
9285 handler
9286 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
9287 .await
9288 .unwrap();
9289 assert!(direct_open.await.unwrap().is_empty());
9290 let _ = direct_rx.recv().await.unwrap();
9291
9292 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
9293 let reserved_handler = handler.clone();
9294 let reserved_open = tokio::spawn(async move {
9295 reserved_handler
9296 .handle_control_frame(
9297 &reserved_ctx,
9298 route_open_frame_with_admission_facts(
9299 3,
9300 "target",
9301 unique_project_root("admission-facts"),
9302 Some(ConsumerIdentity {
9303 module_id: "fed".to_string(),
9304 launch_nonce: "fed-nonce".to_string(),
9305 }),
9306 None,
9307 ),
9308 )
9309 .await
9310 .unwrap()
9311 });
9312 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9313 .await
9314 .expect("no reserved route.bind within 5s")
9315 .expect("target control channel closed before reserved route.bind");
9316 handler
9317 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
9318 .await
9319 .unwrap();
9320 assert!(reserved_open.await.unwrap().is_empty());
9321 let _ = reserved_rx.recv().await.unwrap();
9322
9323 forwarding
9324 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
9325 .unwrap();
9326 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
9327 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
9328 module_id: Some("target".to_string()),
9329 })
9330 .unwrap();
9331 let census_frame =
9332 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
9333 let response = handler
9334 .handle_control_frame(&census_ctx, census_frame)
9335 .await
9336 .unwrap()
9337 .pop()
9338 .unwrap();
9339 let actual: Value = serde_json::from_slice(&response.body).unwrap();
9340 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
9341 assert!(matches!(
9342 decoded,
9343 ClientControlResponse::SupervisorRoutes { .. }
9344 ));
9345 let routes = actual["modules"][0]["routes"].as_array().unwrap();
9346 assert_eq!(routes.len(), 2);
9347 assert!(routes.iter().all(|route| route["draining"] == true));
9348 assert!(
9351 routes.iter().all(|route| route["drain_reason"] == "reload"),
9352 "draining routes must name the drain's reason: {routes:?}"
9353 );
9354 assert!(routes.iter().any(|route| {
9355 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
9356 }));
9357 assert!(routes.iter().any(|route| {
9358 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
9359 }));
9360
9361 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9362 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
9363 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9364 std::fs::write(
9365 &golden_path,
9366 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
9367 )
9368 .unwrap();
9369 }
9370 let expected: Value =
9371 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
9372 assert_eq!(actual, expected);
9373 }
9374
9375 async fn query_live_roots(
9376 handler: &ControlHandler,
9377 module_ctx: &RouteCtx,
9378 ) -> ModuleControlResponseToModule {
9379 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
9380 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
9381 let response = handler
9382 .handle_control_frame(module_ctx, frame)
9383 .await
9384 .unwrap()
9385 .pop()
9386 .unwrap();
9387 serde_json::from_slice(&response.body).unwrap()
9388 }
9389
9390 #[tokio::test(start_paused = true)]
9391 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
9392 let registry = Arc::new(Registry::default());
9393 let forwarding = Arc::new(ForwardingTable::default());
9394 let handler = ControlHandler::with_forwarding(registry, forwarding);
9395 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
9396 hello_via_sink(
9397 &handler,
9398 &target_ctx,
9399 &mut target_rx,
9400 hello_frame("target", PROTOCOL_VERSION, 1),
9401 )
9402 .await;
9403 let root = unique_project_root("live-roots-known");
9404 let path = ProjectRootId::from_path_allowing_missing(root.path())
9405 .unwrap()
9406 .as_path()
9407 .to_path_buf();
9408 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
9409 let open_handler = handler.clone();
9410 let opened = tokio::spawn(async move {
9411 open_handler
9412 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
9413 .await
9414 .unwrap()
9415 });
9416 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9417 .await
9418 .unwrap()
9419 .unwrap();
9420 handler
9421 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
9422 .await
9423 .unwrap();
9424 assert!(opened.await.unwrap().is_empty());
9425 let _ = client_rx.recv().await.unwrap();
9426
9427 let root = unique_project_root("live-roots-pending");
9428 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
9429 .unwrap()
9430 .as_path()
9431 .to_path_buf();
9432 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
9433 let open_handler = handler.clone();
9434 let pending = tokio::spawn(async move {
9435 open_handler
9436 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
9437 .await
9438 .unwrap()
9439 });
9440 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9441 .await
9442 .unwrap()
9443 .unwrap();
9444 let actual = query_live_roots(&handler, &target_ctx).await;
9445 let ModuleControlResponseToModule::LiveRoots {
9446 roots,
9447 unknown_root_bindings,
9448 total_bindings,
9449 } = actual
9450 else {
9451 panic!("expected live roots")
9452 };
9453 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
9454 assert_eq!(unknown_root_bindings, 0);
9455 assert_eq!(
9456 roots.len(),
9457 2,
9458 "root-known arm must retain each canonical root"
9459 );
9460 assert_eq!(
9461 total_bindings,
9462 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9463 );
9464 let counts = roots
9465 .iter()
9466 .map(|root| (root.project_root.clone(), root.bound, root.pending))
9467 .collect::<Vec<_>>();
9468 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
9469 expected.sort_by(|a, b| a.0.cmp(&b.0));
9470 assert_eq!(
9471 counts, expected,
9472 "roots must sort by path and count pending separately"
9473 );
9474 handler
9475 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
9476 .await
9477 .unwrap();
9478 assert!(pending.await.unwrap().is_empty());
9479 }
9480
9481 #[tokio::test(start_paused = true)]
9482 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
9483 let forwarding = Arc::new(ForwardingTable::default());
9484 let handler =
9485 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
9486 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
9487 hello_via_sink(
9488 &handler,
9489 &target_ctx,
9490 &mut target_rx,
9491 hello_frame("target", PROTOCOL_VERSION, 1),
9492 )
9493 .await;
9494 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
9495 let pending = forwarding
9496 .begin_route_bind_relay_for_test(
9497 client_ctx.connection_id,
9498 client_ctx.egress.clone(),
9499 2,
9500 "target",
9501 )
9502 .unwrap();
9503 forwarding
9504 .complete_pending_relay(
9505 target_ctx.connection_id,
9506 pending.corr,
9507 RouteBindRelayOutcome::Accepted,
9508 )
9509 .unwrap();
9510 let actual = query_live_roots(&handler, &target_ctx).await;
9511 let ModuleControlResponseToModule::LiveRoots {
9512 roots,
9513 unknown_root_bindings,
9514 total_bindings,
9515 } = actual
9516 else {
9517 panic!("expected live roots")
9518 };
9519 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
9520 assert_eq!(
9521 unknown_root_bindings, 1,
9522 "unknown-root arm must not read as no bindings"
9523 );
9524 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
9525 assert_eq!(
9526 total_bindings,
9527 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9528 );
9529 }
9530
9531 #[tokio::test(start_paused = true)]
9539 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
9540 let forwarding = Arc::new(ForwardingTable::default());
9541 let handler =
9542 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
9543 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
9544 let replies = handler
9545 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
9546 .await
9547 .unwrap();
9548 let queued_by_hello = module_rx.len();
9549
9550 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
9551 let open_handler = handler.clone();
9552 let open = tokio::spawn(async move {
9553 open_handler
9554 .handle_control_frame(
9555 &client_ctx,
9556 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
9557 )
9558 .await
9559 .unwrap()
9560 });
9561 let mut spins = 0;
9563 while module_rx.len() == queued_by_hello {
9564 spins += 1;
9565 assert!(spins < 10_000, "route.open never queued a route.bind");
9566 tokio::task::yield_now().await;
9567 }
9568
9569 for reply in replies {
9571 module_ctx.egress.send(reply).await.unwrap();
9572 }
9573
9574 let first = module_rx.recv().await.unwrap().frame;
9575 assert_eq!(
9576 first.header.ty,
9577 FrameType::HelloAck,
9578 "the first frame a registering module reads must be its HELLO_ACK"
9579 );
9580 assert_eq!(first.header.corr, 7);
9581 let second = module_rx.recv().await.unwrap().frame;
9582 assert_eq!(second.header.ty, FrameType::Request);
9583 assert!(
9584 matches!(
9585 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
9586 ModuleControlRequest::RouteBind { .. }
9587 ),
9588 "the route.bind follows the ack"
9589 );
9590 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
9591
9592 handler
9593 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
9594 .await
9595 .unwrap();
9596 assert!(open.await.unwrap().is_empty());
9597 let _ = client_rx.recv().await.unwrap();
9598 }
9599
9600 #[tokio::test(start_paused = true)]
9601 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
9602 let handler = ControlHandler::with_forwarding(
9603 Arc::new(Registry::default()),
9604 Arc::new(ForwardingTable::default()),
9605 );
9606 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
9607 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
9608 hello_via_sink(
9609 &handler,
9610 &first_ctx,
9611 &mut first_rx,
9612 hello_frame("first", PROTOCOL_VERSION, 1),
9613 )
9614 .await;
9615 hello_via_sink(
9616 &handler,
9617 &second_ctx,
9618 &mut second_rx,
9619 hello_frame("second", PROTOCOL_VERSION, 2),
9620 )
9621 .await;
9622 let root = unique_project_root("second-only");
9623 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
9624 let cloned = handler.clone();
9625 let open = tokio::spawn(async move {
9626 cloned
9627 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
9628 .await
9629 .unwrap()
9630 });
9631 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
9632 .await
9633 .unwrap()
9634 .unwrap();
9635 let first = query_live_roots(&handler, &first_ctx).await;
9636 let second = query_live_roots(&handler, &second_ctx).await;
9637 assert!(
9638 matches!(
9639 first,
9640 ModuleControlResponseToModule::LiveRoots {
9641 total_bindings: 0,
9642 ..
9643 }
9644 ),
9645 "cross-module scope must not expose another module's roots"
9646 );
9647 assert!(
9648 matches!(
9649 second,
9650 ModuleControlResponseToModule::LiveRoots {
9651 total_bindings: 1,
9652 ..
9653 }
9654 ),
9655 "second module must see its pending route"
9656 );
9657 handler
9658 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
9659 .await
9660 .unwrap();
9661 assert!(open.await.unwrap().is_empty());
9662 }
9663
9664 #[tokio::test(start_paused = true)]
9665 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
9666 let handler = ControlHandler::with_forwarding(
9667 Arc::new(Registry::default()),
9668 Arc::new(ForwardingTable::default()),
9669 );
9670 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
9671 hello_via_sink(
9672 &handler,
9673 &target_ctx,
9674 &mut target_rx,
9675 hello_frame("target", PROTOCOL_VERSION, 1),
9676 )
9677 .await;
9678 let actual = query_live_roots(&handler, &target_ctx).await;
9679 let ModuleControlResponseToModule::LiveRoots {
9680 roots,
9681 unknown_root_bindings,
9682 total_bindings,
9683 } = actual
9684 else {
9685 panic!("expected live roots")
9686 };
9687 assert!(roots.is_empty());
9688 assert_eq!(unknown_root_bindings, 0);
9689 assert_eq!(total_bindings, 0);
9690 assert_eq!(
9691 total_bindings,
9692 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9693 );
9694 }
9695
9696 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
9703 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9704 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
9705 let text = std::fs::read_to_string(&path)
9706 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
9707 let vectors: Vec<(String, Value)> = text
9708 .lines()
9709 .filter(|line| !line.trim().is_empty())
9710 .map(|line| {
9711 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
9712 let id = entry["corpus_id"]
9713 .as_str()
9714 .expect("every vector carries a corpus_id")
9715 .to_string();
9716 (id, entry["package"].clone())
9717 })
9718 .collect();
9719 assert_eq!(
9723 vectors.len(),
9724 3,
9725 "vendored fed corpus changed size; re-sync from subc-federation"
9726 );
9727
9728 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
9743 let richest = vectors
9744 .iter()
9745 .filter_map(|(_, package)| package.as_object())
9746 .map(|object| {
9747 object
9748 .keys()
9749 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
9750 .count()
9751 })
9752 .max()
9753 .unwrap_or(0);
9754 assert!(
9755 richest >= 2,
9756 "vendored corpus no longer carries a package with unmodelled fields, \
9757 so the relay test can no longer distinguish a verbatim relay from a lossy one"
9758 );
9759
9760 vectors
9761 }
9762
9763 #[tokio::test]
9771 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
9772 for (corpus_id, package) in fed_admission_facts_vectors() {
9773 let registry = Arc::new(Registry::default());
9774 let forwarding = Arc::new(ForwardingTable::default());
9775 let supervisor = SupervisorHandle::new();
9776 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9777 let handler =
9778 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9779 .with_supervisor(supervisor)
9780 .with_admission_facts_config(
9781 Some("fed".to_string()),
9782 Some(vec!["target".to_string()]),
9783 );
9784
9785 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9786 hello_via_sink(
9787 &handler,
9788 &target_ctx,
9789 &mut target_rx,
9790 hello_frame("target", PROTOCOL_VERSION, 1),
9791 )
9792 .await;
9793
9794 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
9795 let route_handler = handler.clone();
9796 let expected = package.clone();
9797 let route_task = tokio::spawn(async move {
9798 route_handler
9799 .handle_control_frame(
9800 &client_ctx,
9801 route_open_frame_with_admission_facts(
9802 20,
9803 "target",
9804 unique_project_root("admission-facts"),
9805 Some(subc_control::ConsumerIdentity {
9806 module_id: "fed".to_string(),
9807 launch_nonce: "fed-nonce".to_string(),
9808 }),
9809 Some(package),
9810 ),
9811 )
9812 .await
9813 .unwrap()
9814 });
9815
9816 let bind_frame = target_rx.recv().await.unwrap();
9817 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9818 let ModuleControlRequest::RouteBind {
9819 admission_facts, ..
9820 } = bind
9821 else {
9822 panic!("{corpus_id}: expected route.bind")
9823 };
9824 assert_eq!(
9825 admission_facts,
9826 Some(expected),
9827 "{corpus_id}: relay must not add, drop or reshape any field"
9828 );
9829
9830 handler
9831 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9832 .await
9833 .unwrap();
9834 route_task.await.unwrap();
9835 }
9836 }
9837
9838 #[tokio::test]
9839 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
9840 let registry = Arc::new(Registry::default());
9841 let forwarding = Arc::new(ForwardingTable::default());
9842 let supervisor = SupervisorHandle::new();
9843 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9844 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
9845 let handler =
9846 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9847 .with_supervisor(supervisor)
9848 .with_admission_facts_config(
9849 Some("fed".to_string()),
9850 Some(vec!["target".to_string()]),
9851 );
9852
9853 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
9854 hello_via_sink(
9855 &handler,
9856 &target_ctx,
9857 &mut target_rx,
9858 hello_frame("target", PROTOCOL_VERSION, 1),
9859 )
9860 .await;
9861 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
9862 hello_via_sink(
9863 &handler,
9864 &other_ctx,
9865 &mut other_rx,
9866 hello_frame("other", PROTOCOL_VERSION, 2),
9867 )
9868 .await;
9869
9870 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
9871 let expected_facts = facts.clone();
9872 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
9873 let route_handler = handler.clone();
9874 let route_task = tokio::spawn(async move {
9875 route_handler
9876 .handle_control_frame(
9877 &client_ctx,
9878 route_open_frame_with_admission_facts(
9879 10,
9880 "target",
9881 unique_project_root("admission-facts"),
9882 Some(subc_control::ConsumerIdentity {
9883 module_id: "fed".to_string(),
9884 launch_nonce: "fed-nonce".to_string(),
9885 }),
9886 Some(facts.clone()),
9887 ),
9888 )
9889 .await
9890 .unwrap()
9891 });
9892 let bind_frame = target_rx.recv().await.unwrap();
9893 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9894 let ModuleControlRequest::RouteBind {
9895 admission_facts, ..
9896 } = bind
9897 else {
9898 panic!("expected route.bind")
9899 };
9900 assert_eq!(admission_facts, Some(expected_facts));
9901 handler
9902 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9903 .await
9904 .unwrap();
9905 assert!(route_task.await.unwrap().is_empty());
9906 assert!(matches!(
9907 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
9908 .unwrap(),
9909 ClientControlResponse::RouteOpen { .. }
9910 ));
9911
9912 let direct = handler
9913 .handle_control_frame(
9914 &route_ctx(ConnectionId::new(73)).0,
9915 route_open_frame_with_admission_facts(
9916 11,
9917 "target",
9918 unique_project_root("admission-facts"),
9919 None,
9920 Some(json!({"x": 1})),
9921 ),
9922 )
9923 .await
9924 .unwrap();
9925 assert_eq!(
9926 parse_error(&direct[0])["code"],
9927 "admission_facts_not_permitted"
9928 );
9929
9930 let different_reserved = handler
9931 .handle_control_frame(
9932 &route_ctx(ConnectionId::new(77)).0,
9933 route_open_frame_with_admission_facts(
9934 15,
9935 "target",
9936 unique_project_root("admission-facts"),
9937 Some(subc_control::ConsumerIdentity {
9938 module_id: "other".to_string(),
9939 launch_nonce: "other-nonce".to_string(),
9940 }),
9941 Some(json!({"x": 1})),
9942 ),
9943 )
9944 .await
9945 .unwrap();
9946 assert_eq!(
9947 parse_error(&different_reserved[0])["code"],
9948 "admission_facts_not_permitted"
9949 );
9950
9951 let other_target = handler
9952 .handle_control_frame(
9953 &route_ctx(ConnectionId::new(74)).0,
9954 route_open_frame_with_admission_facts(
9955 12,
9956 "other",
9957 unique_project_root("admission-facts"),
9958 Some(subc_control::ConsumerIdentity {
9959 module_id: "fed".to_string(),
9960 launch_nonce: "fed-nonce".to_string(),
9961 }),
9962 Some(json!({"x": 1})),
9963 ),
9964 )
9965 .await
9966 .unwrap();
9967 assert_eq!(
9968 parse_error(&other_target[0])["code"],
9969 "admission_facts_target_not_allowed"
9970 );
9971
9972 let nonexistent = handler
9973 .handle_control_frame(
9974 &route_ctx(ConnectionId::new(75)).0,
9975 route_open_frame_with_admission_facts(
9976 13,
9977 "missing",
9978 unique_project_root("admission-facts"),
9979 None,
9980 Some(json!({"x": 1})),
9981 ),
9982 )
9983 .await
9984 .unwrap();
9985 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
9986
9987 let described = handler
9988 .handle_control_frame(
9989 &route_ctx(ConnectionId::new(76)).0,
9990 Frame::build(
9991 FrameType::Request,
9992 control_flags(),
9993 0,
9994 0,
9995 14,
9996 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9997 )
9998 .unwrap(),
9999 )
10000 .await
10001 .unwrap();
10002 let ClientControlResponse::ServerDescribe { capabilities, .. } =
10003 serde_json::from_slice(&described[0].body).unwrap()
10004 else {
10005 panic!("expected server.describe response")
10006 };
10007 assert!(capabilities
10008 .iter()
10009 .any(|cap| cap == "admission_facts_relay_v1"));
10010 }
10011
10012 #[tokio::test]
10013 async fn admission_facts_without_configured_carrier_are_rejected() {
10014 let registry = Arc::new(Registry::default());
10015 let forwarding = Arc::new(ForwardingTable::default());
10016 let handler = ControlHandler::with_forwarding(registry, forwarding);
10017 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
10018 hello_via_sink(
10019 &handler,
10020 &target_ctx,
10021 &mut target_rx,
10022 hello_frame("target", PROTOCOL_VERSION, 1),
10023 )
10024 .await;
10025
10026 let responses = handler
10027 .handle_control_frame(
10028 &route_ctx(ConnectionId::new(79)).0,
10029 route_open_frame_with_admission_facts(
10030 16,
10031 "target",
10032 unique_project_root("admission-facts"),
10033 None,
10034 Some(json!({"x": 1})),
10035 ),
10036 )
10037 .await
10038 .unwrap();
10039 assert_eq!(
10040 parse_error(&responses[0])["code"],
10041 "admission_facts_not_permitted"
10042 );
10043 }
10044
10045 #[tokio::test]
10046 async fn route_open_relays_consumer_capabilities_verbatim() {
10047 let registry = Arc::new(Registry::default());
10048 let forwarding = Arc::new(ForwardingTable::default());
10049 let handler =
10050 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10051 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
10052 hello_via_sink(
10053 &handler,
10054 &module_ctx,
10055 &mut module_rx,
10056 hello_frame("aft", PROTOCOL_VERSION, 7),
10057 )
10058 .await;
10059
10060 let expected = vec!["elicitation".to_string(), "roots".to_string()];
10061 let expected_for_request = expected.clone();
10062 let project_root = unique_project_root("consumer-capabilities-present");
10063 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
10064 let route_handler = handler.clone();
10065 let route_task = tokio::spawn(async move {
10066 route_handler
10067 .handle_control_frame(
10068 &client_ctx,
10069 route_open_frame_with_consumer_capabilities(
10070 401,
10071 "aft",
10072 project_root,
10073 Some(expected_for_request),
10074 ),
10075 )
10076 .await
10077 .unwrap()
10078 });
10079 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10080 .await
10081 .unwrap()
10082 .unwrap();
10083 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10084 let ModuleControlRequest::RouteBind {
10085 consumer_capabilities,
10086 ..
10087 } = bind
10088 else {
10089 panic!("expected route.bind request, got {bind:?}");
10090 };
10091 assert_eq!(consumer_capabilities, Some(expected.clone()));
10092
10093 handler
10094 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10095 .await
10096 .unwrap();
10097 let route_response = route_task.await.unwrap();
10098 assert!(route_response.is_empty());
10099 let published = client_rx.recv().await.unwrap();
10100 assert!(matches!(
10101 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10102 ClientControlResponse::RouteOpen { .. }
10103 ));
10104 }
10105
10106 #[tokio::test]
10107 async fn route_open_without_consumer_capabilities_relays_none() {
10108 let registry = Arc::new(Registry::default());
10109 let forwarding = Arc::new(ForwardingTable::default());
10110 let handler =
10111 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10112 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
10113 hello_via_sink(
10114 &handler,
10115 &module_ctx,
10116 &mut module_rx,
10117 hello_frame("aft", PROTOCOL_VERSION, 7),
10118 )
10119 .await;
10120
10121 let project_root = unique_project_root("consumer-capabilities-absent");
10122 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
10123 let route_handler = handler.clone();
10124 let route_task = tokio::spawn(async move {
10125 route_handler
10126 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
10127 .await
10128 .unwrap()
10129 });
10130 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10131 .await
10132 .unwrap()
10133 .unwrap();
10134 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10135 let ModuleControlRequest::RouteBind {
10136 consumer_capabilities,
10137 ..
10138 } = bind
10139 else {
10140 panic!("expected route.bind request, got {bind:?}");
10141 };
10142 assert_eq!(consumer_capabilities, None);
10143
10144 handler
10145 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10146 .await
10147 .unwrap();
10148 let route_response = route_task.await.unwrap();
10149 assert!(route_response.is_empty());
10150 let published = client_rx.recv().await.unwrap();
10151 assert!(matches!(
10152 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10153 ClientControlResponse::RouteOpen { .. }
10154 ));
10155 }
10156
10157 async fn bind_role_versions_for(
10161 sent: Option<BTreeMap<String, String>>,
10162 connection: u64,
10163 ) -> Option<BTreeMap<String, String>> {
10164 let registry = Arc::new(Registry::default());
10165 let forwarding = Arc::new(ForwardingTable::default());
10166 let handler =
10167 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10168 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
10169 hello_via_sink(
10170 &handler,
10171 &module_ctx,
10172 &mut module_rx,
10173 hello_frame("aft", PROTOCOL_VERSION, 7),
10174 )
10175 .await;
10176 let project_root = unique_project_root("role-versions");
10177 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
10178 let route_handler = handler.clone();
10179 let route_task = tokio::spawn(async move {
10180 route_handler
10181 .handle_control_frame(
10182 &client_ctx,
10183 route_open_frame_with_role_versions(403, "aft", project_root, sent),
10184 )
10185 .await
10186 .unwrap()
10187 });
10188 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10189 .await
10190 .expect("a well-formed route.open reaches the module as a bind")
10191 .unwrap();
10192 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10193 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
10194 panic!("expected route.bind request, got {bind:?}");
10195 };
10196 handler
10197 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10198 .await
10199 .unwrap();
10200 assert!(route_task.await.unwrap().is_empty());
10201 let published = client_rx.recv().await.unwrap();
10202 assert!(matches!(
10203 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10204 ClientControlResponse::RouteOpen { .. }
10205 ));
10206 role_versions
10207 }
10208
10209 #[tokio::test]
10210 async fn route_open_relays_role_versions_verbatim() {
10211 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
10212 assert_eq!(
10213 bind_role_versions_for(Some(sent.clone()), 141).await,
10214 Some(sent)
10215 );
10216 }
10217
10218 #[tokio::test]
10221 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
10222 assert_eq!(bind_role_versions_for(None, 143).await, None);
10223 assert_eq!(
10224 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
10225 None
10226 );
10227 }
10228
10229 #[tokio::test]
10230 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
10231 let registry = Arc::new(Registry::default());
10232 let forwarding = Arc::new(ForwardingTable::default());
10233 let handler =
10234 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10235 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
10236 hello_via_sink(
10237 &handler,
10238 &module_ctx,
10239 &mut module_rx,
10240 hello_frame("aft", PROTOCOL_VERSION, 7),
10241 )
10242 .await;
10243
10244 let nine: BTreeMap<String, String> = (0..9)
10245 .map(|index| (format!("role-{index}"), "v1".to_string()))
10246 .collect();
10247 for (label, malformed) in [
10248 (
10249 "invalid role name",
10250 role_versions(&[("Tool_Provider", "v1")]),
10251 ),
10252 ("invalid version", role_versions(&[("tool-provider", "v0")])),
10253 ("nine entries", nine),
10254 ] {
10255 let responses = tokio::time::timeout(
10258 Duration::from_secs(1),
10259 handler.handle_control_frame(
10260 &route_ctx(ConnectionId::new(148)).0,
10261 route_open_frame_with_role_versions(
10262 404,
10263 "aft",
10264 unique_project_root("role-versions-malformed"),
10265 Some(malformed),
10266 ),
10267 ),
10268 )
10269 .await
10270 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
10271 .unwrap();
10272 assert_eq!(responses.len(), 1, "{label}");
10273 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
10274 let error = parse_error(&responses[0]);
10275 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
10276 assert_eq!(
10277 error["detail"]["field"], "role_versions",
10278 "{label}: {error}"
10279 );
10280 assert!(
10281 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
10282 "{label}: a malformed declaration is terminal"
10283 );
10284 assert!(
10285 module_rx.try_recv().is_err(),
10286 "{label}: the module must never see a bind"
10287 );
10288 }
10289 }
10290
10291 #[tokio::test]
10295 async fn route_role_versions_capability_is_advertised() {
10296 let handler = ControlHandler::new(Arc::new(Registry::default()));
10297 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
10298 let ack = hello_via_sink(
10299 &handler,
10300 &ctx,
10301 &mut rx,
10302 hello_frame("m", PROTOCOL_VERSION, 1),
10303 )
10304 .await;
10305 let ack = parse_ack(&ack);
10306 assert!(
10307 ack.subc_capabilities
10308 .iter()
10309 .any(|c| c == "route-role-versions/v1"),
10310 "{:?}",
10311 ack.subc_capabilities
10312 );
10313
10314 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
10315 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
10316 let reply = handler
10317 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
10318 .await
10319 .unwrap()
10320 .pop()
10321 .unwrap();
10322 let ClientControlResponse::ServerDescribe { capabilities, .. } =
10323 serde_json::from_slice(&reply.body).unwrap()
10324 else {
10325 panic!("not a server.describe reply");
10326 };
10327 assert!(
10328 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
10329 "{capabilities:?}"
10330 );
10331 }
10332
10333 #[tokio::test]
10337 async fn catalog_events_capability_is_advertised() {
10338 let handler = ControlHandler::new(Arc::new(Registry::default()));
10339 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
10340 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
10341 let reply = handler
10342 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
10343 .await
10344 .unwrap()
10345 .pop()
10346 .unwrap();
10347 let ClientControlResponse::ServerDescribe { capabilities, .. } =
10348 serde_json::from_slice(&reply.body).unwrap()
10349 else {
10350 panic!("not a server.describe reply");
10351 };
10352 assert!(
10353 capabilities
10354 .iter()
10355 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
10356 "{capabilities:?}"
10357 );
10358 }
10359
10360 #[tokio::test]
10361 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
10362 let registry = Arc::new(Registry::default());
10363 let forwarding = Arc::new(ForwardingTable::default());
10364 let handler =
10365 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10366 .with_health_probe_timeout(Duration::from_secs(5));
10367 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
10368 hello_via_sink(
10369 &handler,
10370 &module_ctx,
10371 &mut module_rx,
10372 non_routable_hello_frame_with_control_ops(
10373 "mcp",
10374 300,
10375 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10376 ),
10377 )
10378 .await;
10379 assert!(registry
10380 .get_module("mcp")
10381 .unwrap()
10382 .unwrap()
10383 .manifest
10384 .provides
10385 .is_empty());
10386
10387 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
10388 let route_response = handler
10389 .handle_control_frame(
10390 &route_client_ctx,
10391 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
10392 )
10393 .await
10394 .unwrap();
10395 assert_eq!(route_response[0].header.ty, FrameType::Error);
10396 assert_eq!(
10397 parse_error(&route_response[0])["code"],
10398 "target_unavailable"
10399 );
10400 assert!(parse_error(&route_response[0])["message"]
10401 .as_str()
10402 .unwrap()
10403 .contains("does not provide the requested target"));
10404 assert!(module_rx.try_recv().is_err());
10405
10406 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
10407 let health_handler = handler.clone();
10408 let health_task = tokio::spawn(async move {
10409 health_handler
10410 .handle_control_frame(
10411 &health_client_ctx,
10412 supervisor_health_probe_frame(302, "mcp"),
10413 )
10414 .await
10415 .unwrap()
10416 });
10417 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10418 .await
10419 .unwrap()
10420 .unwrap();
10421 assert_eq!(
10422 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10423 ModuleControlRequest::HealthCheck {}
10424 );
10425 handler
10426 .handle_control_frame(
10427 &module_ctx,
10428 health_response(health_frame.header.corr, HealthStatus::Ok),
10429 )
10430 .await
10431 .unwrap();
10432 let health_response = health_task.await.unwrap();
10433 assert_eq!(health_response[0].header.ty, FrameType::Response);
10434 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10435 ClientControlResponse::SupervisorHealthProbe {
10436 module_id, status, ..
10437 } => {
10438 assert_eq!(module_id, "mcp");
10439 assert_eq!(status, HealthStatus::Ok);
10440 }
10441 other => panic!("unexpected health response: {other:?}"),
10442 }
10443
10444 forwarding
10449 .cleanup_connection(module_ctx.connection_id)
10450 .unwrap();
10451 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
10452 let cleanup_response = tokio::time::timeout(
10453 Duration::from_millis(200),
10454 handler.handle_control_frame(
10455 &cleanup_probe_ctx,
10456 supervisor_health_probe_frame(303, "mcp"),
10457 ),
10458 )
10459 .await
10460 .expect("probe should fail immediately when the control lane is gone")
10461 .unwrap();
10462 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
10463 assert_eq!(
10464 parse_error(&cleanup_response[0])["code"],
10465 "target_unavailable"
10466 );
10467 assert!(parse_error(&cleanup_response[0])["message"]
10468 .as_str()
10469 .unwrap()
10470 .contains("no module connection"));
10471
10472 handler
10473 .cleanup_connection(module_ctx.connection_id)
10474 .unwrap();
10475 }
10476
10477 #[tokio::test]
10478 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
10479 let registry = Arc::new(Registry::default());
10480 let supervisor_handle = SupervisorHandle::new();
10481 let supervisor =
10482 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
10483 .with_handle(supervisor_handle.clone())
10484 .with_connection_file_path(
10485 std::env::temp_dir()
10486 .join(format!("subc-route-open-warming-{}", std::process::id())),
10487 );
10488 let module = supervisor
10489 .supervise_configured(
10490 ModuleSpec {
10491 module_id: "warming".to_string(),
10492 program: fake_aft_stub_path(),
10493 args: Vec::new(),
10494 env: Vec::new(),
10495 reserved: false,
10496 reserved_prefixes: Vec::new(),
10497 protocol: ModuleProtocol::Subc,
10498 overlap: Default::default(),
10499 },
10500 true,
10501 )
10502 .unwrap();
10503 assert_eq!(module.state().unwrap(), ModuleState::Running);
10504
10505 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10506 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
10507 let response = handler
10508 .handle_control_frame(
10509 &ctx,
10510 route_open_frame(304, "warming", unique_project_root("warming")),
10511 )
10512 .await
10513 .unwrap();
10514 module.stop().await.unwrap();
10515
10516 assert_eq!(response[0].header.ty, FrameType::Error);
10517 let error = parse_error(&response[0]);
10518 assert_eq!(error["code"], "module_warming");
10519 assert!(error["message"]
10520 .as_str()
10521 .unwrap()
10522 .contains("state=running, enabled=true, live=false"));
10523 }
10524
10525 #[test]
10526 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
10527 let handler = ControlHandler::new(Arc::new(Registry::default()));
10528 let capture = EventCapture::default();
10529 let _subscriber =
10530 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10531 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
10532 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
10533 let pending = (0..limit).collect::<Vec<_>>();
10534 let response = handler
10535 .route_open_capacity_refusal(
10536 &ctx,
10537 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
10538 "busy",
10539 pending.len(),
10540 limit,
10541 )
10542 .unwrap();
10543 assert_eq!(parse_error(&response)["code"], "target_unavailable");
10544 let event = capture
10545 .events()
10546 .into_iter()
10547 .find(|event| {
10548 event.target == "control"
10549 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
10550 })
10551 .expect("connection admission refusal event");
10552 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
10553 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
10554 }
10555
10556 #[test]
10557 fn route_open_target_cap_logs_admission_reason_and_capacity() {
10558 let handler = ControlHandler::new(Arc::new(Registry::default()));
10559 let capture = EventCapture::default();
10560 let _subscriber =
10561 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10562 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
10563 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
10564 let guards = (0..limit)
10565 .map(|_| {
10566 handler
10567 .route_bind_concurrency
10568 .try_admit("busy", limit)
10569 .unwrap()
10570 })
10571 .collect::<Vec<_>>();
10572 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
10573 Err(in_flight) => in_flight,
10574 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
10575 };
10576 let response = handler
10577 .route_open_target_capacity_refusal(
10578 &ctx,
10579 &route_open_frame(397, "busy", unique_project_root("target-cap")),
10580 "busy",
10581 in_flight,
10582 )
10583 .unwrap();
10584 assert_eq!(parse_error(&response)["code"], "target_unavailable");
10585 let event = capture
10586 .events()
10587 .into_iter()
10588 .find(|event| {
10589 event.target == "control"
10590 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
10591 })
10592 .expect("target admission refusal event");
10593 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
10594 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
10595 drop(guards);
10596 }
10597
10598 #[tokio::test(flavor = "current_thread")]
10599 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
10600 let handler = ControlHandler::new(Arc::new(Registry::default()));
10601 let capture = EventCapture::default();
10602 let _subscriber =
10603 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10604 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
10605 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
10606
10607 let _ = handler
10608 .handle_client_control_request(
10609 &ctx,
10610 frame,
10611 ClientControlRequest::SupervisorSetEnabled {
10612 module_id: "broca".to_string(),
10613 enabled: false,
10614 },
10615 )
10616 .await
10617 .unwrap();
10618
10619 let events = capture
10620 .events()
10621 .into_iter()
10622 .filter(|event| {
10623 event.target == "control"
10624 && event.fields.get("message").map(String::as_str)
10625 == Some("supervisor request received")
10626 })
10627 .collect::<Vec<_>>();
10628 assert_eq!(events.len(), 1, "one supervisor request log line");
10629 let event = &events[0];
10630 assert_eq!(event.level, tracing::Level::INFO);
10631 assert_eq!(
10632 event.fields.get("op"),
10633 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
10634 );
10635 assert_eq!(
10636 event.fields.get("module_id"),
10637 Some(&"\"broca\"".to_string())
10638 );
10639 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
10640 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
10641 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
10642 }
10643
10644 #[tokio::test(flavor = "current_thread")]
10645 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
10646 let registry = Arc::new(Registry::default());
10647 let module_connection = ConnectionId::new(303);
10648 registry
10649 .register_with_control_ops(
10650 manifest("aft", PROTOCOL_VERSION),
10651 PROTOCOL_VERSION,
10652 module_connection,
10653 module_baseline_control_ops(),
10654 )
10655 .unwrap();
10656 let handler = ControlHandler::new(registry);
10657 let capture = EventCapture::default();
10658 let _subscriber =
10659 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10660 let (ctx, _rx) = route_ctx(module_connection);
10661 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
10662
10663 let _ = handler
10664 .handle_client_control_request(
10665 &ctx,
10666 frame,
10667 ClientControlRequest::SupervisorSetEnabled {
10668 module_id: "broca".to_string(),
10669 enabled: false,
10670 },
10671 )
10672 .await
10673 .unwrap();
10674
10675 let callers = capture
10676 .events()
10677 .into_iter()
10678 .filter(|event| {
10679 event.fields.get("message").map(String::as_str)
10680 == Some("supervisor request received")
10681 })
10682 .map(|event| event.fields.get("caller").cloned())
10683 .collect::<Vec<_>>();
10684 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
10685 }
10686
10687 #[tokio::test(flavor = "current_thread")]
10688 async fn supervisor_list_does_not_log_request_received() {
10689 let handler = ControlHandler::new(Arc::new(Registry::default()));
10690 let capture = EventCapture::default();
10691 let _subscriber =
10692 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10693 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
10694 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
10695
10696 handler
10697 .handle_client_control_request(&ctx, frame, ClientControlRequest::SupervisorList {})
10698 .await
10699 .unwrap();
10700
10701 assert!(capture.events().into_iter().all(|event| {
10702 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
10703 }));
10704 }
10705
10706 #[tokio::test(flavor = "current_thread")]
10707 async fn supervisor_rescan_preview_does_not_log_request_received() {
10708 let handler = ControlHandler::new(Arc::new(Registry::default()));
10709 let capture = EventCapture::default();
10710 let _subscriber =
10711 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10712 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
10713 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
10714
10715 handler
10716 .handle_client_control_request(
10717 &ctx,
10718 frame,
10719 ClientControlRequest::SupervisorRescan { preview: true },
10720 )
10721 .await
10722 .unwrap();
10723
10724 assert!(capture.events().into_iter().all(|event| {
10725 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
10726 }));
10727 }
10728
10729 #[tokio::test]
10734 async fn route_open_refusal_names_the_check_that_refused() {
10735 let handler = ControlHandler::new(Arc::new(Registry::default()));
10736 let capture = EventCapture::default();
10737 let _subscriber =
10738 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10739 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
10740 let response = handler
10741 .handle_control_frame(
10742 &ctx,
10743 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
10744 )
10745 .await
10746 .unwrap();
10747
10748 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10749 let event = capture
10750 .events()
10751 .into_iter()
10752 .find(|event| {
10753 event.target == "control"
10754 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
10755 })
10756 .expect("route.open refusal event");
10757 assert_eq!(
10758 event.fields.get("reason"),
10759 Some(&"\"not_registered\"".to_string())
10760 );
10761 }
10762
10763 #[tokio::test]
10764 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
10765 let registry = Arc::new(Registry::default());
10766 let supervisor_handle = SupervisorHandle::new();
10767 let supervisor =
10768 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
10769 .with_handle(supervisor_handle.clone())
10770 .with_connection_file_path(std::env::temp_dir().join(format!(
10771 "subc-route-open-refusal-info-{}",
10772 std::process::id()
10773 )));
10774 let module = supervisor
10775 .supervise_configured(
10776 ModuleSpec {
10777 module_id: "warming".to_string(),
10778 program: fake_aft_stub_path(),
10779 args: Vec::new(),
10780 env: Vec::new(),
10781 reserved: false,
10782 reserved_prefixes: Vec::new(),
10783 protocol: ModuleProtocol::Subc,
10784 overlap: Default::default(),
10785 },
10786 true,
10787 )
10788 .unwrap();
10789 assert_eq!(module.state().unwrap(), ModuleState::Running);
10790
10791 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10792 assert!(handler
10793 .counters()
10794 .snapshot()
10795 .get("route_open_refused_by_code")
10796 .is_none());
10797 let capture = EventCapture::default();
10798 let _subscriber =
10799 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10800 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
10801 let response = handler
10802 .handle_control_frame(
10803 &ctx,
10804 route_open_frame(394, "warming", unique_project_root("refusal-info")),
10805 )
10806 .await
10807 .unwrap();
10808 module.stop().await.unwrap();
10809
10810 assert_eq!(parse_error(&response[0])["code"], "module_warming");
10811 let event = capture
10812 .events()
10813 .into_iter()
10814 .find(|event| {
10815 event.target == "control"
10816 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
10817 })
10818 .expect("route.open refusal event");
10819 assert_eq!(
10820 event.fields.get("module_id"),
10821 Some(&"\"warming\"".to_string())
10822 );
10823 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
10824 assert_eq!(
10825 event.fields.get("reason"),
10826 Some(&"\"supervised_not_registered\"".to_string())
10827 );
10828 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
10829 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
10830 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
10831 assert_eq!(
10832 handler.counters().snapshot()["route_open_refused_by_code"],
10833 json!({ "module_warming": 1 })
10834 );
10835 }
10836
10837 const OUTAGE_START: &str = "route.open refusing module: not serving";
10838 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
10839
10840 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
10841 capture
10842 .events()
10843 .into_iter()
10844 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
10845 .collect()
10846 }
10847
10848 fn supervise_stub(
10849 registry: &Arc<Registry>,
10850 module_id: &str,
10851 enabled: bool,
10852 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
10853 let supervisor_handle = SupervisorHandle::new();
10854 let supervisor =
10855 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
10856 .with_handle(supervisor_handle.clone())
10857 .with_connection_file_path(std::env::temp_dir().join(format!(
10858 "subc-route-outage-{module_id}-{}",
10859 std::process::id()
10860 )));
10861 let module = supervisor
10862 .supervise_configured(
10863 ModuleSpec {
10864 module_id: module_id.to_string(),
10865 program: fake_aft_stub_path(),
10866 args: Vec::new(),
10867 env: Vec::new(),
10868 reserved: false,
10869 reserved_prefixes: Vec::new(),
10870 protocol: ModuleProtocol::Subc,
10871 overlap: Default::default(),
10872 },
10873 enabled,
10874 )
10875 .unwrap();
10876 (supervisor_handle, module)
10877 }
10878
10879 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
10880 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
10881 module_id: module_id.to_string(),
10882 drain_timeout_ms: Some(50),
10883 })
10884 .unwrap();
10885 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
10886 }
10887
10888 #[test]
10892 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
10893 let registry = Arc::new(Registry::default());
10894 let forwarding = Arc::new(ForwardingTable::default());
10895 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10896 let second = ControlHandler::with_forwarding(registry, forwarding);
10897 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
10898 }
10899
10900 #[tokio::test(flavor = "current_thread")]
10904 async fn route_open_unknown_module_refusals_add_no_outage_state() {
10905 let handler = ControlHandler::new(Arc::new(Registry::default()));
10906 let capture = EventCapture::default();
10907 let _subscriber =
10908 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10909 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
10910 for corr in 0..8 {
10911 let response = handler
10912 .handle_control_frame(
10913 &ctx,
10914 route_open_frame(
10915 380 + corr,
10916 &format!("nobody-{corr}"),
10917 unique_project_root("outage-unknown"),
10918 ),
10919 )
10920 .await
10921 .unwrap();
10922 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10923 }
10924
10925 assert_eq!(handler.route_outages.tracked_module_count(), 0);
10926 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
10927 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
10928 }
10929
10930 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10935 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
10936 let registry = Arc::new(Registry::default());
10937 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
10938 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10939 let capture = EventCapture::default();
10940 let _subscriber =
10941 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10942 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
10943 handler.route_outages.record_accepted("outage-restart");
10946
10947 let response = handler
10948 .handle_control_frame(
10949 &ctx,
10950 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
10951 )
10952 .await
10953 .unwrap();
10954 assert_eq!(response[0].header.ty, FrameType::Error);
10955 let starts = outage_lines(&capture, OUTAGE_START);
10956 assert_eq!(starts.len(), 1, "{starts:?}");
10957 assert_eq!(starts[0].level, tracing::Level::WARN);
10958 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
10959 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
10960 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
10961 handler.route_outages.record_accepted("outage-restart");
10962 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
10963
10964 let restart = handler
10965 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
10966 .await
10967 .unwrap();
10968 assert_eq!(
10969 restart[0].header.ty,
10970 FrameType::Response,
10971 "{:?}",
10972 parse_error(&restart[0])
10973 );
10974 handler
10975 .handle_control_frame(
10976 &ctx,
10977 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
10978 )
10979 .await
10980 .unwrap();
10981 module.stop().await.unwrap();
10982
10983 let starts = outage_lines(&capture, OUTAGE_START);
10984 assert_eq!(starts.len(), 2, "{starts:?}");
10985 assert_eq!(starts[1].level, tracing::Level::INFO);
10986 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
10987 }
10988
10989 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10993 async fn failed_operator_restart_leaves_no_operator_mark() {
10994 let registry = Arc::new(Registry::default());
10995 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
10996 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10997 let capture = EventCapture::default();
10998 let _subscriber =
10999 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11000 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
11001 handler.route_outages.record_accepted("outage-disabled");
11002
11003 let restart = handler
11004 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
11005 .await
11006 .unwrap();
11007 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
11008 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
11009
11010 handler
11011 .handle_control_frame(
11012 &ctx,
11013 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
11014 )
11015 .await
11016 .unwrap();
11017 let starts = outage_lines(&capture, OUTAGE_START);
11018 assert_eq!(starts.len(), 1, "{starts:?}");
11019 assert_eq!(starts[0].level, tracing::Level::WARN);
11020 }
11021
11022 #[tokio::test(flavor = "current_thread")]
11023 async fn route_open_unknown_module_escapes_target_module_id() {
11024 let handler = ControlHandler::new(Arc::new(Registry::default()));
11025 let capture = EventCapture::default();
11026 let _subscriber =
11027 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11028 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
11029 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
11030 let response = handler
11031 .handle_control_frame(
11032 &ctx,
11033 route_open_frame(
11034 395,
11035 hostile_module_id,
11036 unique_project_root("hostile-target-module-id"),
11037 ),
11038 )
11039 .await
11040 .unwrap();
11041
11042 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11043 let event = capture
11044 .events()
11045 .into_iter()
11046 .find(|event| {
11047 event.target == "control"
11048 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
11049 })
11050 .expect("route.open unknown-module refusal event");
11051 let logged = event.fields.get("module_id").expect("module_id field");
11052 assert!(!logged.bytes().any(|byte| byte < 0x20));
11053 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
11054 }
11055
11056 #[tokio::test(flavor = "current_thread")]
11057 async fn route_open_module_rejection_uses_daemon_counter_key() {
11058 let registry = Arc::new(Registry::default());
11059 let forwarding = Arc::new(ForwardingTable::default());
11060 let handler =
11061 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11062 let module_connection = ConnectionId::new(95);
11063 let (module_ctx, mut module_rx) = route_ctx(module_connection);
11064 hello_via_sink(
11065 &handler,
11066 &module_ctx,
11067 &mut module_rx,
11068 hello_frame("aft", PROTOCOL_VERSION, 395),
11069 )
11070 .await;
11071
11072 let client_connection = ConnectionId::new(96);
11073 let (client_ctx, _client_rx) = route_ctx(client_connection);
11074 let capture = EventCapture::default();
11075 let _subscriber =
11076 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11077 let (route_task, bind) = relay_route_open(
11078 &handler,
11079 client_connection,
11080 &client_ctx.egress,
11081 &mut module_rx,
11082 396,
11083 "aft",
11084 "hostile-module-code",
11085 )
11086 .await;
11087 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
11088 let rejection = Frame::build(
11089 FrameType::Error,
11090 control_flags(),
11091 0,
11092 0,
11093 bind.header.corr,
11094 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
11095 )
11096 .unwrap();
11097 handler
11098 .handle_control_frame(&module_ctx, rejection)
11099 .await
11100 .unwrap();
11101
11102 let response = route_task.await.unwrap();
11103 assert_eq!(parse_error(&response[0])["code"], hostile_code);
11104 let counters = handler.counters().snapshot();
11105 assert_eq!(
11106 counters["route_open_refused_by_code"],
11107 json!({ "module_rejected": 1 })
11108 );
11109 assert!(counters["route_open_refused_by_code"]
11110 .get(hostile_code)
11111 .is_none());
11112
11113 let event = capture
11114 .events()
11115 .into_iter()
11116 .find(|event| {
11117 event.target == "control"
11118 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
11119 })
11120 .expect("route.open module-rejection refusal event");
11121 let logged = event.fields.get("module_code").expect("module_code field");
11122 assert!(!logged.bytes().any(|byte| byte < 0x20));
11123 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
11124 }
11125
11126 #[tokio::test]
11127 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
11128 let registry = Arc::new(Registry::default());
11129 let supervisor_handle = SupervisorHandle::new();
11130 let missing_program = std::env::temp_dir().join(format!(
11131 "subc-route-open-missing-program-{}",
11132 std::process::id()
11133 ));
11134 let supervisor =
11135 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11136 .with_handle(supervisor_handle.clone());
11137 let module = supervisor
11138 .supervise_configured(
11139 ModuleSpec {
11140 module_id: "failed".to_string(),
11141 program: missing_program,
11142 args: Vec::new(),
11143 env: Vec::new(),
11144 reserved: false,
11145 reserved_prefixes: Vec::new(),
11146 protocol: ModuleProtocol::Subc,
11147 overlap: Default::default(),
11148 },
11149 true,
11150 )
11151 .unwrap();
11152 assert_eq!(module.state().unwrap(), ModuleState::Failed);
11153
11154 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11155 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
11156 let response = handler
11157 .handle_control_frame(
11158 &ctx,
11159 route_open_frame(305, "failed", unique_project_root("failed")),
11160 )
11161 .await
11162 .unwrap();
11163
11164 assert_eq!(response[0].header.ty, FrameType::Error);
11165 let error = parse_error(&response[0]);
11166 assert_eq!(error["code"], "target_unavailable");
11167 assert!(error["message"]
11168 .as_str()
11169 .unwrap()
11170 .contains("state=failed, enabled=true, live=false"));
11171 }
11172
11173 #[tokio::test]
11174 async fn route_open_role_mismatch_remains_target_unavailable() {
11175 let registry = Arc::new(Registry::default());
11176 let handler = ControlHandler::new(Arc::clone(®istry));
11177 handler
11178 .handle_control(
11179 ConnectionId::new(41),
11180 non_routable_hello_frame_with_control_ops("health-only", 306, None),
11181 )
11182 .unwrap();
11183
11184 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
11185 let response = handler
11186 .handle_control_frame(
11187 &ctx,
11188 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
11189 )
11190 .await
11191 .unwrap();
11192
11193 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11194 assert!(parse_error(&response[0])["message"]
11195 .as_str()
11196 .unwrap()
11197 .contains("does not provide the requested target"));
11198 }
11199
11200 #[tokio::test]
11201 async fn route_open_inactive_registration_remains_target_unavailable() {
11202 let registry = Arc::new(Registry::default());
11203 let handler = ControlHandler::new(Arc::clone(®istry));
11204 handler
11205 .handle_control(
11206 ConnectionId::new(43),
11207 hello_frame("inactive", PROTOCOL_VERSION, 308),
11208 )
11209 .unwrap();
11210 assert!(registry
11211 .set_module_state_for_test("inactive", ChannelState::Closed)
11212 .unwrap());
11213
11214 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
11215 let response = handler
11216 .handle_control_frame(
11217 &ctx,
11218 route_open_frame(309, "inactive", unique_project_root("inactive")),
11219 )
11220 .await
11221 .unwrap();
11222
11223 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11224 assert!(parse_error(&response[0])["message"]
11225 .as_str()
11226 .unwrap()
11227 .contains("is not active"));
11228 }
11229
11230 #[tokio::test]
11231 async fn late_health_reply_is_recorded_through_the_module_response_path() {
11232 let registry = Arc::new(Registry::default());
11233 let forwarding = Arc::new(ForwardingTable::default());
11234 let supervisor_handle = SupervisorHandle::new();
11235 let supervisor =
11236 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
11237 .with_forwarding(Arc::clone(&forwarding))
11238 .with_handle(supervisor_handle.clone());
11239 let module = supervisor
11240 .supervise_configured(
11241 crate::ModuleSpec {
11242 module_id: "late-health-response".to_string(),
11243 program: PathBuf::from("disabled-module"),
11244 args: Vec::new(),
11245 env: Vec::new(),
11246 reserved: false,
11247 reserved_prefixes: Vec::new(),
11248 protocol: ModuleProtocol::Subc,
11249 overlap: Default::default(),
11250 },
11251 false,
11252 )
11253 .unwrap();
11254 let handler =
11255 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11256 .with_supervisor(supervisor_handle);
11257 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
11258 handler
11259 .handle_control_frame(
11260 &module_ctx,
11261 hello_frame_with_control_ops(
11262 "late-health-response",
11263 PROTOCOL_VERSION,
11264 7,
11265 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11266 ),
11267 )
11268 .await
11269 .unwrap();
11270 let probe_started_at = Instant::now() - Duration::from_millis(80);
11271 let pending = forwarding
11272 .begin_health_probe_rpc_for(
11273 "late-health-response",
11274 MODULE_CONTROL_OP_HEALTH_CHECK,
11275 probe_started_at,
11276 Instant::now() - Duration::from_millis(1),
11277 )
11278 .unwrap();
11279 assert!(forwarding
11280 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
11281 .unwrap());
11282
11283 let responses = handler
11284 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
11285 .await
11286 .unwrap();
11287
11288 assert!(responses.is_empty());
11289 let health = module.status().unwrap().health;
11290 assert_eq!(health.late_answer_count, 1);
11291 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
11292 }
11293
11294 #[tokio::test]
11295 async fn health_probe_timeout_and_module_death_are_typed() {
11296 let registry = Arc::new(Registry::default());
11297 let forwarding = Arc::new(ForwardingTable::default());
11298 let handler =
11299 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11300 .with_health_probe_timeout(Duration::from_millis(50));
11301 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11302 hello_via_sink(
11303 &handler,
11304 &module_ctx,
11305 &mut module_rx,
11306 hello_frame_with_control_ops(
11307 "aft",
11308 PROTOCOL_VERSION,
11309 7,
11310 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11311 ),
11312 )
11313 .await;
11314
11315 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11316 let responses = handler
11317 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
11318 .await
11319 .unwrap();
11320 assert_eq!(responses[0].header.ty, FrameType::Error);
11321 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
11322 let _ = module_rx.try_recv();
11323
11324 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
11325 let health_handler = handler.clone();
11326 let death_task = tokio::spawn(async move {
11327 health_handler
11328 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
11329 .await
11330 .unwrap()
11331 });
11332 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11333 .await
11334 .unwrap()
11335 .unwrap();
11336 handler
11337 .cleanup_connection(module_ctx.connection_id)
11338 .unwrap();
11339 let responses = death_task.await.unwrap();
11340 assert_eq!(responses[0].header.ty, FrameType::Error);
11341 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
11342 }
11343
11344 #[test]
11345 fn hello_requires_exact_protocol_version() {
11346 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
11347 let registry = Arc::new(Registry::default());
11348 let handler = ControlHandler::new(Arc::clone(®istry));
11349 let responses = handler
11350 .handle_control(
11351 ConnectionId::new(connection),
11352 hello_frame("aft", offered, 9),
11353 )
11354 .unwrap();
11355
11356 assert_eq!(responses.len(), 1);
11357 assert_eq!(responses[0].header.ty, FrameType::Error);
11358 let error = parse_error(&responses[0]);
11359 assert_eq!(error["code"], "version_unsupported");
11360 assert!(registry.get_module("aft").unwrap().is_none());
11361 assert_eq!(registry.active_registration_count().unwrap(), 0);
11362 }
11363 }
11364
11365 #[test]
11366 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
11367 let registry = Arc::new(Registry::default());
11368 let forwarding = Arc::new(ForwardingTable::default());
11369 let handler =
11370 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11371 let module_connection = ConnectionId::new(301);
11372 let registration = registry
11373 .register_with_control_ops(
11374 manifest("aft-push", PROTOCOL_VERSION),
11375 PROTOCOL_VERSION,
11376 module_connection,
11377 module_baseline_control_ops(),
11378 )
11379 .unwrap();
11380 let (module_tx, _module_rx) = mpsc::channel(8);
11381 let endpoint = forwarding
11382 .register_module_connection(
11383 module_connection,
11384 "aft-push".to_string(),
11385 PROTOCOL_VERSION,
11386 manifest_concurrency(®istration.manifest),
11387 FrameSink::new(module_tx),
11388 )
11389 .unwrap();
11390
11391 let unknown = Frame::build(
11393 FrameType::Push,
11394 control_flags(),
11395 0,
11396 0,
11397 5,
11398 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
11399 )
11400 .unwrap();
11401 let out = handler.handle_status_update(endpoint, unknown).unwrap();
11402 assert!(
11403 out.is_empty(),
11404 "unknown push op must be ignored, got {out:?}"
11405 );
11406
11407 let malformed = Frame::build(
11409 FrameType::Push,
11410 control_flags(),
11411 0,
11412 0,
11413 6,
11414 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
11415 )
11416 .unwrap();
11417 let out = handler.handle_status_update(endpoint, malformed).unwrap();
11418 assert_eq!(out.len(), 1);
11419 assert_eq!(out[0].header.ty, FrameType::Error);
11420 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
11421 }
11422
11423 #[test]
11424 fn hello_rejected_when_connection_already_owns_client_routes() {
11425 let registry = Arc::new(Registry::default());
11426 let forwarding = Arc::new(ForwardingTable::default());
11427 let handler =
11428 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11429 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
11431 let client_connection = ConnectionId::new(202);
11432
11433 let responses = handler
11436 .handle_control(
11437 client_connection,
11438 hello_frame("aft-second", PROTOCOL_VERSION, 9),
11439 )
11440 .unwrap();
11441 assert_eq!(responses[0].header.ty, FrameType::Error);
11442 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
11443 assert!(registry.get_module("aft-second").unwrap().is_none());
11444 }
11445
11446 #[tokio::test]
11447 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
11448 let registry = Arc::new(Registry::default());
11449 let forwarding = Arc::new(ForwardingTable::default());
11450 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
11451 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
11452 hello_via_sink(
11453 &handler,
11454 &module_ctx,
11455 &mut module_rx,
11456 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
11457 )
11458 .await;
11459 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
11460 let pending = forwarding
11461 .begin_route_bind_relay_for_test(
11462 client_ctx.connection_id,
11463 client_ctx.egress.clone(),
11464 2,
11465 "alpha",
11466 )
11467 .unwrap();
11468 forwarding
11469 .complete_pending_relay(
11470 module_ctx.connection_id,
11471 pending.corr,
11472 RouteBindRelayOutcome::Accepted,
11473 )
11474 .unwrap();
11475 client_rx.try_recv().unwrap();
11476 for module_id in ["beta", "alpha"] {
11477 let replies = handler
11478 .handle_control_frame(
11479 &module_ctx,
11480 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
11481 )
11482 .await
11483 .unwrap();
11484 assert_eq!(replies.len(), 1, "second HELLO must be refused");
11485 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
11486 }
11487 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
11488 assert!(registry.get_module("beta").unwrap().is_none());
11489 assert!(matches!(
11490 forwarding
11491 .lookup_data_route(
11492 client_ctx.connection_id,
11493 pending.client_channel,
11494 pending.client_epoch,
11495 )
11496 .unwrap(),
11497 DataRoute::Client(DataRouteState::Bound(_))
11498 ));
11499 assert!(handler
11500 .hello_launch_nonces
11501 .lock()
11502 .unwrap()
11503 .presented(module_ctx.connection_id, Some("alpha-nonce")));
11504 assert!(module_rx.try_recv().is_err());
11505 }
11506
11507 #[test]
11508 fn reserved_module_hello_requires_matching_launch_nonce() {
11509 let registry = Arc::new(Registry::default());
11510 let supervisor = SupervisorHandle::new();
11511 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
11514 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11515
11516 let no_nonce = handler
11518 .handle_control(
11519 ConnectionId::new(1),
11520 hello_frame("vault", PROTOCOL_VERSION, 1),
11521 )
11522 .unwrap();
11523 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
11524 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
11525 assert!(registry.get_module("vault").unwrap().is_none());
11526
11527 let wrong = handler
11529 .handle_control(
11530 ConnectionId::new(2),
11531 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
11532 )
11533 .unwrap();
11534 assert_eq!(wrong[0].header.ty, FrameType::Error);
11535 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
11536 assert!(registry.get_module("vault").unwrap().is_none());
11537
11538 let ok = handler
11540 .handle_control(
11541 ConnectionId::new(3),
11542 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
11543 )
11544 .unwrap();
11545 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
11546 assert!(registry.get_module("vault").unwrap().is_some());
11547 }
11548
11549 #[test]
11550 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
11551 let registry = Arc::new(Registry::default());
11552 let supervisor = SupervisorHandle::new();
11553 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
11554 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
11555 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11556
11557 let squat = handler
11558 .handle_control(
11559 ConnectionId::new(1),
11560 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
11561 )
11562 .unwrap();
11563 assert_eq!(squat[0].header.ty, FrameType::Error);
11564 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
11565 assert!(parse_error(&squat[0])["message"]
11566 .as_str()
11567 .unwrap()
11568 .contains("fed:"));
11569
11570 let accepted_peer = handler
11571 .handle_control(
11572 ConnectionId::new(2),
11573 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
11574 )
11575 .unwrap();
11576 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
11577
11578 let accepted_short = handler
11579 .handle_control(
11580 ConnectionId::new(3),
11581 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
11582 )
11583 .unwrap();
11584 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
11585
11586 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
11587 let response = handler
11588 .handle_control(
11589 ConnectionId::new(conn),
11590 hello_frame(module_id, PROTOCOL_VERSION, conn),
11591 )
11592 .unwrap();
11593 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
11594 }
11595 }
11596
11597 #[test]
11598 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
11599 let registry = Arc::new(Registry::default());
11600 let supervisor = SupervisorHandle::new();
11601 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
11602 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
11603 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
11604 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11605
11606 let owner_nonce = handler
11607 .handle_control(
11608 ConnectionId::new(1),
11609 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
11610 )
11611 .unwrap();
11612 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
11613 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
11614 assert!(registry.get_module("fed:special").unwrap().is_none());
11615
11616 let exact_nonce = handler
11617 .handle_control(
11618 ConnectionId::new(2),
11619 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
11620 )
11621 .unwrap();
11622 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
11623 assert!(registry.get_module("fed:special").unwrap().is_some());
11624 }
11625
11626 #[test]
11627 fn non_reserved_module_ignores_launch_nonce() {
11628 let registry = Arc::new(Registry::default());
11629 let handler = ControlHandler::new(Arc::clone(®istry));
11632 let no_nonce = handler
11633 .handle_control(
11634 ConnectionId::new(1),
11635 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
11636 )
11637 .unwrap();
11638 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
11639 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
11640
11641 let echoed_nonce = handler
11642 .handle_control(
11643 ConnectionId::new(2),
11644 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
11645 )
11646 .unwrap();
11647 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
11648 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
11649 }
11650
11651 #[test]
11652 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
11653 let handler = ControlHandler::default();
11654 let conn = ConnectionId::new(1);
11655 let malformed = Frame::build(
11656 FrameType::Hello,
11657 control_flags(),
11658 0,
11659 0,
11660 3,
11661 b"{not json".to_vec(),
11662 )
11663 .unwrap();
11664
11665 let error = handler.handle_control(conn, malformed).unwrap();
11666 assert_eq!(error[0].header.ty, FrameType::Error);
11667 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
11668
11669 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
11670 let pong = handler.handle_control(conn, ping).unwrap();
11671 assert_eq!(pong[0].header.ty, FrameType::Pong);
11672 assert_eq!(pong[0].header.corr, 4);
11673 }
11674
11675 #[test]
11676 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
11677 let registry = Arc::new(Registry::default());
11678 let handler = ControlHandler::new(Arc::clone(®istry));
11679
11680 handler
11681 .handle_control(
11682 ConnectionId::new(1),
11683 hello_frame("aft", PROTOCOL_VERSION, 1),
11684 )
11685 .unwrap();
11686 let duplicate = handler
11687 .handle_control(
11688 ConnectionId::new(2),
11689 hello_frame("aft", PROTOCOL_VERSION, 2),
11690 )
11691 .unwrap();
11692
11693 assert_eq!(duplicate[0].header.ty, FrameType::Error);
11694 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11695 let registration = registry.get_module("aft").unwrap().unwrap();
11696 assert_eq!(registration.connection_id, ConnectionId::new(1));
11697 }
11698
11699 #[test]
11700 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
11701 let registry = Arc::new(Registry::default());
11702 let forwarding = Arc::new(ForwardingTable::default());
11703 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
11704 let handler =
11705 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11706 .with_process_liveness(process_liveness);
11707 let (ctx, route_channel, route_epoch) =
11708 bind_liveness_route(®istry, &forwarding, "aft-dead");
11709 let responses = handler
11710 .handle_route_poll(
11711 &ctx,
11712 route_poll_frame(41, PollKind::Liveness, route_channel),
11713 route_channel,
11714 route_epoch,
11715 PollKind::Liveness,
11716 )
11717 .unwrap();
11718
11719 assert_eq!(responses.len(), 1);
11720 assert_eq!(responses[0].header.ty, FrameType::Response);
11721 assert_route_poll_liveness(&responses[0], false);
11722 }
11723
11724 #[test]
11725 fn liveness_poll_without_process_source_uses_bound_route() {
11726 let registry = Arc::new(Registry::default());
11727 let forwarding = Arc::new(ForwardingTable::default());
11728 let handler =
11729 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11730 let (ctx, route_channel, route_epoch) =
11731 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
11732 let responses = handler
11733 .handle_route_poll(
11734 &ctx,
11735 route_poll_frame(42, PollKind::Liveness, route_channel),
11736 route_channel,
11737 route_epoch,
11738 PollKind::Liveness,
11739 )
11740 .unwrap();
11741
11742 assert_route_poll_liveness(&responses[0], true);
11743 }
11744
11745 #[test]
11746 fn liveness_poll_untracked_process_source_uses_bound_route() {
11747 let registry = Arc::new(Registry::default());
11748 let forwarding = Arc::new(ForwardingTable::default());
11749 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
11750 let handler =
11751 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11752 .with_process_liveness(process_liveness);
11753 let (ctx, route_channel, route_epoch) =
11754 bind_liveness_route(®istry, &forwarding, "aft-untracked");
11755 let responses = handler
11756 .handle_route_poll(
11757 &ctx,
11758 route_poll_frame(43, PollKind::Liveness, route_channel),
11759 route_channel,
11760 route_epoch,
11761 PollKind::Liveness,
11762 )
11763 .unwrap();
11764
11765 assert_route_poll_liveness(&responses[0], true);
11766 }
11767
11768 #[tokio::test]
11769 async fn unknown_op_returns_unknown_control_op() {
11770 let handler = ControlHandler::default();
11771 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
11772 let request = Frame::build(
11773 FrameType::Request,
11774 control_flags(),
11775 0,
11776 0,
11777 55,
11778 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
11779 )
11780 .unwrap();
11781
11782 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11783
11784 assert_eq!(response.len(), 1);
11785 assert_eq!(response[0].header.ty, FrameType::Error);
11786 assert_eq!(response[0].header.corr, 55);
11787 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
11788 }
11789
11790 #[tokio::test]
11791 async fn supervisor_provenance_rejects_unknown_exact_module() {
11792 let handler = ControlHandler::default();
11793 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
11794 let request = Frame::build(
11795 FrameType::Request,
11796 control_flags(),
11797 0,
11798 0,
11799 57,
11800 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
11801 )
11802 .unwrap();
11803
11804 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11805
11806 assert_eq!(response.len(), 1);
11807 assert_eq!(response[0].header.ty, FrameType::Error);
11808 assert_eq!(response[0].header.corr, 57);
11809 let error = parse_error(&response[0]);
11810 assert_eq!(error["code"], "unknown_module");
11811 assert_eq!(error["message"], "module_id 'missing' is not supervised");
11812 }
11813
11814 #[test]
11815 fn provenance_probe_override_keeps_handler_tests_deterministic() {
11816 let expected = subc_control::RunningImageAgreement::Unavailable {
11817 reason: subc_control::RunningImageUnavailableReason::HashFailed,
11818 };
11819 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
11820 assert_eq!(handler.provenance_probe_override, Some(expected));
11821 }
11822
11823 #[test]
11824 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
11825 let verdict = reload_verdict(
11826 std::path::Path::new("/bin/new"),
11827 Some(std::path::Path::new("/bin/old")),
11828 subc_control::RunningImageAgreement::Unavailable {
11829 reason: subc_control::RunningImageUnavailableReason::HashFailed,
11830 },
11831 );
11832 assert!(matches!(
11833 verdict.path,
11834 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
11835 if configured == std::path::Path::new("/bin/new")
11836 && spawned_from == std::path::Path::new("/bin/old")
11837 ));
11838 }
11839
11840 #[cfg(windows)]
11841 #[test]
11842 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
11843 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
11844 let configured = dir.join("module.exe");
11845 std::fs::write(&configured, b"executable path fixture").unwrap();
11846 let spawned = std::fs::canonicalize(&configured).unwrap();
11847 assert_ne!(
11848 configured, spawned,
11849 "fixture must exercise different path spellings"
11850 );
11851 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
11852 let lower_case = configured.to_string_lossy().to_lowercase();
11853 for spelling in [
11854 &configured,
11855 Path::new(&forward_slashes),
11856 Path::new(&lower_case),
11857 ] {
11858 let verdict = reload_verdict(
11859 spelling,
11860 Some(&spawned),
11861 subc_control::RunningImageAgreement::Unavailable {
11862 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
11863 },
11864 );
11865 assert_eq!(
11866 verdict.path,
11867 ReloadPathAgreement::Match,
11868 "configured {spelling:?}, spawned {spawned:?}"
11869 );
11870 }
11871 }
11872
11873 #[cfg(windows)]
11874 #[test]
11875 fn windows_reload_path_keeps_different_existing_program_pending() {
11876 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
11877 let old = dir.join("old.exe");
11878 let new = dir.join("new.exe");
11879 std::fs::write(&old, b"same executable content").unwrap();
11880 std::fs::copy(&old, &new).unwrap();
11881 let spawned = std::fs::canonicalize(&old).unwrap();
11882 let image = subc_control::RunningImageAgreement::Unavailable {
11883 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
11884 };
11885 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
11886 assert!(
11887 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
11888 if configured == new && spawned_from == spawned)
11889 );
11890 let missing = dir.join("missing.exe");
11891 assert!(matches!(
11892 reload_verdict(&missing, Some(&spawned), image).path,
11893 ReloadPathAgreement::Mismatch { .. }
11894 ));
11895 }
11896
11897 #[test]
11898 fn reload_verdict_detects_replaced_image_at_same_path() {
11899 let image = subc_control::RunningImageAgreement::Mismatch {
11900 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
11901 digest: "old".into(),
11902 },
11903 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
11904 digest: "new".into(),
11905 },
11906 };
11907 let verdict = reload_verdict(
11908 std::path::Path::new("/bin/same"),
11909 Some(std::path::Path::new("/bin/same")),
11910 image.clone(),
11911 );
11912 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11913 assert_eq!(verdict.image, image);
11914 }
11915
11916 #[test]
11917 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
11918 let image = subc_control::RunningImageAgreement::Unavailable {
11919 reason: subc_control::RunningImageUnavailableReason::NotRunning,
11920 };
11921 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
11922 assert_eq!(
11923 verdict.path,
11924 subc_control::ReloadPathAgreement::Unavailable {
11925 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
11926 }
11927 );
11928 assert_eq!(verdict.image, image);
11929
11930 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
11931 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
11932 };
11933 let verdict = reload_verdict(
11934 std::path::Path::new("/bin/same"),
11935 Some(std::path::Path::new("/bin/same")),
11936 unconfirmed.clone(),
11937 );
11938 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11939 assert_eq!(verdict.image, unconfirmed);
11940 }
11941
11942 #[test]
11943 fn reload_verdict_preserves_each_image_unavailability_reason() {
11944 use subc_control::RunningImageUnavailableReason as Reason;
11945
11946 for reason in [
11947 Reason::NotRunning,
11948 Reason::UnsupportedPlatform,
11949 Reason::RunningExecutableUnreadable,
11950 Reason::SpawnedPathUnreadable,
11951 Reason::HashFailed,
11952 Reason::ProcessIdentityUnconfirmed,
11953 Reason::Unknown("future_probe_reason".to_string()),
11954 ] {
11955 let image = subc_control::RunningImageAgreement::Unavailable {
11956 reason: reason.clone(),
11957 };
11958 let verdict = reload_verdict(
11959 std::path::Path::new("/bin/same"),
11960 Some(std::path::Path::new("/bin/same")),
11961 image.clone(),
11962 );
11963 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11964 assert_eq!(verdict.image, image, "{reason:?}");
11965 }
11966 }
11967
11968 #[tokio::test]
11969 async fn malformed_control_bodies_return_invalid_control_body() {
11970 let handler = ControlHandler::default();
11971 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
11972
11973 for (corr, body) in [
11974 (56, br#"{"route_channel":1}"#.as_slice()),
11975 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
11976 (
11977 58,
11978 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
11979 ),
11980 ] {
11981 let request = Frame::build(
11982 FrameType::Request,
11983 control_flags(),
11984 0,
11985 0,
11986 corr,
11987 body.to_vec(),
11988 )
11989 .unwrap();
11990 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11991
11992 assert_eq!(response.len(), 1);
11993 assert_eq!(response[0].header.ty, FrameType::Error);
11994 assert_eq!(response[0].header.corr, corr);
11995 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
11996 }
11997 }
11998
11999 #[tokio::test]
12000 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
12001 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12002 let registry = Arc::new(Registry::default());
12003 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
12004 let router = Router::with_control_handler(Arc::clone(&control));
12005 let connection = router.begin_connection();
12006 let (ctx, mut rx) = route_ctx(connection.id());
12007
12008 router
12009 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
12010 .await
12011 .unwrap();
12012 let response = rx.recv().await.unwrap();
12013 let ack = parse_ack(&response);
12014 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
12015 let channel = 1;
12016
12017 let goodbye =
12018 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
12019 router.route_for_connection(&ctx, goodbye).await.unwrap();
12020 assert!(rx.try_recv().is_err());
12021 assert!(registry.get_module("aft").unwrap().is_none());
12022
12023 router
12024 .route_for_connection(&ctx, channel_request(channel, 13))
12025 .await
12026 .unwrap();
12027 let error_frame = rx.recv().await.unwrap();
12028 assert_eq!(error_frame.header.ty, FrameType::Error);
12029 assert_eq!(error_frame.header.channel, channel);
12030 let captured = crate::router::test_log::captured_logs(&logs);
12031 assert_eq!(
12032 captured
12033 .lines()
12034 .filter(|line| {
12035 line.contains("module_id=aft")
12036 && line.contains("reason=explicit_goodbye")
12037 && line.contains("module registration ended")
12038 })
12039 .count(),
12040 1,
12041 "unexpected GOODBYE registry log: {captured}"
12042 );
12043 }
12044
12045 #[tokio::test]
12046 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
12047 let registry = Arc::new(Registry::default());
12048 let handler = ControlHandler::new(registry).with_capability_config(
12049 [("prov".to_string(), true), ("cons".to_string(), true)],
12050 BTreeMap::new(),
12051 );
12052 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
12053 register_capability_manifest(
12054 &handler,
12055 &provider_ctx,
12056 &mut provider_rx,
12057 capability_manifest("prov", &["thing/v1"], &[]),
12058 1,
12059 )
12060 .await;
12061 let mut consumer = capability_manifest("cons", &[], &[]);
12062 consumer.capabilities.as_mut().unwrap().requires.push(
12063 subc_protocol::manifest::CapabilityRequirement {
12064 capability: "thing/v1".to_string(),
12065 need: subc_protocol::manifest::CapabilityNeed::Required,
12066 },
12067 );
12068 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
12069 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
12070 assert_eq!(
12071 handler.capability_evaluator.verdict("cons", "thing/v1"),
12072 Some(CapabilityVerdict::Provided)
12073 );
12074 let (mut client_rx, _) = open_route_for_capability_test(
12075 &handler,
12076 &provider_ctx,
12077 &mut provider_rx,
12078 703,
12079 3,
12080 "prov",
12081 None,
12082 )
12083 .await;
12084 let goodbye =
12085 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
12086 handler
12087 .handle_control_frame(&provider_ctx, goodbye)
12088 .await
12089 .unwrap();
12090 assert_eq!(
12091 handler.capability_evaluator.verdict("cons", "thing/v1"),
12092 Some(CapabilityVerdict::NeverProvided)
12093 );
12094 let closed = client_rx
12095 .try_recv()
12096 .expect("GOODBYE pushes route.closed before route GOODBYE");
12097 assert!(
12098 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
12099 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
12100 );
12101 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
12102 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
12103 }
12104
12105 struct RouteClosedFixture {
12106 handler: ControlHandler,
12107 #[cfg(unix)]
12108 supervisor: Supervisor,
12109 module_ctx: RouteCtx,
12110 #[cfg(unix)]
12111 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12112 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12113 route: Arc<crate::forwarding::RouteBinding>,
12114 }
12115
12116 async fn route_closed_fixture() -> RouteClosedFixture {
12117 let registry = Arc::new(Registry::default());
12118 let handler = ControlHandler::new(Arc::clone(®istry));
12119 #[cfg(unix)]
12120 let supervisor = Supervisor::new(registry, RestartPolicy::default())
12121 .with_forwarding(Arc::clone(&handler.forwarding));
12122 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
12123 register_capability_manifest(
12124 &handler,
12125 &module_ctx,
12126 &mut module_rx,
12127 capability_manifest("prov", &[], &[]),
12128 1,
12129 )
12130 .await;
12131 let (client_rx, bind) = open_route_for_capability_test(
12132 &handler,
12133 &module_ctx,
12134 &mut module_rx,
12135 702,
12136 2,
12137 "prov",
12138 None,
12139 )
12140 .await;
12141 let ModuleControlRequest::RouteBind {
12142 route_channel: channel,
12143 epoch: route_epoch,
12144 ..
12145 } = bind
12146 else {
12147 panic!("expected route.bind");
12148 };
12149 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
12150 handler
12151 .forwarding
12152 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
12153 .unwrap()
12154 else {
12155 panic!("the acknowledged route must be bound");
12156 };
12157 RouteClosedFixture {
12158 handler,
12159 #[cfg(unix)]
12160 supervisor,
12161 module_ctx,
12162 #[cfg(unix)]
12163 module_rx,
12164 client_rx,
12165 route,
12166 }
12167 }
12168
12169 impl RouteClosedFixture {
12170 #[cfg(unix)]
12171 async fn cut(&mut self) {
12172 let notices = async {
12173 let mut draining = self.module_rx.recv().await.unwrap();
12174 assert!(matches!(
12175 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
12176 &draining.body
12177 )
12178 .unwrap(),
12179 subc_protocol::session::ModuleControlCommand::Draining {
12180 reason: RouteCloseReason::Restart,
12181 ..
12182 }
12183 ));
12184 draining.flushed.take().unwrap().send(()).unwrap();
12185 let mut closing = self.client_rx.recv().await.unwrap();
12186 assert!(matches!(
12187 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
12188 ClientControlPush::RouteClosing {
12189 reason: RouteCloseReason::Restart,
12190 ..
12191 }
12192 ));
12193 closing.flushed.take().unwrap().send(()).unwrap();
12194 };
12195 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
12196 result.unwrap();
12197 }
12198
12199 fn disconnect(&mut self) -> ClientControlPush {
12200 self.handler
12201 .cleanup_connection(self.module_ctx.connection_id)
12202 .unwrap();
12203 let closed = self.client_rx.try_recv().unwrap();
12204 assert_eq!(closed.header.ty, FrameType::Push);
12205 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
12206 assert!(matches!(
12207 &push,
12208 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
12209 if channels == &[self.route.client_channel]
12210 ));
12211 assert_eq!(
12212 self.client_rx.try_recv().unwrap().header.ty,
12213 FrameType::Goodbye
12214 );
12215 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
12216 push
12217 }
12218 }
12219
12220 #[cfg(unix)]
12221 #[tokio::test(start_paused = true)]
12222 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
12223 let mut fixture = route_closed_fixture().await;
12224 fixture.cut().await;
12225 assert!(matches!(
12226 fixture.disconnect(),
12227 ClientControlPush::RouteClosed {
12228 reason: RouteCloseReason::Restart,
12229 drained: true,
12230 ..
12231 }
12232 ));
12233 }
12234
12235 #[cfg(unix)]
12236 #[tokio::test(start_paused = true)]
12237 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
12238 let mut fixture = route_closed_fixture().await;
12239 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
12240 let started = tokio::time::Instant::now();
12241 fixture.cut().await;
12242 assert!(started.elapsed() >= Duration::from_secs(2));
12243 assert_eq!(fixture.route.flow.in_flight(), 1);
12244 assert!(matches!(
12245 fixture.disconnect(),
12246 ClientControlPush::RouteClosed {
12247 reason: RouteCloseReason::Restart,
12248 drained: false,
12249 ..
12250 }
12251 ));
12252 }
12253
12254 #[tokio::test]
12255 async fn module_connection_crash_reports_crash_and_not_drained() {
12256 let mut fixture = route_closed_fixture().await;
12257 assert!(matches!(
12261 fixture.disconnect(),
12262 ClientControlPush::RouteClosed {
12263 reason: RouteCloseReason::Crash,
12264 drained: false,
12265 ..
12266 }
12267 ));
12268 }
12269
12270 #[tokio::test]
12271 async fn dropping_router_connection_releases_registration() {
12272 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12273 let registry = Arc::new(Registry::default());
12274 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
12275 let router = Router::with_control_handler(Arc::clone(&control));
12276 let connection = router.begin_connection();
12277 let connection_id = connection.id();
12278 let (ctx, mut rx) = route_ctx(connection_id);
12279
12280 router
12281 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
12282 .await
12283 .unwrap();
12284 let response = rx.recv().await.unwrap();
12285 let ack = parse_ack(&response);
12286 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
12287 assert!(registry.get_module("aft").unwrap().is_some());
12288
12289 drop(connection);
12290
12291 assert!(registry.get_module("aft").unwrap().is_none());
12292 assert_eq!(registry.active_registration_count().unwrap(), 0);
12293
12294 control
12295 .cleanup_connection(ConnectionId::new(u64::MAX))
12296 .unwrap();
12297 let captured = crate::router::test_log::captured_logs(&logs);
12298 println!("captured registry lifecycle logs:\n{captured}");
12299 let events: Vec<_> = captured
12300 .lines()
12301 .filter(|line| {
12302 line.contains("module registered module_id=aft ")
12303 || line.contains("module registration ended")
12304 })
12305 .collect();
12306 assert_eq!(
12307 events.len(),
12308 2,
12309 "unexpected registry lifecycle logs: {captured}"
12310 );
12311 assert!(events[0].contains("module registered module_id=aft "));
12312 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
12313 assert!(events[1].contains(&format!(
12314 "module_id=aft connection_id={}",
12315 connection_id.get()
12316 )));
12317 assert!(events[1].contains("reason=connection_closed"));
12318 assert!(events[1].contains("module registration ended"));
12319 assert_eq!(
12320 captured
12321 .lines()
12322 .filter(|line| line.contains("module registered module_id=aft "))
12323 .count(),
12324 1,
12325 "legacy registration admission line must appear once: {captured}"
12326 );
12327 }
12328
12329 #[tokio::test]
12330 async fn hello_registration_keeps_legacy_module_registered_line_once() {
12331 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12332 let registry = Arc::new(Registry::default());
12333 let control = ControlHandler::new(Arc::clone(®istry));
12334 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
12335 hello_via_sink(
12336 &control,
12337 &ctx,
12338 &mut rx,
12339 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
12340 )
12341 .await;
12342
12343 let captured = crate::router::test_log::captured_logs(&logs);
12344 let admissions: Vec<_> = captured
12345 .lines()
12346 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
12347 .collect();
12348 assert_eq!(
12349 admissions.len(),
12350 1,
12351 "legacy admission line must remain exactly once: {captured}"
12352 );
12353 assert!(admissions[0].contains("routable_provider=true"));
12354 assert!(admissions[0].contains("connection_id=777"));
12355 }
12356
12357 fn capability_manifest(
12358 module_id: &str,
12359 provides: &[&str],
12360 must_never_reach: &[&str],
12361 ) -> ModuleManifest {
12362 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
12363 manifest.capabilities = Some(CapabilityDeclarations {
12364 provides: provides
12365 .iter()
12366 .map(|capability| (*capability).to_string())
12367 .collect(),
12368 requires: Vec::new(),
12369 must_never_reach: must_never_reach
12370 .iter()
12371 .map(|capability| (*capability).to_string())
12372 .collect(),
12373 });
12374 manifest
12375 }
12376
12377 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
12378 Frame::build(
12379 FrameType::Hello,
12380 control_flags(),
12381 0,
12382 0,
12383 corr,
12384 serde_json::to_vec(&ModuleHelloBody {
12385 protocol_ver: manifest.protocol_ver,
12386 manifest,
12387 control_ops: None,
12388 launch_nonce: None,
12389 })
12390 .expect("capability test HELLO serializes"),
12391 )
12392 .expect("capability test HELLO frame builds")
12393 }
12394
12395 fn catalog_update_with_capabilities_frame(
12396 corr: u64,
12397 capabilities: CapabilityDeclarations,
12398 ) -> Frame {
12399 Frame::build(
12400 FrameType::Request,
12401 control_flags(),
12402 0,
12403 0,
12404 corr,
12405 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
12406 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
12407 capabilities: Some(capabilities),
12408 ready: None,
12409 })
12410 .expect("capability catalog.update serializes"),
12411 )
12412 .expect("capability catalog.update frame builds")
12413 }
12414
12415 async fn register_capability_manifest(
12416 handler: &ControlHandler,
12417 ctx: &RouteCtx,
12418 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
12419 manifest: ModuleManifest,
12420 corr: u64,
12421 ) {
12422 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
12423 }
12424
12425 async fn open_route_for_capability_test(
12426 handler: &ControlHandler,
12427 target_ctx: &RouteCtx,
12428 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
12429 client_connection_id: u64,
12430 corr: u64,
12431 target_module_id: &str,
12432 consumer_identity: Option<ConsumerIdentity>,
12433 ) -> (
12434 mpsc::Receiver<crate::router::OutboundFrame>,
12435 ModuleControlRequest,
12436 ) {
12437 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
12438 let route_handler = handler.clone();
12439 let target_module_id = target_module_id.to_string();
12440 let route_task = tokio::spawn(async move {
12441 route_handler
12442 .handle_control_frame(
12443 &client_ctx,
12444 route_open_frame_with_admission_facts(
12445 corr,
12446 &target_module_id,
12447 unique_project_root("admission-facts"),
12448 consumer_identity,
12449 None,
12450 ),
12451 )
12452 .await
12453 .expect("capability test route.open succeeds")
12454 });
12455 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
12456 .await
12457 .expect("capability test route.open must reach route.bind")
12458 .expect("target control receiver stays open");
12459 let bind_request: ModuleControlRequest =
12460 serde_json::from_slice(&bind.body).expect("route.bind decodes");
12461 handler
12462 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
12463 .await
12464 .expect("capability test route.bind ACK succeeds");
12465 assert!(route_task.await.expect("route.open task joins").is_empty());
12466 let opened = client_rx
12467 .recv()
12468 .await
12469 .expect("successful route.open publishes a response");
12470 assert!(matches!(
12471 serde_json::from_slice::<ClientControlResponse>(&opened.body),
12472 Ok(ClientControlResponse::RouteOpen { .. })
12473 ));
12474 (client_rx, bind_request)
12475 }
12476
12477 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
12478 assert_eq!(frame.header.ty, FrameType::Push);
12479 assert_eq!(frame.header.channel, 0);
12480 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
12481 .expect("route.closed control push decodes");
12482 let ClientControlPush::RouteClosed { channels, .. } = &push else {
12483 panic!("expected route.closed");
12484 };
12485 assert_eq!(channels.len(), 1, "exactly one violating route closed");
12486 let channels = channels.clone();
12487 assert_eq!(
12488 push,
12489 ClientControlPush::RouteClosed {
12490 module_id: target_module_id.to_string(),
12491 channels,
12492 reason: RouteCloseReason::CapabilityDenied,
12493 drained: false,
12494 abandoned: 0,
12495 excluded_subscriptions: 0,
12496 terminal: Some(false),
12497 }
12498 );
12499 }
12500
12501 #[tokio::test]
12502 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
12503 let registry = Arc::new(Registry::default());
12504 let forwarding = Arc::new(ForwardingTable::default());
12505 let supervisor = SupervisorHandle::new();
12506 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12507 let handler =
12508 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12509 .with_supervisor(supervisor);
12510 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
12511 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
12512 register_capability_manifest(
12513 &handler,
12514 &target_ctx,
12515 &mut target_rx,
12516 capability_manifest("target", &["credentials-provider/v1"], &[]),
12517 1,
12518 )
12519 .await;
12520 register_capability_manifest(
12521 &handler,
12522 &opener_ctx,
12523 &mut opener_rx,
12524 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12525 2,
12526 )
12527 .await;
12528
12529 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
12530 let replies = handler
12531 .handle_control_frame(
12532 &client_ctx,
12533 route_open_frame_with_admission_facts(
12534 3,
12535 "target",
12536 unique_project_root("admission-facts"),
12537 Some(ConsumerIdentity {
12538 module_id: "opener".to_string(),
12539 launch_nonce: "opener-nonce".to_string(),
12540 }),
12541 None,
12542 ),
12543 )
12544 .await
12545 .expect("denied route.open returns a typed frame");
12546 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
12547 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12548 assert!(
12549 target_rx.try_recv().is_err(),
12550 "forbidden route.open must not relay route.bind"
12551 );
12552 }
12553
12554 #[tokio::test]
12555 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
12556 let registry = Arc::new(Registry::default());
12557 let forwarding = Arc::new(ForwardingTable::default());
12558 let supervisor = SupervisorHandle::new();
12559 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12560 let handler =
12561 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12562 .with_supervisor(supervisor);
12563 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
12564 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
12565 register_capability_manifest(
12566 &handler,
12567 &target_ctx,
12568 &mut target_rx,
12569 capability_manifest("target", &["credentials-provider/v1"], &[]),
12570 1,
12571 )
12572 .await;
12573 register_capability_manifest(
12574 &handler,
12575 &old_opener_ctx,
12576 &mut old_opener_rx,
12577 capability_manifest("opener", &[], &[]),
12578 2,
12579 )
12580 .await;
12581 let (mut client_rx, _) = open_route_for_capability_test(
12582 &handler,
12583 &target_ctx,
12584 &mut target_rx,
12585 712,
12586 3,
12587 "target",
12588 Some(ConsumerIdentity {
12589 module_id: "opener".to_string(),
12590 launch_nonce: "opener-nonce".to_string(),
12591 }),
12592 )
12593 .await;
12594 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12595
12596 handler
12597 .cleanup_connection(old_opener_ctx.connection_id)
12598 .expect("old opener registration cleans up");
12599 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
12600 register_capability_manifest(
12601 &handler,
12602 &new_opener_ctx,
12603 &mut new_opener_rx,
12604 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12605 4,
12606 )
12607 .await;
12608
12609 assert_capability_denied_push(
12610 client_rx
12611 .try_recv()
12612 .expect("HELLO deny addition must emit route.closed")
12613 .frame,
12614 "target",
12615 );
12616 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12617 assert!(matches!(
12618 target_rx.try_recv(),
12619 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
12620 ));
12621 }
12622
12623 #[tokio::test]
12624 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
12625 let registry = Arc::new(Registry::default());
12626 let forwarding = Arc::new(ForwardingTable::default());
12627 let supervisor = SupervisorHandle::new();
12628 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12629 let handler =
12630 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12631 .with_supervisor(supervisor);
12632 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
12633 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
12634 register_capability_manifest(
12635 &handler,
12636 &target_ctx,
12637 &mut target_rx,
12638 capability_manifest("target", &[], &[]),
12639 1,
12640 )
12641 .await;
12642 register_capability_manifest(
12643 &handler,
12644 &opener_ctx,
12645 &mut opener_rx,
12646 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12647 2,
12648 )
12649 .await;
12650 let (mut client_rx, _) = open_route_for_capability_test(
12651 &handler,
12652 &target_ctx,
12653 &mut target_rx,
12654 722,
12655 3,
12656 "target",
12657 Some(ConsumerIdentity {
12658 module_id: "opener".to_string(),
12659 launch_nonce: "opener-nonce".to_string(),
12660 }),
12661 )
12662 .await;
12663 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12664
12665 let replies = handler
12666 .handle_control_frame(
12667 &target_ctx,
12668 catalog_update_with_capabilities_frame(
12669 4,
12670 CapabilityDeclarations {
12671 provides: vec!["credentials-provider/v1".to_string()],
12672 requires: Vec::new(),
12673 must_never_reach: Vec::new(),
12674 },
12675 ),
12676 )
12677 .await
12678 .expect("claim catalog.update succeeds");
12679 assert!(matches!(
12680 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
12681 Ok(ModuleControlResponseToModule::CatalogUpdate {})
12682 ));
12683 assert_capability_denied_push(
12684 client_rx
12685 .try_recv()
12686 .expect("claim addition must emit route.closed")
12687 .frame,
12688 "target",
12689 );
12690 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12691 assert!(matches!(
12692 target_rx.try_recv(),
12693 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
12694 ));
12695 }
12696
12697 #[tokio::test]
12698 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
12699 let registry = Arc::new(Registry::default());
12700 let forwarding = Arc::new(ForwardingTable::default());
12701 let supervisor = SupervisorHandle::new();
12702 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12703 let handler =
12704 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12705 .with_supervisor(supervisor);
12706 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
12707 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
12708 register_capability_manifest(
12709 &handler,
12710 &target_ctx,
12711 &mut target_rx,
12712 capability_manifest("target", &["credentials-provider/v1"], &[]),
12713 1,
12714 )
12715 .await;
12716 register_capability_manifest(
12717 &handler,
12718 &opener_ctx,
12719 &mut opener_rx,
12720 capability_manifest("opener", &[], &[]),
12721 2,
12722 )
12723 .await;
12724 let (mut client_rx, _) = open_route_for_capability_test(
12725 &handler,
12726 &target_ctx,
12727 &mut target_rx,
12728 732,
12729 3,
12730 "target",
12731 Some(ConsumerIdentity {
12732 module_id: "opener".to_string(),
12733 launch_nonce: "opener-nonce".to_string(),
12734 }),
12735 )
12736 .await;
12737 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12738
12739 handler
12740 .handle_control_frame(
12741 &target_ctx,
12742 catalog_update_with_capabilities_frame(
12743 4,
12744 CapabilityDeclarations {
12745 provides: Vec::new(),
12746 requires: Vec::new(),
12747 must_never_reach: Vec::new(),
12748 },
12749 ),
12750 )
12751 .await
12752 .expect("claim removal catalog.update succeeds");
12753 assert_eq!(
12754 forwarding.active_binding_count().unwrap(),
12755 1,
12756 "removing an attested target claim must leave the route census unchanged"
12757 );
12758 assert!(
12759 client_rx.try_recv().is_err(),
12760 "claim removal must not emit route.closed capability_denied"
12761 );
12762 assert!(
12763 target_rx.try_recv().is_err(),
12764 "claim removal must not send the target a route GOODBYE"
12765 );
12766 }
12767
12768 #[tokio::test]
12771 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
12772 let registry = Arc::new(Registry::default());
12773 let forwarding = Arc::new(ForwardingTable::default());
12774 let supervisor = SupervisorHandle::new();
12775 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12776 let handler =
12777 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12778 .with_supervisor(supervisor);
12779 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
12780 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
12781 register_capability_manifest(
12782 &handler,
12783 &target_ctx,
12784 &mut target_rx,
12785 capability_manifest("target", &["credentials-provider/v1"], &[]),
12786 1,
12787 )
12788 .await;
12789 register_capability_manifest(
12790 &handler,
12791 &opener_ctx,
12792 &mut opener_rx,
12793 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12794 2,
12795 )
12796 .await;
12797
12798 let (_client_rx, bind) = open_route_for_capability_test(
12799 &handler,
12800 &target_ctx,
12801 &mut target_rx,
12802 742,
12803 3,
12804 "target",
12805 None,
12806 )
12807 .await;
12808 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
12809 panic!("direct scope-honesty route must bind");
12810 };
12811 assert_eq!(principal, Some(Principal::Direct));
12812 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12813 }
12814
12815 #[tokio::test]
12818 async fn must_never_reach_self_route_is_capability_forbidden() {
12819 let registry = Arc::new(Registry::default());
12820 let forwarding = Arc::new(ForwardingTable::default());
12821 let supervisor = SupervisorHandle::new();
12822 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
12823 let handler =
12824 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12825 .with_supervisor(supervisor);
12826 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
12827 register_capability_manifest(
12828 &handler,
12829 &self_ctx,
12830 &mut self_rx,
12831 capability_manifest(
12832 "self-provider",
12833 &["credentials-provider/v1"],
12834 &["credentials-provider/v1"],
12835 ),
12836 1,
12837 )
12838 .await;
12839
12840 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
12841 let replies = handler
12842 .handle_control_frame(
12843 &client_ctx,
12844 route_open_frame_with_admission_facts(
12845 2,
12846 "self-provider",
12847 unique_project_root("admission-facts"),
12848 Some(ConsumerIdentity {
12849 module_id: "self-provider".to_string(),
12850 launch_nonce: "self-nonce".to_string(),
12851 }),
12852 None,
12853 ),
12854 )
12855 .await
12856 .expect("self-route refusal returns a typed frame");
12857 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
12858 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12859 assert!(
12860 self_rx.try_recv().is_err(),
12861 "self denial must not relay route.bind"
12862 );
12863 }
12864
12865 #[test]
12866 fn unsupported_channel_zero_frame_returns_error() {
12867 let handler = ControlHandler::default();
12868 let request = Frame::build(
12869 FrameType::Request,
12870 control_flags(),
12871 0,
12872 0,
12873 21,
12874 b"opaque".to_vec(),
12875 )
12876 .unwrap();
12877
12878 let response = handler
12879 .handle_control(ConnectionId::new(1), request)
12880 .unwrap();
12881
12882 assert_eq!(response[0].header.ty, FrameType::Error);
12883 assert_eq!(
12884 parse_error(&response[0])["code"],
12885 "unsupported_control_frame"
12886 );
12887 }
12888
12889 mod swap {
12894 use super::*;
12895
12896 const INCUMBENT: ConnectionId = ConnectionId::new(30);
12897 const CANDIDATE: ConnectionId = ConnectionId::new(40);
12898
12899 struct Swap {
12900 registry: Arc<Registry>,
12901 forwarding: Arc<ForwardingTable>,
12902 handler: ControlHandler,
12903 incumbent_ctx: RouteCtx,
12904 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12905 candidate_ctx: RouteCtx,
12906 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12907 }
12908
12909 async fn swap_with_incumbent() -> Swap {
12910 let registry = Arc::new(Registry::default());
12911 let forwarding = Arc::new(ForwardingTable::default());
12912 let handler =
12913 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12914 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
12915 hello_via_sink(
12916 &handler,
12917 &incumbent_ctx,
12918 &mut incumbent_rx,
12919 hello_frame("aft", PROTOCOL_VERSION, 7),
12920 )
12921 .await;
12922 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
12923 Swap {
12924 registry,
12925 forwarding,
12926 handler,
12927 incumbent_ctx,
12928 incumbent_rx,
12929 candidate_ctx,
12930 candidate_rx,
12931 }
12932 }
12933
12934 fn register_candidate(swap: &Swap, ready: Option<bool>) {
12935 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
12936 candidate_manifest.ready = ready;
12937 let registration = swap
12938 .registry
12939 .register_candidate_with_control_ops(
12940 candidate_manifest,
12941 PROTOCOL_VERSION,
12942 CANDIDATE,
12943 module_baseline_control_ops(),
12944 )
12945 .unwrap();
12946 swap.forwarding
12947 .register_candidate_module_connection(
12948 CANDIDATE,
12949 "aft".to_string(),
12950 PROTOCOL_VERSION,
12951 manifest_concurrency(®istration.manifest),
12952 swap.candidate_ctx.egress.clone(),
12953 )
12954 .unwrap();
12955 }
12956
12957 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
12958 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
12959 swap.registry.promote_candidate("aft").unwrap().unwrap();
12960 cutover.incumbent.unwrap()
12961 }
12962
12963 fn keyed_total(counters: &Value, key: &str) -> u64 {
12964 counters[key]
12965 .as_object()
12966 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
12967 .unwrap_or(0)
12968 }
12969
12970 #[tokio::test]
12971 async fn replacement_logs_old_end_and_new_admission_once() {
12972 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12973 let mut swap = swap_with_incumbent().await;
12974 swap.handler
12975 .supervisor
12976 .open_swap("aft", "candidate-nonce".to_string());
12977 hello_via_sink(
12978 &swap.handler,
12979 &swap.candidate_ctx,
12980 &mut swap.candidate_rx,
12981 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
12982 )
12983 .await;
12984 cutover(&swap);
12985 swap.handler.cleanup_connection(INCUMBENT).unwrap();
12986
12987 let captured = crate::router::test_log::captured_logs(&logs);
12988 println!("captured replacement registry logs:\n{captured}");
12989 let new_admissions: Vec<_> = captured
12990 .lines()
12991 .filter(|line| {
12992 line.contains("connection_id=40")
12993 && line.contains("swap candidate registered; not routable until cutover")
12994 })
12995 .collect();
12996 assert_eq!(
12997 new_admissions.len(),
12998 1,
12999 "unexpected admission logs: {captured}"
13000 );
13001 assert!(new_admissions[0].contains("module_id=aft"));
13002 assert!(new_admissions[0].contains("connection_id=40"));
13003 assert!(
13004 new_admissions[0].contains("swap candidate registered; not routable until cutover")
13005 );
13006 assert!(new_admissions[0].contains("ready=true"));
13007
13008 let old_admissions: Vec<_> = captured
13009 .lines()
13010 .filter(|line| {
13011 line.contains("module registered module_id=aft ")
13012 && line.contains("connection_id=30")
13013 })
13014 .collect();
13015 assert_eq!(
13016 old_admissions.len(),
13017 1,
13018 "unexpected incumbent admission: {captured}"
13019 );
13020
13021 let old_ends: Vec<_> = captured
13022 .lines()
13023 .filter(|line| {
13024 line.contains("connection_id=30") && line.contains("module registration ended")
13025 })
13026 .collect();
13027 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
13028 assert!(old_ends[0].contains("module_id=aft"));
13029 assert!(old_ends[0].contains("connection_id=30"));
13030 assert!(old_ends[0].contains("reason=replaced"));
13031 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
13032 assert_eq!(
13033 captured
13034 .lines()
13035 .filter(|line| line.contains("module registration promoted"))
13036 .count(),
13037 1,
13038 "unexpected promotion logs: {captured}"
13039 );
13040 }
13041
13042 #[tokio::test]
13048 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
13049 let mut swap = swap_with_incumbent().await;
13050 let handler = swap.handler.clone();
13051
13052 let cotenant = ConnectionId::new(31);
13054 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
13055 let (cotenant_task, cotenant_bind) = relay_route_open(
13056 &handler,
13057 cotenant,
13058 &cotenant_ctx.egress,
13059 &mut swap.incumbent_rx,
13060 100,
13061 "aft",
13062 "swap-cotenant",
13063 )
13064 .await;
13065 handler
13066 .handle_control_frame(
13067 &swap.incumbent_ctx,
13068 route_bind_ack(cotenant_bind.header.corr),
13069 )
13070 .await
13071 .unwrap();
13072 assert!(cotenant_task.await.unwrap().is_empty());
13073 let (cotenant_channel, cotenant_epoch) =
13074 published_route(&cotenant_rx.recv().await.unwrap());
13075
13076 let caller = ConnectionId::new(32);
13078 let (caller_ctx, mut caller_rx) = route_ctx(caller);
13079 let (caller_task, caller_bind) = relay_route_open(
13080 &handler,
13081 caller,
13082 &caller_ctx.egress,
13083 &mut swap.incumbent_rx,
13084 101,
13085 "aft",
13086 "swap-caller",
13087 )
13088 .await;
13089 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
13090
13091 register_candidate(&swap, None);
13092 cutover(&swap);
13093
13094 let ack = handler
13096 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
13097 .await;
13098 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
13099 if module_loop_error.is_some() {
13100 handler.cleanup_connection(INCUMBENT).unwrap();
13103 }
13104
13105 assert!(
13107 cotenant_rx.try_recv().is_err(),
13108 "the co-tenant route on the incumbent was torn down by one late ack: \
13109 {module_loop_error:?}"
13110 );
13111 assert!(matches!(
13112 swap.forwarding
13113 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
13114 .unwrap(),
13115 DataRoute::Client(DataRouteState::Bound(_))
13116 ));
13117 assert_eq!(module_loop_error, None);
13118 assert!(swap
13119 .registry
13120 .get_module_by_connection(INCUMBENT)
13121 .unwrap()
13122 .is_some());
13123
13124 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
13126 .await
13127 .expect("the incumbent is told to drop the abandoned binding")
13128 .unwrap()
13129 .frame;
13130 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13131 assert_eq!(goodbye.header.channel, abandoned_channel);
13132 assert_eq!(goodbye.header.epoch, abandoned_epoch);
13133 assert!(swap.incumbent_rx.try_recv().is_err());
13134
13135 let response = caller_task.await.unwrap();
13137 assert_eq!(response.len(), 1);
13138 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
13139 assert!(caller_rx.try_recv().is_err());
13140
13141 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
13145 let counters = handler.counters().snapshot();
13146 assert_eq!(
13147 keyed_total(&counters, "route_open_accepted_by_principal"),
13148 1
13149 );
13150 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
13151 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
13152 }
13153
13154 #[tokio::test]
13158 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
13159 let mut swap = swap_with_incumbent().await;
13160 register_candidate(&swap, None);
13161 let incumbent = cutover(&swap);
13162 swap.forwarding
13163 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
13164 .unwrap()
13165 .expect("the incumbent is still registered");
13166
13167 let client = ConnectionId::new(33);
13168 let (client_ctx, mut client_rx) = route_ctx(client);
13169 let route_handler = swap.handler.clone();
13170 let open_ctx = RouteCtx {
13171 connection_id: client,
13172 egress: client_ctx.egress.clone(),
13173 };
13174 let mut route_task = tokio::spawn(async move {
13175 route_handler
13176 .handle_control_frame(
13177 &open_ctx,
13178 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
13179 )
13180 .await
13181 .unwrap()
13182 });
13183 let bind = tokio::select! {
13184 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
13185 response = &mut route_task => {
13186 let response = response.unwrap();
13187 panic!(
13188 "post-cutover route.open was refused instead of relayed to the candidate: {}",
13189 parse_error(&response[0])["code"]
13190 );
13191 }
13192 };
13193 swap.handler
13194 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
13195 .await
13196 .unwrap();
13197 assert!(route_task.await.unwrap().is_empty());
13198 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
13199 match swap
13200 .forwarding
13201 .lookup_data_route(client, channel, epoch)
13202 .unwrap()
13203 {
13204 DataRoute::Client(DataRouteState::Bound(route)) => {
13205 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
13206 }
13207 other => panic!("expected a bound route on the candidate, got {other:?}"),
13208 }
13209 assert!(swap.incumbent_rx.try_recv().is_err());
13210 }
13211
13212 #[tokio::test]
13217 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
13218 let swap = swap_with_incumbent().await;
13219 register_candidate(&swap, Some(false));
13220 let update = Frame::build(
13221 FrameType::Request,
13222 control_flags(),
13223 0,
13224 0,
13225 55,
13226 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
13227 provides: manifest("aft", PROTOCOL_VERSION).provides,
13228 capabilities: None,
13229 ready: Some(true),
13230 })
13231 .unwrap(),
13232 )
13233 .unwrap();
13234
13235 let replies = swap
13236 .handler
13237 .handle_control_frame(&swap.candidate_ctx, update)
13238 .await
13239 .unwrap();
13240
13241 assert_eq!(replies.len(), 1);
13242 assert_eq!(
13243 replies[0].header.ty,
13244 FrameType::Response,
13245 "candidate catalog.update was refused: {:?}",
13246 serde_json::from_slice::<Value>(&replies[0].body).ok()
13247 );
13248 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
13249 assert_eq!(
13250 swap.registry
13251 .get_module("aft")
13252 .unwrap()
13253 .unwrap()
13254 .connection_id,
13255 INCUMBENT
13256 );
13257 }
13258 }
13259
13260 mod swap_admission {
13264 use super::*;
13265
13266 const INCUMBENT_NONCE: &str = "incumbent-nonce";
13267 const CANDIDATE_NONCE: &str = "candidate-nonce";
13268
13269 fn handler_with_incumbent(
13270 module_id: &str,
13271 reserved: bool,
13272 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
13273 let registry = Arc::new(Registry::default());
13274 let supervisor = SupervisorHandle::new();
13275 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
13276 if reserved {
13277 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
13278 }
13279 let handler =
13280 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
13281 let incumbent = handler
13282 .handle_control(
13283 ConnectionId::new(1),
13284 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
13285 )
13286 .unwrap();
13287 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
13288 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
13289 (registry, supervisor, handler)
13290 }
13291
13292 #[test]
13298 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
13299 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
13300
13301 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
13302 let replies = handler
13303 .handle_control(
13304 ConnectionId::new(connection),
13305 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
13306 )
13307 .unwrap();
13308 assert_eq!(replies[0].header.ty, FrameType::Error);
13309 assert_eq!(
13310 parse_error(&replies[0])["code"],
13311 "swap_token_invalid",
13312 "nonce {nonce:?}"
13313 );
13314 }
13315 assert!(registry.get_candidate("aft").unwrap().is_none());
13316 assert_eq!(
13317 registry.get_module("aft").unwrap().unwrap().connection_id,
13318 ConnectionId::new(1)
13319 );
13320
13321 let admitted = handler
13324 .handle_control(
13325 ConnectionId::new(4),
13326 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
13327 )
13328 .unwrap();
13329 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
13330 assert_eq!(
13331 registry
13332 .get_candidate("aft")
13333 .unwrap()
13334 .unwrap()
13335 .connection_id,
13336 ConnectionId::new(4)
13337 );
13338 assert_eq!(
13339 registry.get_module("aft").unwrap().unwrap().connection_id,
13340 ConnectionId::new(1),
13341 "the candidate must not take the active slot"
13342 );
13343 let replayed = handler
13344 .handle_control(
13345 ConnectionId::new(5),
13346 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
13347 )
13348 .unwrap();
13349 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
13350
13351 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
13355 let squatter = handler
13356 .handle_control(
13357 ConnectionId::new(6),
13358 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
13359 )
13360 .unwrap();
13361 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
13362 assert!(
13363 registry.get_module("aft").unwrap().is_none(),
13364 "a squatter took the active slot of an id being swapped"
13365 );
13366 }
13367
13368 #[test]
13373 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
13374 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
13375
13376 let replies = handler
13377 .handle_control(
13378 ConnectionId::new(2),
13379 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
13380 )
13381 .unwrap();
13382
13383 assert_eq!(
13384 replies[0].header.ty,
13385 FrameType::HelloAck,
13386 "reserved candidate refused: {:?}",
13387 serde_json::from_slice::<Value>(&replies[0].body).ok()
13388 );
13389 assert_eq!(
13390 registry
13391 .get_candidate("vault")
13392 .unwrap()
13393 .unwrap()
13394 .connection_id,
13395 ConnectionId::new(2)
13396 );
13397 }
13398
13399 #[test]
13402 fn without_an_open_swap_the_ordinary_gates_decide() {
13403 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
13404 supervisor.close_swap("vault");
13405
13406 let candidate = handler
13407 .handle_control(
13408 ConnectionId::new(2),
13409 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
13410 )
13411 .unwrap();
13412 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
13413 let duplicate = handler
13414 .handle_control(
13415 ConnectionId::new(3),
13416 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
13417 )
13418 .unwrap();
13419 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
13420 assert!(registry.get_candidate("vault").unwrap().is_none());
13421 }
13422 }
13423
13424 mod scopes {
13428 mod activity {
13429 include!("activity_scope_tests.rs");
13430 }
13431 mod delta {
13432 include!("scope_handler_delta_tests.rs");
13433 }
13434 mod call_modes {
13435 include!("scope_call_mode_tests.rs");
13436 }
13437
13438 use subc_protocol::scope::{
13439 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
13440 ScopeStatus,
13441 };
13442
13443 use super::*;
13444
13445 const OWNER: &str = "prefrontal-core";
13446
13447 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
13448 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
13449 }
13450
13451 async fn call(
13452 handler: &ControlHandler,
13453 ctx: &RouteCtx,
13454 request: &ModuleControlRequestFromModule,
13455 ) -> Frame {
13456 let body = serde_json::to_vec(request).unwrap();
13457 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
13458 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
13459 assert_eq!(replies.len(), 1, "{replies:?}");
13460 replies.pop().unwrap()
13461 }
13462
13463 async fn sync(
13464 handler: &ControlHandler,
13465 ctx: &RouteCtx,
13466 generation: u64,
13467 scopes: Vec<ScopeRecord>,
13468 ) -> Result<ModuleControlResponseToModule, String> {
13469 let reply = call(
13470 handler,
13471 ctx,
13472 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
13473 )
13474 .await;
13475 match reply.header.ty {
13476 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
13477 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
13478 }
13479 }
13480
13481 async fn describe(
13482 handler: &ControlHandler,
13483 ctx: &RouteCtx,
13484 owner: &str,
13485 scope_ref: &str,
13486 ) -> ModuleControlResponseToModule {
13487 let reply = call(
13488 handler,
13489 ctx,
13490 &ModuleControlRequestFromModule::ScopeDescribe {
13491 owner: Principal::Reserved {
13492 module_id: owner.to_string(),
13493 },
13494 scope_ref: scope_ref.to_string(),
13495 },
13496 )
13497 .await;
13498 assert_eq!(
13499 reply.header.ty,
13500 FrameType::Response,
13501 "{:?}",
13502 parse_error(&reply)
13503 );
13504 serde_json::from_slice(&reply.body).unwrap()
13505 }
13506
13507 async fn module(
13509 handler: &ControlHandler,
13510 connection: u64,
13511 module_id: &str,
13512 nonce: Option<&str>,
13513 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
13514 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
13515 hello_via_sink(
13516 handler,
13517 &ctx,
13518 &mut rx,
13519 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
13520 )
13521 .await;
13522 (ctx, rx)
13523 }
13524
13525 #[tokio::test]
13528 async fn a_client_connection_cannot_sync_or_describe() {
13529 let handler = ControlHandler::new(Arc::new(Registry::default()));
13530 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
13531 for request in [
13532 ModuleControlRequestFromModule::ScopeSync {
13533 generation: 1,
13534 scopes: vec![head("s", 1)],
13535 },
13536 ModuleControlRequestFromModule::ScopeDescribe {
13537 owner: Principal::Direct,
13538 scope_ref: "s".to_string(),
13539 },
13540 ] {
13541 let reply = call(&handler, &ctx, &request).await;
13542 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
13543 }
13544 assert!(
13545 !handler
13546 .scopes
13547 .read()
13548 .unwrap()
13549 .describe(
13550 &Principal::Reserved {
13551 module_id: OWNER.to_string()
13552 },
13553 "s"
13554 )
13555 .owner_synced
13556 );
13557 }
13558
13559 #[tokio::test]
13562 async fn a_module_without_a_supervised_launch_cannot_sync() {
13563 let handler = ControlHandler::new(Arc::new(Registry::default()));
13564 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
13565 assert_eq!(
13566 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
13567 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13568 );
13569 }
13570
13571 #[tokio::test]
13572 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
13573 let supervisor = SupervisorHandle::new();
13574 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13575 let handler = ControlHandler::new(Arc::new(Registry::default()))
13576 .with_supervisor(supervisor.clone());
13577 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
13578 sync(&handler, &incumbent, 1, vec![head("s", 1)])
13579 .await
13580 .expect("the current launch syncs");
13581
13582 supervisor.open_swap(OWNER, "n2".to_string());
13585 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
13586 assert_eq!(
13587 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
13588 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13589 );
13590 sync(&handler, &incumbent, 2, vec![head("s", 1)])
13591 .await
13592 .expect("the serving owner syncs during the swap");
13593
13594 supervisor.close_swap(OWNER);
13597 assert_eq!(
13598 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
13599 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13600 );
13601 sync(&handler, &incumbent, 3, vec![head("s", 1)])
13602 .await
13603 .expect("the serving owner syncs after the rollback");
13604 handler.cleanup_connection(candidate.connection_id).unwrap();
13605
13606 supervisor.open_swap(OWNER, "n3".to_string());
13611 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
13612 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
13613 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
13614 .await
13615 .expect("the promoted launch takes authority");
13616 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
13617 panic!("unexpected reply {reply:?}");
13618 };
13619 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
13620 assert_eq!(
13621 sync(&handler, &incumbent, 4, Vec::new()).await,
13622 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13623 );
13624 }
13625
13626 #[tokio::test]
13629 async fn closing_the_authority_connection_frees_sync_authority() {
13630 let supervisor = SupervisorHandle::new();
13631 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13632 let handler = ControlHandler::new(Arc::new(Registry::default()))
13633 .with_supervisor(supervisor.clone());
13634 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
13635 sync(&handler, &first, 10, vec![head("s", 1)])
13636 .await
13637 .unwrap();
13638 handler.cleanup_connection(first.connection_id).unwrap();
13639
13640 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
13641 sync(&handler, &second, 1, vec![head("s", 1)])
13642 .await
13643 .expect("the next connection takes the released authority");
13644 }
13645
13646 #[tokio::test]
13647 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
13648 let supervisor = SupervisorHandle::new();
13649 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13650 let handler =
13651 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
13652 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
13653 sync(&handler, &first, 10, vec![head("s", 1)])
13654 .await
13655 .unwrap();
13656 handler
13657 .handle_control_frame(
13658 &first,
13659 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
13660 )
13661 .await
13662 .unwrap();
13663 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
13664 sync(&handler, &second, 1, vec![head("s", 1)])
13665 .await
13666 .expect("GOODBYE releases authority even if the old socket remains open");
13667 }
13668
13669 #[tokio::test]
13670 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
13671 let registry = Arc::new(Registry::default());
13672 let supervisor_handle = SupervisorHandle::new();
13673 let supervisor =
13674 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
13675 .with_handle(supervisor_handle.clone())
13676 .with_daemon_incarnation("incarnation-7".to_string());
13677 supervisor
13680 .supervise_configured(
13681 ModuleSpec {
13682 module_id: OWNER.to_string(),
13683 program: PathBuf::from("/nonexistent/prefrontal-core"),
13684 args: Vec::new(),
13685 env: Vec::new(),
13686 reserved: false,
13687 reserved_prefixes: Vec::new(),
13688 protocol: ModuleProtocol::Subc,
13689 overlap: Default::default(),
13690 },
13691 false,
13692 )
13693 .unwrap();
13694 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
13695 let handler =
13696 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13697 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
13698
13699 let ModuleControlResponseToModule::ScopeDescribe {
13701 status,
13702 daemon_incarnation,
13703 owner_synced,
13704 owner_configured,
13705 scope,
13706 ..
13707 } = describe(&handler, &reader, OWNER, "s").await
13708 else {
13709 panic!("not a describe reply");
13710 };
13711 assert_eq!(status, ScopeStatus::NotLive);
13712 assert_eq!(daemon_incarnation, "incarnation-7");
13713 assert!(!owner_synced);
13714 assert!(owner_configured);
13715 assert!(scope.is_none());
13716
13717 let ModuleControlResponseToModule::ScopeDescribe {
13720 status,
13721 owner_configured,
13722 ..
13723 } = describe(&handler, &reader, "ghost", "s").await
13724 else {
13725 panic!("not a describe reply");
13726 };
13727 assert_eq!(status, ScopeStatus::NotLive);
13728 assert!(!owner_configured);
13729
13730 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
13732 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
13733 let ModuleControlResponseToModule::ScopeDescribe {
13734 status,
13735 scope_epoch,
13736 owner_synced,
13737 scope,
13738 ..
13739 } = describe(&handler, &reader, OWNER, "s").await
13740 else {
13741 panic!("not a describe reply");
13742 };
13743 assert_eq!(status, ScopeStatus::Live);
13744 assert_eq!(scope_epoch, Some(4));
13745 assert!(owner_synced);
13746 let stamp = scope.expect("a live scope carries its stamp");
13747 assert!(
13748 stamp.owner_authorized,
13749 "prefrontal-core is the default authority"
13750 );
13751 assert_eq!(stamp.kind, ScopeKind::Head);
13752 }
13753
13754 #[tokio::test]
13755 async fn scope_authority_owners_decides_owner_authorized() {
13756 let supervisor = SupervisorHandle::new();
13757 supervisor.set_spawn_nonce("broca", "b1".to_string());
13758 let handler = ControlHandler::new(Arc::new(Registry::default()))
13759 .with_supervisor(supervisor)
13760 .with_scope_authority_owners(vec!["broca".to_string()]);
13761 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
13762 let mut gated = head("s", 1);
13763 gated.attributes.agent_id = Some("agent".to_string());
13764 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
13765 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
13766 describe(&handler, &broca, "broca", "s").await
13767 else {
13768 panic!("not a describe reply");
13769 };
13770 assert!(scope.unwrap().owner_authorized);
13771 }
13772
13773 #[tokio::test]
13777 async fn scope_ops_and_the_scopes_capability_are_advertised() {
13778 let handler = ControlHandler::new(Arc::new(Registry::default()));
13779 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
13780 let ack = hello_via_sink(
13781 &handler,
13782 &ctx,
13783 &mut rx,
13784 hello_frame("m", PROTOCOL_VERSION, 1),
13785 )
13786 .await;
13787 let ack = parse_ack(&ack);
13788 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
13789 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
13790 }
13791 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
13792
13793 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
13794 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13795 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13796 let reply = handler
13797 .handle_control_frame(&client, frame)
13798 .await
13799 .unwrap()
13800 .pop()
13801 .unwrap();
13802 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13803 serde_json::from_slice(&reply.body).unwrap()
13804 else {
13805 panic!("not a server.describe reply");
13806 };
13807 assert!(
13808 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
13809 "{capabilities:?}"
13810 );
13811 }
13812
13813 const PLEXUS: &str = "plexus";
13816 const OTHER: &str = "other";
13817 const AFT: &str = "aft";
13818 const BROCA: &str = "broca";
13819 const MAGIC: &str = "magic-context";
13820
13821 fn nonce(module_id: &str) -> String {
13822 format!("nonce-{module_id}")
13823 }
13824
13825 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
13826 let (tx, rx) = mpsc::channel(64);
13827 (
13828 RouteCtx {
13829 connection_id: ConnectionId::new(connection),
13830 egress: FrameSink::new(tx),
13831 },
13832 rx,
13833 )
13834 }
13835
13836 struct Rig {
13840 handler: ControlHandler,
13841 forwarding: Arc<ForwardingTable>,
13842 owner: RouteCtx,
13843 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13844 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
13845 generation: u64,
13846 next_connection: u64,
13847 _supervisor: Supervisor,
13848 }
13849
13850 async fn rig() -> Rig {
13851 rig_with_flow_support(true).await
13852 }
13853
13854 async fn rig_with_flow_support(flow_support: bool) -> Rig {
13855 rig_with_scope_capabilities(flow_support, false).await
13856 }
13857
13858 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
13859 let mut provides = Vec::new();
13860 if flow_support {
13861 provides.push("flow-scopes/v1");
13862 }
13863 if run_support {
13864 provides.push("agent-run-scopes/v1");
13865 }
13866 rig_with_target_capabilities(&provides).await
13867 }
13868
13869 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
13870 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
13871 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
13872 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
13875 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
13876 Frame::build(
13877 FrameType::Hello,
13878 control_flags(),
13879 0,
13880 0,
13881 connection,
13882 serde_json::to_vec(&body).unwrap(),
13883 )
13884 .unwrap()
13885 }
13886
13887 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
13888 let registry = Arc::new(Registry::default());
13889 let forwarding = Arc::new(ForwardingTable::default());
13890 let supervisor_handle = SupervisorHandle::new();
13891 let supervisor =
13892 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
13893 .with_handle(supervisor_handle.clone());
13894 supervisor
13895 .supervise_configured(
13896 ModuleSpec {
13897 module_id: OWNER.to_string(),
13898 program: PathBuf::from("/nonexistent/prefrontal-core"),
13899 args: Vec::new(),
13900 env: Vec::new(),
13901 reserved: false,
13902 reserved_prefixes: Vec::new(),
13903 protocol: ModuleProtocol::Subc,
13904 overlap: Default::default(),
13905 },
13906 false,
13907 )
13908 .unwrap();
13909 for module_id in [OWNER, AFT, BROCA, MAGIC] {
13910 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
13911 }
13912 let handler =
13913 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13914 .with_supervisor(supervisor_handle);
13915 let (owner, mut owner_rx) = wide_ctx(1);
13916 hello_via_sink(
13917 &handler,
13918 &owner,
13919 &mut owner_rx,
13920 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
13921 )
13922 .await;
13923 let mut modules = BTreeMap::new();
13924 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
13925 let (ctx, mut rx) = wide_ctx(connection);
13926 let hello = hello_with_capabilities(module_id, connection, provides);
13927 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
13928 modules.insert(module_id.to_string(), (ctx, rx));
13929 }
13930 Rig {
13931 handler,
13932 forwarding,
13933 owner,
13934 _owner_rx: owner_rx,
13935 modules,
13936 generation: 0,
13937 next_connection: 100,
13938 _supervisor: supervisor,
13939 }
13940 }
13941
13942 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
13943 ScopeCarrier::new(Principal::Reserved {
13944 module_id: module_id.to_string(),
13945 })
13946 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
13947 }
13948
13949 fn session(scope_epoch: u64) -> ScopeRecord {
13952 let mut record = head("s", scope_epoch);
13953 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
13954 record.attributes.agent_id = Some("agent-1".to_string());
13955 record.attributes.delegates = true;
13956 record
13957 }
13958
13959 impl Rig {
13960 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
13961 self.generation += 1;
13962 sync(&self.handler, &self.owner, self.generation, scopes)
13963 .await
13964 .expect("the owner's sync is accepted");
13965 }
13966
13967 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
13968 ScopeSelector {
13969 owner: Principal::Reserved {
13970 module_id: OWNER.to_string(),
13971 },
13972 scope_ref: scope_ref.to_string(),
13973 scope_epoch,
13974 }
13975 }
13976
13977 fn open_frame(
13978 &mut self,
13979 opener: Option<&str>,
13980 target: &str,
13981 scope: Option<ScopeSelector>,
13982 ) -> (
13983 RouteCtx,
13984 mpsc::Receiver<crate::router::OutboundFrame>,
13985 Frame,
13986 ) {
13987 self.next_connection += 1;
13988 let (ctx, rx) = wide_ctx(self.next_connection);
13989 let root = unique_project_root("scoped-open");
13990 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
13991 target: RouteTarget::ToolProvider {
13992 module_id: target.to_string(),
13993 },
13994 identity: BindIdentity::new(
13995 root.path().to_path_buf(),
13996 "unit".to_string(),
13997 "session".to_string(),
13998 ),
13999 consumer_identity: opener.map(|module_id| ConsumerIdentity {
14000 module_id: module_id.to_string(),
14001 launch_nonce: nonce(module_id),
14002 }),
14003 consumer_capabilities: None,
14004 role_versions: None,
14005 admission_facts: None,
14006 scope,
14007 })
14008 .unwrap();
14009 let frame = Frame::build(
14010 FrameType::Request,
14011 control_flags(),
14012 0,
14013 0,
14014 self.next_connection,
14015 body,
14016 )
14017 .unwrap();
14018 (ctx, rx, frame)
14019 }
14020
14021 async fn refused(
14023 &mut self,
14024 opener: Option<&str>,
14025 target: &str,
14026 scope: Option<ScopeSelector>,
14027 ) -> String {
14028 self.refusal_body(opener, target, scope).await["code"]
14029 .as_str()
14030 .unwrap()
14031 .to_string()
14032 }
14033
14034 async fn refusal_body(
14035 &mut self,
14036 opener: Option<&str>,
14037 target: &str,
14038 scope: Option<ScopeSelector>,
14039 ) -> Value {
14040 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
14041 let replies = tokio::time::timeout(
14042 Duration::from_secs(2),
14043 self.handler.handle_control_frame(&ctx, frame),
14044 )
14045 .await
14046 .expect("the open must be refused before waiting for a bind ack")
14047 .unwrap();
14048 assert_eq!(replies.len(), 1, "{replies:?}");
14049 assert_eq!(replies[0].header.ty, FrameType::Error);
14050 let (_, module_rx) = self.modules.get_mut(target).unwrap();
14051 assert!(
14052 module_rx.try_recv().is_err(),
14053 "a refused open relays nothing"
14054 );
14055 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
14056 parse_error(&replies[0])
14057 }
14058
14059 async fn relayed(
14061 &mut self,
14062 opener: Option<&str>,
14063 target: &str,
14064 scope: Option<ScopeSelector>,
14065 ) -> Relayed {
14066 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
14067 let handler = self.handler.clone();
14068 let task_ctx = ctx.clone();
14069 let task = tokio::spawn(async move {
14070 handler
14071 .handle_control_frame(&task_ctx, frame)
14072 .await
14073 .unwrap()
14074 });
14075 let (_, module_rx) = self.modules.get_mut(target).unwrap();
14076 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
14077 .await
14078 .expect("the target receives the relayed route.bind")
14079 .unwrap()
14080 .frame;
14081 Relayed {
14082 target: target.to_string(),
14083 client: ctx,
14084 client_rx: rx,
14085 task,
14086 bind,
14087 }
14088 }
14089
14090 async fn ack(&self, relayed: &Relayed) {
14091 let (module, _) = &self.modules[&relayed.target];
14092 self.handler
14093 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
14094 .await
14095 .unwrap();
14096 }
14097
14098 async fn bound(
14100 &mut self,
14101 opener: Option<&str>,
14102 target: &str,
14103 scope: Option<ScopeSelector>,
14104 ) -> Bound {
14105 let relayed = self.relayed(opener, target, scope).await;
14106 self.ack(&relayed).await;
14107 let Relayed {
14108 target,
14109 client,
14110 mut client_rx,
14111 task,
14112 bind,
14113 } = relayed;
14114 assert!(
14115 task.await.unwrap().is_empty(),
14116 "the open is answered by commit"
14117 );
14118 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
14119 Bound {
14120 target,
14121 client,
14122 client_rx,
14123 channel,
14124 epoch,
14125 bind,
14126 }
14127 }
14128
14129 fn live(&self, route: &Bound) -> bool {
14130 matches!(
14131 self.forwarding
14132 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
14133 .unwrap(),
14134 DataRoute::Client(DataRouteState::Bound(_))
14135 )
14136 }
14137 }
14138
14139 struct Relayed {
14140 target: String,
14141 client: RouteCtx,
14142 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14143 task: tokio::task::JoinHandle<Vec<Frame>>,
14144 bind: Frame,
14145 }
14146
14147 struct Bound {
14148 target: String,
14149 client: RouteCtx,
14150 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14151 channel: u16,
14152 epoch: u32,
14153 bind: Frame,
14154 }
14155
14156 impl Bound {
14157 fn closed_reason(&mut self) -> RouteCloseReason {
14160 let mut reason = None;
14161 let mut goodbye = false;
14162 while let Ok(outbound) = self.client_rx.try_recv() {
14163 let frame = outbound.frame;
14164 match frame.header.ty {
14165 FrameType::Goodbye => {
14166 assert_eq!(
14167 (frame.header.channel, frame.header.epoch),
14168 (self.channel, self.epoch)
14169 );
14170 goodbye = true;
14171 }
14172 FrameType::Push => {
14173 let ClientControlPush::RouteClosed {
14174 reason: r,
14175 module_id,
14176 ..
14177 } = serde_json::from_slice(&frame.body).unwrap()
14178 else {
14179 panic!("unexpected push");
14180 };
14181 assert_eq!(module_id, self.target);
14182 reason = Some(r);
14183 }
14184 other => panic!("unexpected frame {other:?}"),
14185 }
14186 }
14187 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
14188 reason.expect("the client is told why the route closed")
14189 }
14190
14191 fn untouched(&mut self) -> bool {
14192 self.client_rx.try_recv().is_err()
14193 }
14194
14195 fn stamp(&self) -> Option<ScopeStamp> {
14196 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
14197 ModuleControlRequest::RouteBind { scope, .. } => scope,
14198 other => panic!("expected a route.bind, got {other:?}"),
14199 }
14200 }
14201 }
14202
14203 #[tokio::test]
14204 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
14205 ) {
14206 let mut rig = rig().await;
14207 let mut record = session(1);
14208 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
14209 record.child_owners = vec![Principal::Reserved {
14210 module_id: MAGIC.to_string(),
14211 }];
14212 rig.sync(vec![record]).await;
14213 let scope = || Some(rig_selector("s", Some(1)));
14214
14215 rig.bound(Some(OWNER), PLEXUS, scope()).await;
14218 rig.bound(Some(AFT), OTHER, scope()).await;
14219 rig.bound(Some(BROCA), PLEXUS, scope()).await;
14220
14221 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
14225 assert_eq!(
14226 rig.refused(opener, target, scope()).await,
14227 error_codes::SCOPE_NOT_CARRIER,
14228 "{opener:?} -> {target}"
14229 );
14230 }
14231 }
14232
14233 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
14234 ScopeSelector {
14235 owner: Principal::Reserved {
14236 module_id: OWNER.to_string(),
14237 },
14238 scope_ref: scope_ref.to_string(),
14239 scope_epoch,
14240 }
14241 }
14242
14243 #[tokio::test]
14244 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
14245 let mut rig = rig().await;
14246 rig.sync(vec![session(1)]).await;
14247 for opener in [OWNER, AFT] {
14248 assert_eq!(
14249 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
14250 .await,
14251 error_codes::SCOPE_EPOCH_REQUIRED,
14252 "{opener}"
14253 );
14254 }
14255 }
14256
14257 #[tokio::test]
14258 async fn admission_separates_not_synced_not_live_and_ended() {
14259 let mut rig = rig().await;
14260 let code = rig
14262 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14263 .await;
14264 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
14265 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
14266
14267 let ghost = ScopeSelector {
14269 owner: Principal::Reserved {
14270 module_id: "ghost".to_string(),
14271 },
14272 scope_ref: "s".to_string(),
14273 scope_epoch: Some(1),
14274 };
14275 assert_eq!(
14276 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
14277 error_codes::SCOPE_NOT_LIVE
14278 );
14279
14280 rig.sync(vec![session(2)]).await;
14281 assert_eq!(
14282 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
14283 .await,
14284 error_codes::SCOPE_NOT_LIVE
14285 );
14286 for epoch in [1, 3] {
14287 assert_eq!(
14288 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
14289 .await,
14290 error_codes::SCOPE_ENDED,
14291 "epoch {epoch}"
14292 );
14293 }
14294 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
14296 .await;
14297 }
14298
14299 #[tokio::test]
14300 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
14301 let mut rig = rig().await;
14302 rig.sync(vec![session(1)]).await;
14303 let route = rig
14304 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14305 .await;
14306 let stamp = route.stamp().expect("a scoped bind carries the stamp");
14307 assert_eq!(stamp.scope_ref, "s");
14308 assert_eq!(stamp.scope_epoch, 1);
14309 assert_eq!(stamp.kind, ScopeKind::Head);
14310 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
14311 assert!(stamp.attributes.delegates);
14312 assert!(stamp.owner_authorized);
14313 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
14314 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
14315
14316 let (broca, mut broca_rx) = wide_ctx(50);
14319 hello_via_sink(
14320 &rig.handler,
14321 &broca,
14322 &mut broca_rx,
14323 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
14324 )
14325 .await;
14326 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
14327 .await
14328 .unwrap();
14329 let own = ScopeSelector {
14330 owner: Principal::Reserved {
14331 module_id: BROCA.to_string(),
14332 },
14333 scope_ref: "b".to_string(),
14334 scope_epoch: Some(1),
14335 };
14336 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
14337 assert!(!route.stamp().unwrap().owner_authorized);
14338 }
14339
14340 #[tokio::test]
14341 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
14342 let mut rig = rig().await;
14343 let mut record = head("s", 1);
14344 record.carriers = vec![carrier(AFT, None)];
14345 let flow_id = "Flow:run-7/step_2!~";
14346 record.attributes.flow_id = Some(flow_id.to_string());
14347 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
14348 .await
14349 .unwrap();
14350 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
14351 panic!("not a sync reply");
14352 };
14353 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
14354 let route = rig
14355 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14356 .await;
14357 assert!(rig.live(&route), "the stamped bind committed");
14358 let stamp = route.stamp().expect("a flow scope carries a stamp");
14359 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
14360 assert_eq!(stamp.attributes.agent_id, None);
14361 assert!(!stamp.attributes.delegates);
14362 assert!(stamp.owner_authorized);
14363 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
14364 assert_eq!(unscoped.stamp(), None);
14365 }
14366
14367 #[tokio::test]
14368 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
14369 let mut rig = rig_with_flow_support(false).await;
14370 let mut record = session(1);
14371 record.attributes.flow_id = Some("flow:7".to_string());
14372 rig.sync(vec![record]).await;
14373 for opener in [OWNER, AFT] {
14374 let body = rig
14375 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
14376 .await;
14377 assert_eq!(body["code"], "target_flow_unsupported");
14378 let message = body["message"].as_str().unwrap();
14379 for required in [PLEXUS, "flow-scopes/v1"] {
14380 assert!(message.contains(required), "{message}");
14381 }
14382 }
14383 }
14384
14385 #[tokio::test]
14386 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
14387 let mut rig = rig_with_flow_support(true).await;
14388 let mut record = session(1);
14389 record.attributes.flow_id = Some("flow:7".to_string());
14390 rig.sync(vec![record]).await;
14391 for opener in [OWNER, AFT] {
14392 let route = rig
14393 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
14394 .await;
14395 assert!(rig.live(&route));
14396 assert_eq!(
14397 route.stamp().unwrap().attributes.flow_id.as_deref(),
14398 Some("flow:7")
14399 );
14400 }
14401 }
14402
14403 #[tokio::test]
14404 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
14405 use std::future::Future;
14406
14407 let mut rig = rig_with_flow_support(true).await;
14408 let mut record = session(1);
14409 record.attributes.flow_id = Some("flow:7".to_string());
14410 rig.sync(vec![record]).await;
14411 let (client, mut client_rx, frame) =
14412 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
14413 for _ in 0..64 {
14416 client.egress.try_send(route_bind_ack(1)).unwrap();
14417 }
14418 let handler = rig.handler.clone();
14419 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
14420 std::future::poll_fn(|cx| {
14421 assert!(open.as_mut().poll(cx).is_pending());
14422 std::task::Poll::Ready(())
14423 })
14424 .await;
14425 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
14426
14427 let old_connection = rig.modules[PLEXUS].0.connection_id;
14428 rig.handler.cleanup_connection(old_connection).unwrap();
14429 let (replacement, mut replacement_rx) = wide_ctx(200);
14430 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
14431 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
14432 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
14433 let hello = Frame::build(
14434 FrameType::Hello,
14435 control_flags(),
14436 0,
14437 0,
14438 200,
14439 serde_json::to_vec(&body).unwrap(),
14440 )
14441 .unwrap();
14442 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
14443
14444 client_rx.try_recv().unwrap();
14445 let replies = tokio::time::timeout(Duration::from_secs(2), open)
14446 .await
14447 .expect("the replacement is refused without waiting for a bind ack")
14448 .unwrap();
14449 assert_eq!(replies.len(), 1);
14450 let body = parse_error(&replies[0]);
14451 assert_eq!(body["code"], "target_flow_unsupported");
14452 for required in [PLEXUS, "flow-scopes/v1"] {
14453 assert!(body["message"].as_str().unwrap().contains(required));
14454 }
14455 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
14456 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
14457 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
14458 assert!(rig
14459 .handler
14460 .registry
14461 .get_module_by_connection(replacement.connection_id)
14462 .unwrap()
14463 .is_some());
14464 }
14465
14466 #[tokio::test]
14467 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
14468 {
14469 let mut rig = rig_with_flow_support(false).await;
14470 rig.sync(vec![session(1)]).await;
14471 let route = rig
14472 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14473 .await;
14474 assert!(rig.live(&route));
14475 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
14476 let mut record = session(1);
14477 record.attributes.flow_id = Some("flow:7".to_string());
14478 rig.sync(vec![record]).await;
14479 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
14480 assert!(rig.live(&unscoped));
14481 assert_eq!(unscoped.stamp(), None);
14482 }
14483
14484 #[tokio::test]
14485 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
14486 let mut rig = rig().await;
14487 let mut record = session(1);
14488 record.attributes.flow_id = Some("flow:7".to_string());
14489 rig.sync(vec![record.clone()]).await;
14490 let before = rig.forwarding.published_scope_tag(OWNER, "s");
14491 let mut owner_route = rig
14492 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
14493 .await;
14494 let mut carrier_route = rig
14495 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
14496 .await;
14497 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
14498 rig.sync(vec![record.clone()]).await;
14499 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
14500 assert!(rig.live(&owner_route) && owner_route.untouched());
14501 assert!(rig.live(&carrier_route) && carrier_route.untouched());
14502
14503 record.attributes.flow_id = Some("flow:8".to_string());
14504 rig.sync(vec![record]).await;
14505 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
14506 let before = before.unwrap();
14507 assert_eq!(after.scope_epoch, before.scope_epoch);
14508 assert!(after.version > before.version);
14509 for route in [&mut owner_route, &mut carrier_route] {
14510 assert!(!rig.live(route));
14511 assert_eq!(
14512 route.closed_reason(),
14513 RouteCloseReason::ScopeDelegationChanged
14514 );
14515 }
14516 assert!(rig.live(&unscoped) && unscoped.untouched());
14517 for target in [PLEXUS, OTHER] {
14520 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
14521 let goodbye = module_rx
14522 .try_recv()
14523 .expect("the provider sees the drain")
14524 .frame;
14525 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14526 assert!(module_rx.try_recv().is_err());
14527 }
14528 let rebound = rig
14529 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14530 .await;
14531 assert_eq!(
14532 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
14533 Some("flow:8")
14534 );
14535 }
14536
14537 #[tokio::test]
14542 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
14543 let mut rig = rig().await;
14544 rig.sync(vec![session(1)]).await;
14545 let mut cotenant = rig
14546 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
14547 .await;
14548
14549 let mut changed = session(1);
14550 changed.child_owners.push(Principal::Reserved {
14551 module_id: MAGIC.to_string(),
14552 });
14553 let mut ended = None;
14554 for (code, next) in [
14555 (error_codes::SCOPE_CHANGED, vec![changed]),
14556 (error_codes::SCOPE_ENDED, Vec::new()),
14557 ] {
14558 let relayed = rig
14559 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14560 .await;
14561 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
14562 ended = Some(next.is_empty());
14563 rig.sync(next).await;
14564 rig.ack(&relayed).await;
14565 let replies = relayed.task.await.unwrap();
14566 assert_eq!(replies.len(), 1, "{replies:?}");
14567 assert_eq!(parse_error(&replies[0])["code"], code);
14568 assert_eq!(
14569 subc_protocol::error_codes::is_retryable_route_open(code),
14570 code == error_codes::SCOPE_CHANGED
14571 );
14572 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
14573 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
14575 let mut goodbyes = Vec::new();
14578 while let Ok(outbound) = plexus_rx.try_recv() {
14579 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
14580 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
14581 }
14582 assert!(
14583 goodbyes.contains(&(bind_channel, bind_epoch)),
14584 "{goodbyes:?}"
14585 );
14586 assert!(rig
14587 .handler
14588 .registry
14589 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
14590 .unwrap()
14591 .is_some());
14592 }
14593 assert_eq!(ended, Some(true));
14594 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
14597 }
14598
14599 #[tokio::test]
14602 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
14603 struct Case {
14604 name: &'static str,
14605 change: fn(&mut ScopeRecord),
14606 closed: [Option<RouteCloseReason>; 4],
14609 }
14610 use RouteCloseReason::*;
14611 let cases = [
14612 Case {
14613 name: "a carrier entry removed",
14614 change: |r| {
14615 r.carriers.retain(|c| {
14616 c.principal
14617 != Principal::Reserved {
14618 module_id: AFT.to_string(),
14619 }
14620 })
14621 },
14622 closed: [None, Some(ScopeCarrierRemoved), None, None],
14623 },
14624 Case {
14625 name: "a target removed from a carrier",
14626 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
14627 closed: [None, None, None, Some(ScopeCarrierRemoved)],
14628 },
14629 Case {
14630 name: "a bare carrier narrowed to targets",
14631 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
14632 closed: [None, Some(ScopeCarrierRemoved), None, None],
14633 },
14634 Case {
14635 name: "delegates turned off",
14636 change: |r| r.attributes.delegates = false,
14637 closed: [Some(ScopeDelegationChanged); 4],
14638 },
14639 Case {
14640 name: "agent_id changed",
14641 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
14642 closed: [Some(ScopeDelegationChanged); 4],
14643 },
14644 Case {
14645 name: "a carrier added, child owners changed, the record re-sent",
14646 change: |r| {
14647 r.carriers.push(carrier(MAGIC, None));
14648 r.child_owners.push(Principal::Reserved {
14649 module_id: MAGIC.to_string(),
14650 });
14651 },
14652 closed: [None; 4],
14653 },
14654 Case {
14655 name: "a target added",
14656 change: |r| {
14657 r.carriers[1]
14658 .targets
14659 .as_mut()
14660 .unwrap()
14661 .push("third".to_string())
14662 },
14663 closed: [None; 4],
14664 },
14665 Case {
14666 name: "delegates turned on",
14667 change: |r| r.attributes.delegates = true,
14668 closed: [None; 4],
14669 },
14670 ];
14671 for case in cases {
14672 let mut rig = rig().await;
14673 rig.sync(vec![session(1)]).await;
14674 let scope = || Some(rig_selector("s", Some(1)));
14675 let mut routes = [
14676 rig.bound(Some(OWNER), PLEXUS, scope()).await,
14677 rig.bound(Some(AFT), PLEXUS, scope()).await,
14678 rig.bound(Some(BROCA), PLEXUS, scope()).await,
14679 rig.bound(Some(BROCA), OTHER, scope()).await,
14680 ];
14681 let mut record = session(1);
14682 (case.change)(&mut record);
14683 rig.sync(vec![record]).await;
14684 for (index, expected) in case.closed.iter().enumerate() {
14685 let route = &mut routes[index];
14686 match expected {
14687 Some(reason) => {
14688 assert!(!rig.live(route), "{}: route {index} still live", case.name);
14689 assert_eq!(
14690 route.closed_reason(),
14691 *reason,
14692 "{}: route {index}",
14693 case.name
14694 );
14695 }
14696 None => {
14697 assert!(rig.live(route), "{}: route {index} closed", case.name);
14698 assert!(
14699 route.untouched(),
14700 "{}: route {index} was told something",
14701 case.name
14702 );
14703 }
14704 }
14705 }
14706 }
14707 }
14708
14709 #[tokio::test]
14710 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
14711 for next in [Vec::new(), vec![session(2)]] {
14713 let mut rig = rig().await;
14714 rig.sync(vec![session(1)]).await;
14715 let mut route = rig
14716 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14717 .await;
14718 rig.sync(next).await;
14719 assert!(!rig.live(&route));
14720 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
14721 }
14722
14723 let mut rig = rig().await;
14726 let mut child = session(1);
14727 child.scope_ref = "child".to_string();
14728 child.kind = ScopeKind::Worker;
14729 child.parent = Some(ScopeParent::new(
14730 Principal::Reserved {
14731 module_id: OWNER.to_string(),
14732 },
14733 "s".to_string(),
14734 1,
14735 ));
14736 rig.sync(vec![session(1), child.clone()]).await;
14737 let mut child_route = rig
14738 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
14739 .await;
14740 assert_eq!(
14741 child_route.stamp().unwrap().parent_state,
14742 Some(ParentState::Linked)
14743 );
14744 rig.sync(vec![child]).await;
14745 assert!(!rig.live(&child_route));
14746 assert_eq!(
14747 child_route.closed_reason(),
14748 RouteCloseReason::ScopeParentEnded
14749 );
14750 }
14751
14752 #[tokio::test]
14753 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
14754 ) {
14755 let mut rig = rig().await;
14756 rig.sync(vec![session(1)]).await;
14757 let mut route = rig
14758 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14759 .await;
14760 let before = rig.forwarding.published_scope_tag(OWNER, "s");
14761 rig.sync(vec![session(1)]).await;
14762 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
14763 assert!(rig.live(&route) && route.untouched());
14764
14765 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
14769 .forwarding
14770 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
14771 .unwrap()
14772 else {
14773 panic!("the route is bound");
14774 };
14775 binding.flow.acquire_tagged(9, false).await.unwrap();
14776 let mut widened = session(1);
14777 widened.carriers.push(carrier(MAGIC, None));
14778 rig.sync(vec![widened]).await;
14779 assert!(rig.live(&route) && route.untouched());
14780 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
14781 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
14782 assert_eq!(binding.flow.in_flight(), 1);
14785 binding
14786 .flow
14787 .acquire_tagged(10, false)
14788 .await
14789 .expect("the flow is still open");
14790 }
14791
14792 #[tokio::test]
14795 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
14796 let mut rig = rig().await;
14797 rig.sync(vec![session(1)]).await;
14798 let mut on_incumbent = rig
14799 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14800 .await;
14801
14802 let (candidate, _candidate_rx) = wide_ctx(9);
14805 let registration = rig
14806 .handler
14807 .registry
14808 .register_candidate_with_control_ops(
14809 manifest(PLEXUS, PROTOCOL_VERSION),
14810 PROTOCOL_VERSION,
14811 candidate.connection_id,
14812 module_baseline_control_ops(),
14813 )
14814 .unwrap();
14815 rig.forwarding
14816 .register_candidate_module_connection(
14817 candidate.connection_id,
14818 PLEXUS.to_string(),
14819 PROTOCOL_VERSION,
14820 manifest_concurrency(®istration.manifest),
14821 candidate.egress.clone(),
14822 )
14823 .unwrap();
14824 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
14825 rig.handler
14826 .registry
14827 .promote_candidate(PLEXUS)
14828 .unwrap()
14829 .unwrap();
14830 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
14831
14832 rig.sync(Vec::new()).await;
14833 assert!(!rig.live(&on_incumbent));
14834 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
14835 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
14836 let goodbye = incumbent_rx
14837 .try_recv()
14838 .expect("the superseded endpoint is told")
14839 .frame;
14840 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14841 }
14842 }
14843}
14844
14845#[cfg(test)]
14846mod concurrency_default_exposure_tests {
14847 use super::*;
14848
14849 fn hello_body(role_json: &str) -> Vec<u8> {
14850 format!(
14851 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
14852 )
14853 .into_bytes()
14854 }
14855
14856 fn manifest_from(body: &[u8]) -> ModuleManifest {
14857 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
14858 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
14859 .expect("manifest parses")
14860 }
14861
14862 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
14863
14864 #[test]
14865 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
14866 let body = hello_body(&format!(
14867 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
14868 ));
14869 let manifest = manifest_from(&body);
14870 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
14873 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
14874 }
14875
14876 #[test]
14877 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
14878 let body = hello_body(&format!(
14879 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
14880 ));
14881 let manifest = manifest_from(&body);
14882 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
14883 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
14884 }
14885
14886 #[test]
14887 fn non_management_roles_are_never_reported() {
14888 let body = hello_body(
14889 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
14890 );
14891 let manifest = manifest_from(&body);
14892 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
14893 }
14894}