1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_event_declarations,
25 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
26 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
27 },
28 scope::{
29 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
30 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
31 SCOPE_SYNC_OP,
32 },
33 session::{
34 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
35 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
36 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
37 ROLE_VERSIONS_FIELD,
38 },
39 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
40 Priority, RouteTarget, PROTOCOL_VERSION,
41};
42use tokio::time::{timeout_at, Instant};
43use tracing::{debug, info, warn};
44
45use crate::{
46 caller_identity::CallerIdentity,
47 capability_requirements::{
48 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
49 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
50 RuntimeModule,
51 },
52 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
53 forwarding::{
54 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
55 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
56 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
57 },
58 observability::{
59 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
60 },
61 privileged_prompt::{prompt_text, PrivilegedOperation},
62 provenance::{
63 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
64 },
65 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
66 router::{RouteCtx, RouterError},
67 scopes::{HelloLaunchNonces, ScopeTable},
68 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
69 stderr_tail::{CaptureState, TailEntry},
70 supervise::{
71 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
72 SupervisorHandle, SwapHelloAdmission,
73 },
74 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
75};
76
77pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
83
84const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
85const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
86const CAP_PING_PONG: &str = "ping_pong_v1";
87const CAP_SESSION_ATTACH: &str = "session_attach_v1";
88const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
89
90const SUBC_CONTROL_OPS: &[&str] = &[
91 ops::ACTIVITY_RECORD,
92 ops::ACTIVITY_QUERY,
93 ops::ACTIVITY_SUBSCRIBE,
94 ops::SERVER_DESCRIBE,
95 ops::CATALOG_LIST,
96 ops::ROUTE_OPEN,
97 ops::ROUTE_POLL,
98 ops::ROUTE_CLOSING,
99 ops::ROUTE_CLOSED,
100 ops::SUPERVISOR_LIST,
101 ops::SUPERVISOR_RESTART,
102 ops::SUPERVISOR_SWAP,
103 ops::SUPERVISOR_RELOAD,
104 ops::SUPERVISOR_RESCAN,
105 ops::SUPERVISOR_RELEASE_RESERVED,
106 ops::SUPERVISOR_SET_ENABLED,
107 ops::SUPERVISOR_HEALTH_PROBE,
108 ops::SUPERVISOR_HEALTH,
109 ops::SUPERVISOR_STDERR_TAIL,
110 ops::SUPERVISOR_TERMINALS,
111 ops::SUPERVISOR_ROUTES,
112 ops::SUPERVISOR_PROVENANCE,
113 ops::SUPERVISOR_SPAWN_SNAPSHOT,
114 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
115];
116
117const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
118 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
119 "supervisor.live_roots",
120 SCOPE_SYNC_OP,
121 SCOPE_APPLY_OP,
122 SCOPE_DESCRIBE_OP,
123 "operator.confirm",
124];
125
126const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
130
131const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
132
133pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
141
142pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
157
158pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
176
177const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
178const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
179
180fn reload_verdict(
181 configured: &Path,
182 spawned_from: Option<&Path>,
183 image: subc_control::RunningImageAgreement,
184) -> PendingReloadVerdict {
185 let path = match spawned_from {
186 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
187 ReloadPathAgreement::Match
188 }
189 Some(spawned_from) => ReloadPathAgreement::Mismatch {
190 configured: configured.to_path_buf(),
191 spawned_from: spawned_from.to_path_buf(),
192 },
193 None => ReloadPathAgreement::Unavailable {
194 reason: if matches!(
195 image,
196 subc_control::RunningImageAgreement::Unavailable {
197 reason: subc_control::RunningImageUnavailableReason::NotRunning
198 }
199 ) {
200 ReloadPathUnavailableReason::NotRunning
201 } else {
202 ReloadPathUnavailableReason::SpawnedPathUnavailable
203 },
204 },
205 };
206 PendingReloadVerdict { path, image }
207}
208
209fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
210 if configured == spawned_from {
211 return true;
212 }
213 #[cfg(windows)]
214 {
215 match (
221 std::fs::canonicalize(configured),
222 std::fs::canonicalize(spawned_from),
223 ) {
224 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
225 _ => false,
226 }
227 }
228 #[cfg(not(windows))]
229 {
230 false
231 }
232}
233
234#[derive(Clone)]
235struct DaemonProvenanceFacts {
236 build: DaemonBuildProvenance,
237 pid: Option<u32>,
238 started_at_ms: Option<u64>,
239 start_clock: Option<crate::clock::StartClock>,
240 executable_path: Option<PathBuf>,
241 executable_identity: Option<SpawnedFileIdentity>,
242 process_start_time: Option<u64>,
243 probe: ExecutableIdentityProbe,
244}
245
246impl Default for DaemonProvenanceFacts {
247 fn default() -> Self {
248 Self {
249 build: DaemonBuildProvenance {
250 build_git_sha: None,
251 build_lock_digest: None,
252 },
253 pid: None,
254 started_at_ms: None,
255 start_clock: None,
256 executable_path: None,
257 executable_identity: None,
258 process_start_time: None,
259 probe: ExecutableIdentityProbe::default(),
260 }
261 }
262}
263
264#[derive(Debug, Clone)]
265struct SupervisorRescanContext {
266 supervisor: Supervisor,
267 config_path: PathBuf,
268 configured_port: Option<u16>,
269 storage_config: Option<crate::daemon_config::StorageConfig>,
270 admission_facts_carrier_module_id: Option<String>,
271 admission_facts_targets: Option<Vec<String>>,
272 scope_authority_owners: Vec<String>,
273 privileged_operator_prompt: PrivilegedOperatorPrompt,
274}
275
276const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
297 "reloading",
298 "supervisor_not_live",
299 "registration_not_active",
300 "no_forwarding_connection",
301 "relay_send_failed",
302];
303
304#[derive(Clone)]
306pub struct ControlHandler {
307 registry: Arc<Registry>,
308 activity: Option<crate::activity::ActivityLedger>,
309 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
310 forwarding: Arc<ForwardingTable>,
311 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
312 supervisor: SupervisorHandle,
313 subc_capabilities: Arc<[String]>,
314 route_bind_relay_timeout: Duration,
318 route_bind_relay_timeouts: BTreeMap<String, Duration>,
322 route_bind_breakers: RouteBindBreakers,
325 route_bind_concurrency: RouteBindConcurrency,
328 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
333 route_bind_breaker_threshold: u32,
335 route_bind_breaker_cooldown: Duration,
337 health_probe_timeout: Duration,
338 storage_config: Option<crate::daemon_config::StorageConfig>,
341 machine_id: Option<crate::machine_id::MachineId>,
345 admission_facts_carrier_module_id: Option<String>,
346 admission_facts_targets: Option<Vec<String>>,
347 scopes: Arc<RwLock<ScopeTable>>,
351 scope_deadline_added: Arc<tokio::sync::Notify>,
355 scope_authority_owners: Vec<String>,
358 privileged_operator_prompt: PrivilegedOperatorPrompt,
360 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
363 #[cfg(test)]
364 caller_lookup: Option<TestCallerLookup>,
365 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
368 rescan: Option<SupervisorRescanContext>,
369 connected_clients: ConnectedClients,
370 counters: DaemonCounters,
371 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
372 daemon_provenance: DaemonProvenanceFacts,
373 #[cfg(test)]
374 control_dispatch_delay: Option<Duration>,
375 #[cfg(test)]
376 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
377 #[cfg(test)]
381 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
382 #[cfg(test)]
388 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
389}
390
391impl fmt::Debug for ControlHandler {
392 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
393 f.debug_struct("ControlHandler")
394 .field("registry", &self.registry)
395 .field("forwarding", &self.forwarding)
396 .field("process_liveness", &self.process_liveness.is_some())
397 .field("supervisor", &self.supervisor)
398 .field("subc_capabilities", &self.subc_capabilities)
399 .finish()
400 }
401}
402
403struct RouteOpenRequest {
404 target: RouteTarget,
405 identity: BindIdentity,
406 consumer_identity: Option<ConsumerIdentity>,
407 consumer_capabilities: Option<Vec<String>>,
408 role_versions: Option<BTreeMap<String, String>>,
409 admission_facts: Option<serde_json::Value>,
410 scope: Option<ScopeSelector>,
411}
412
413#[derive(Clone, Copy)]
414enum RouteOpenPhase {
415 Admission,
416 ProjectRoot,
417 EgressReservation,
418 BindSend,
419 BindAckWait,
420}
421
422struct RouteOpenTimings {
426 module_id: String,
427 principal: String,
428 admission: Duration,
429 project_root: Duration,
430 egress_reservation: Duration,
431 bind_send: Duration,
432 bind_ack_wait: Duration,
433 phase: Option<RouteOpenPhase>,
434 phase_started_at: Instant,
435}
436
437impl RouteOpenTimings {
438 fn new() -> Self {
439 Self {
440 module_id: String::new(),
441 principal: "unresolved".to_string(),
443 admission: Duration::ZERO,
444 project_root: Duration::ZERO,
445 egress_reservation: Duration::ZERO,
446 bind_send: Duration::ZERO,
447 bind_ack_wait: Duration::ZERO,
448 phase: Some(RouteOpenPhase::Admission),
449 phase_started_at: Instant::now(),
450 }
451 }
452
453 fn finish(&mut self) {
454 let elapsed = self.phase_started_at.elapsed();
455 match self.phase.take() {
456 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
457 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
458 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
459 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
460 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
461 None => {}
462 }
463 }
464
465 fn enter(&mut self, phase: RouteOpenPhase) {
466 self.finish();
467 self.phase = Some(phase);
468 self.phase_started_at = Instant::now();
469 }
470}
471
472struct RouteBindReservationGuard {
473 forwarding: Arc<ForwardingTable>,
474 endpoint: ModuleEndpointId,
475 relay_corr: u64,
476 armed: bool,
477}
478
479struct ModuleControlRpcGuard {
480 forwarding: Arc<ForwardingTable>,
481 endpoint: ModuleEndpointId,
482 corr: u64,
483 armed: bool,
484}
485
486impl ModuleControlRpcGuard {
487 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
488 Self {
489 forwarding,
490 endpoint,
491 corr,
492 armed: true,
493 }
494 }
495
496 fn disarm(&mut self) {
497 self.armed = false;
498 }
499}
500
501impl Drop for ModuleControlRpcGuard {
502 fn drop(&mut self) {
503 if self.armed {
504 let _ = self
505 .forwarding
506 .cancel_module_control_rpc(self.endpoint, self.corr);
507 }
508 }
509}
510
511impl RouteBindReservationGuard {
512 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
513 Self {
514 forwarding,
515 endpoint,
516 relay_corr,
517 armed: true,
518 }
519 }
520
521 fn release_and_disarm(&mut self) {
522 if !self.armed {
523 return;
524 }
525 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
526 self.endpoint,
527 self.relay_corr,
528 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
529 ) {
530 send_goodbye_target_best_effort(
531 &self.forwarding.counters(),
532 &target,
533 "canceled route.bind",
534 );
535 }
536 self.armed = false;
537 }
538
539 fn disarm(&mut self) {
540 self.armed = false;
541 }
542}
543
544impl Drop for RouteBindReservationGuard {
545 fn drop(&mut self) {
546 self.release_and_disarm();
547 }
548}
549
550#[derive(Debug, Clone, Default)]
580pub(crate) struct RouteBindBreakers {
581 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
582}
583
584#[derive(Debug, Clone, Default)]
585pub(crate) struct RouteBindConcurrency {
586 modules: Arc<Mutex<HashMap<String, usize>>>,
587}
588
589struct RouteBindConcurrencyGuard {
590 concurrency: RouteBindConcurrency,
591 module_id: String,
592}
593
594impl RouteBindConcurrency {
595 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
598 let mut modules = self
599 .modules
600 .lock()
601 .expect("route.bind concurrency mutex poisoned");
602 let in_flight = modules.entry(module_id.to_string()).or_default();
603 if *in_flight >= limit {
604 return Err(*in_flight);
605 }
606 *in_flight += 1;
607 Ok(RouteBindConcurrencyGuard {
608 concurrency: self.clone(),
609 module_id: module_id.to_string(),
610 })
611 }
612}
613
614impl Drop for RouteBindConcurrencyGuard {
615 fn drop(&mut self) {
616 let mut modules = self
617 .concurrency
618 .modules
619 .lock()
620 .expect("route.bind concurrency mutex poisoned");
621 let remove = {
622 let in_flight = modules
623 .get_mut(&self.module_id)
624 .expect("admitted route.bind has a concurrency entry");
625 *in_flight -= 1;
626 *in_flight == 0
627 };
628 if remove {
629 modules.remove(&self.module_id);
630 }
631 }
632}
633
634#[derive(Debug, Default)]
635struct ModuleBreakerState {
636 consecutive_timeouts: u32,
638 cooldown_until: Option<Instant>,
641 probe_in_flight: Option<Arc<()>>,
646}
647
648enum RouteBindAdmission<'a> {
650 Admitted {
651 guard: RouteBindBreakerGuard<'a>,
652 probe: bool,
655 },
656 Refused {
657 consecutive_timeouts: u32,
658 retry_in: Duration,
662 probe_in_flight: bool,
663 },
664}
665
666struct RouteBindBreakerGuard<'a> {
673 breakers: RouteBindBreakers,
674 module_id: &'a str,
675 probe_token: Option<Arc<()>>,
676 settled: bool,
677}
678
679impl RouteBindBreakerGuard<'_> {
680 fn record_accepted(&mut self) -> bool {
684 self.settled = true;
685 self.breakers.record_accepted(self.module_id)
686 }
687
688 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
691 self.settled = true;
692 self.breakers.record_timeout(
693 self.module_id,
694 self.probe_token.as_ref(),
695 threshold,
696 cooldown,
697 )
698 }
699
700 fn record_inconclusive(&mut self) {
709 self.settled = true;
710 self.breakers
711 .record_inconclusive(self.module_id, self.probe_token.as_ref());
712 }
713}
714
715impl Drop for RouteBindBreakerGuard<'_> {
716 fn drop(&mut self) {
717 if !self.settled {
718 self.breakers
719 .record_inconclusive(self.module_id, self.probe_token.as_ref());
720 }
721 }
722}
723
724struct BreakerOpened {
728 consecutive_timeouts: u32,
729 reopened_after_probe: bool,
732}
733
734impl RouteBindBreakers {
735 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
736 self.modules
737 .lock()
738 .expect("route.bind breaker mutex poisoned")
739 }
740
741 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
744 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
745 probe: probe_token.is_some(),
746 guard: RouteBindBreakerGuard {
747 breakers: self.clone(),
748 module_id,
749 probe_token,
750 settled: false,
751 },
752 };
753
754 let mut modules = self.lock();
755 let Some(state) = modules.get_mut(module_id) else {
756 return admitted(None);
757 };
758 let Some(cooldown_until) = state.cooldown_until else {
759 return admitted(None);
760 };
761 if state.probe_in_flight.is_some() {
762 return RouteBindAdmission::Refused {
763 consecutive_timeouts: state.consecutive_timeouts,
764 retry_in: Duration::ZERO,
765 probe_in_flight: true,
766 };
767 }
768 let now = Instant::now();
769 if now < cooldown_until {
770 return RouteBindAdmission::Refused {
771 consecutive_timeouts: state.consecutive_timeouts,
772 retry_in: cooldown_until - now,
773 probe_in_flight: false,
774 };
775 }
776 let token = Arc::new(());
777 state.probe_in_flight = Some(Arc::clone(&token));
778 admitted(Some(token))
779 }
780
781 fn record_accepted(&self, module_id: &str) -> bool {
782 self.lock()
783 .remove(module_id)
784 .is_some_and(|state| state.cooldown_until.is_some())
785 }
786
787 fn record_timeout(
788 &self,
789 module_id: &str,
790 probe_token: Option<&Arc<()>>,
791 threshold: u32,
792 cooldown: Duration,
793 ) -> Option<BreakerOpened> {
794 let mut modules = self.lock();
795 let state = modules.entry(module_id.to_string()).or_default();
796 let was_open = state.cooldown_until.is_some();
797 let was_probe = Self::owns_probe(state, probe_token);
798 if was_probe {
799 state.probe_in_flight = None;
800 }
801 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
802 if state.consecutive_timeouts < threshold {
803 return None;
804 }
805 state.cooldown_until = Some(Instant::now() + cooldown);
806 Some(BreakerOpened {
807 consecutive_timeouts: state.consecutive_timeouts,
808 reopened_after_probe: was_open && was_probe,
809 })
810 }
811
812 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
813 state
817 .probe_in_flight
818 .as_ref()
819 .zip(token)
820 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
821 }
822
823 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
824 if let Some(state) = self.lock().get_mut(module_id) {
825 if Self::owns_probe(state, probe_token) {
826 state.probe_in_flight = None;
827 }
828 }
829 }
830
831 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
846 self.lock()
847 .remove(module_id)
848 .map(|state| state.consecutive_timeouts)
849 .filter(|discarded| *discarded > 0)
850 }
851
852 fn open_snapshot(&self) -> Option<serde_json::Value> {
859 let now = Instant::now();
860 let modules = self.lock();
861 let open = modules
862 .iter()
863 .filter_map(|(module_id, state)| {
864 let cooldown_until = state.cooldown_until?;
865 Some((
866 module_id.clone(),
867 serde_json::json!({
868 "consecutive_timeouts": state.consecutive_timeouts,
869 "cooldown_remaining_ms":
870 cooldown_until.saturating_duration_since(now).as_millis() as u64,
871 "probe_in_flight": state.probe_in_flight.is_some(),
872 }),
873 ))
874 })
875 .collect::<serde_json::Map<String, serde_json::Value>>();
876 (!open.is_empty()).then_some(serde_json::Value::Object(open))
877 }
878}
879
880impl ControlHandler {
881 pub fn new(registry: Arc<Registry>) -> Self {
882 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
883 }
884
885 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
886 let counters = forwarding.counters();
887 let route_bind_breakers = forwarding.route_bind_breakers();
891 let route_bind_concurrency = forwarding.route_bind_concurrency();
892 let route_outages = forwarding.route_outages();
893 Self {
894 registry,
895 activity: None,
896 activity_streams: Arc::new(Mutex::new(HashMap::new())),
897 forwarding,
898 process_liveness: None,
899 supervisor: SupervisorHandle::new(),
900 subc_capabilities: Arc::from([
901 CAP_MANIFEST_REGISTRATION.to_string(),
902 CAP_CHANNEL_LIFECYCLE.to_string(),
903 CAP_PING_PONG.to_string(),
904 CAP_SESSION_ATTACH.to_string(),
905 CAP_ADMISSION_FACTS_RELAY.to_string(),
906 CAP_SCOPES_V1.to_string(),
907 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
908 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
909 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
910 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
911 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
912 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
913 ]),
914 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
915 route_bind_relay_timeouts: BTreeMap::new(),
916 route_bind_breakers,
917 route_bind_concurrency,
918 route_outages,
919 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
920 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
921 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
922 storage_config: None,
923 machine_id: None,
924 admission_facts_carrier_module_id: None,
925 admission_facts_targets: None,
926 scopes: Arc::new(RwLock::new(ScopeTable::new(
927 crate::daemon_config::default_scope_authority_owners(),
928 ))),
929 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
930 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
931 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
932 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
933 #[cfg(test)]
934 caller_lookup: None,
935 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
936 rescan: None,
937 connected_clients: ConnectedClients::new(),
938 counters,
939 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
940 daemon_provenance: DaemonProvenanceFacts::default(),
941 #[cfg(test)]
942 control_dispatch_delay: None,
943 #[cfg(test)]
944 provenance_probe_override: None,
945 #[cfg(test)]
946 route_bind_relay_sent_hold: None,
947 #[cfg(test)]
948 hello_registry_to_forwarding_hold: None,
949 }
950 }
951
952 pub fn with_storage_config(
955 mut self,
956 storage_config: Option<crate::daemon_config::StorageConfig>,
957 ) -> Self {
958 self.storage_config = storage_config;
959 self
960 }
961
962 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
965 self.activity = Some(crate::activity::ActivityLedger::open(
966 run_dir.join("activity.jsonl"),
967 incarnation,
968 ));
969 self.supervisor.set_activity(self.activity.clone());
970 self
971 }
972
973 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
976 self.machine_id = machine_id;
977 self
978 }
979
980 pub fn with_admission_facts_config(
984 mut self,
985 carrier_module_id: Option<String>,
986 targets: Option<Vec<String>>,
987 ) -> Self {
988 self.admission_facts_carrier_module_id = carrier_module_id;
989 self.admission_facts_targets = targets;
990 self
991 }
992
993 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
997 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
998 self.scope_authority_owners = owners;
999 self
1000 }
1001
1002 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1004 self.privileged_operator_prompt = mode;
1005 self
1006 }
1007
1008 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1011 self.route_bind_relay_timeout = timeout;
1012 self
1013 }
1014
1015 pub fn with_route_bind_relay_timeouts(
1021 mut self,
1022 timeouts: impl IntoIterator<Item = (String, Duration)>,
1023 ) -> Self {
1024 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1025 self
1026 }
1027
1028 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1034 self.route_bind_relay_timeouts
1035 .get(module_id)
1036 .copied()
1037 .unwrap_or(self.route_bind_relay_timeout)
1038 }
1039
1040 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1048 self.route_bind_breaker_threshold = threshold.max(1);
1049 self.route_bind_breaker_cooldown = cooldown;
1050 self
1051 }
1052
1053 #[cfg(test)]
1054 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1055 self.health_probe_timeout = timeout;
1056 self
1057 }
1058
1059 #[cfg(test)]
1060 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1061 self.control_dispatch_delay = Some(delay);
1062 self
1063 }
1064
1065 #[cfg(test)]
1069 pub(crate) fn with_route_bind_relay_sent_hold(
1070 mut self,
1071 hold: Arc<tokio::sync::Barrier>,
1072 ) -> Self {
1073 self.route_bind_relay_sent_hold = Some(hold);
1074 self
1075 }
1076
1077 #[cfg(test)]
1082 pub(crate) fn with_hello_registry_to_forwarding_hold(
1083 mut self,
1084 hold: Arc<std::sync::Barrier>,
1085 ) -> Self {
1086 self.hello_registry_to_forwarding_hold = Some(hold);
1087 self
1088 }
1089
1090 #[cfg(test)]
1091 fn hold_between_registry_and_forwarding(&self) {
1092 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1093 hold.wait();
1094 hold.wait();
1095 }
1096 }
1097
1098 pub fn with_process_liveness(
1099 mut self,
1100 process_liveness: Arc<dyn ModuleProcessLiveness>,
1101 ) -> Self {
1102 self.process_liveness = Some(process_liveness);
1103 self
1104 }
1105
1106 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1107 self.supervisor = supervisor;
1108 self.supervisor.set_activity(self.activity.clone());
1109 self
1110 }
1111
1112 pub fn with_daemon_provenance(
1113 mut self,
1114 pid: u32,
1115 started_at_ms: u64,
1116 executable_path: Option<PathBuf>,
1117 build_git_sha: Option<String>,
1118 build_lock_digest: Option<String>,
1119 ) -> Self {
1120 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1121 let process_start_time = process_start_time(pid);
1122 self.daemon_provenance = DaemonProvenanceFacts {
1123 build: DaemonBuildProvenance {
1124 build_git_sha,
1125 build_lock_digest,
1126 },
1127 pid: Some(pid),
1128 started_at_ms: Some(started_at_ms),
1129 start_clock: None,
1130 executable_path,
1131 executable_identity,
1132 process_start_time,
1133 probe: ExecutableIdentityProbe::default(),
1134 };
1135 self
1136 }
1137
1138 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1139 self.daemon_provenance.start_clock = Some(clock);
1140 self
1141 }
1142
1143 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1145 self.scopes
1146 .write()
1147 .unwrap_or_else(|poisoned| poisoned.into_inner())
1148 .set_wall_clock(Arc::new(clock));
1149 self
1150 }
1151
1152 #[cfg(test)]
1153 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1154 self.provenance_probe_override = Some(result);
1155 self
1156 }
1157
1158 pub fn with_capability_config(
1162 self,
1163 modules: impl IntoIterator<Item = (String, bool)>,
1164 reserved_capabilities: BTreeMap<String, String>,
1165 ) -> Self {
1166 self.capability_evaluator
1167 .configure(modules, reserved_capabilities);
1168 self
1169 }
1170
1171 pub fn with_supervisor_rescan(
1172 mut self,
1173 supervisor: Supervisor,
1174 config_path: impl Into<PathBuf>,
1175 configured_port: Option<u16>,
1176 ) -> Self {
1177 self.rescan = Some(SupervisorRescanContext {
1178 supervisor,
1179 config_path: config_path.into(),
1180 configured_port,
1181 storage_config: self.storage_config.clone(),
1182 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1183 admission_facts_targets: self.admission_facts_targets.clone(),
1184 scope_authority_owners: self.scope_authority_owners.clone(),
1185 privileged_operator_prompt: self.privileged_operator_prompt,
1186 });
1187 self
1188 }
1189
1190 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1191 self.connected_clients = connected_clients;
1192 self
1193 }
1194
1195 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1196 Arc::clone(&self.forwarding)
1197 }
1198
1199 pub(crate) fn counters(&self) -> DaemonCounters {
1200 self.counters.clone()
1201 }
1202
1203 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1206 tokio::spawn(async move {
1207 loop {
1208 self.capability_evaluator
1209 .wait_for_change_or_deadline()
1210 .await;
1211 self.refresh_capability_requirements();
1212 }
1213 });
1214 }
1215
1216 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1222 tokio::spawn(async move {
1223 loop {
1224 let has_deadlines = self
1225 .scopes
1226 .read()
1227 .unwrap_or_else(|poisoned| poisoned.into_inner())
1228 .has_deadlines();
1229 if !has_deadlines {
1230 self.scope_deadline_added.notified().await;
1231 continue;
1232 }
1233 tokio::time::sleep(Duration::from_secs(1)).await;
1234 if let Err(error) = self.sweep_expired_scopes() {
1235 warn!(%error, "scope expiry sweep failed");
1236 }
1237 }
1238 });
1239 }
1240
1241 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1242 let mut table = self
1243 .scopes
1244 .write()
1245 .unwrap_or_else(|poisoned| poisoned.into_inner());
1246 let swept = table.sweep_expired();
1247 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1248 drop(table);
1249 self.close_scope_drained_routes(drained);
1250 Ok(())
1251 }
1252
1253 fn publish_scope_changes(
1254 &self,
1255 changes: &[crate::scopes::ScopeTagChange],
1256 expired: &[crate::scopes::ScopeExpired],
1257 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1258 if changes.is_empty() {
1259 return Ok(Vec::new());
1260 }
1261 let drained = self
1262 .forwarding
1263 .publish_scope_changes(changes)
1264 .map_err(RouterError::Forwarding)?;
1265 let mut counts = HashMap::new();
1266 for route in &drained {
1267 *counts
1268 .entry((
1269 route.scope.owner.as_str(),
1270 route.scope.scope_ref.as_str(),
1271 route.scope.tag.scope_epoch,
1272 ))
1273 .or_insert(0usize) += 1;
1274 }
1275 for scope in expired {
1276 let routes_closed = counts
1277 .get(&(
1278 scope.owner.as_str(),
1279 scope.scope_ref.as_str(),
1280 scope.scope_epoch,
1281 ))
1282 .copied()
1283 .unwrap_or(0);
1284 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1285 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1286 routes_closed, "scope expired");
1287 }
1288 Ok(drained)
1289 }
1290
1291 fn runtime_capability_snapshot(
1292 &self,
1293 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1294 let runtime = self
1295 .supervisor
1296 .list()
1297 .into_iter()
1298 .map(|module| {
1299 let status = module.status().map_err(|err| {
1300 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1301 })?;
1302 Ok(RuntimeModule {
1303 module_id: status.module_id,
1304 state: status.state,
1305 enabled: status.enabled,
1306 })
1307 })
1308 .collect::<Result<Vec<_>, RouterError>>()?;
1309 let registrations = self.registry.list_registered_modules().map_err(|err| {
1315 RouterError::backend(
1316 0,
1317 0,
1318 format!("failed to list capability registrations: {err}"),
1319 )
1320 })?;
1321 let registrations = registrations
1322 .into_iter()
1323 .map(|registration| RegisteredModule {
1324 module_id: registration.manifest.module_id,
1325 module_version: registration.manifest.module_version,
1326 capabilities: registration.manifest.capabilities,
1327 })
1328 .collect();
1329 Ok((runtime, registrations))
1330 }
1331
1332 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1339 let cached_registration = RegisteredModule {
1340 module_id: registration.manifest.module_id.clone(),
1341 module_version: registration.manifest.module_version.clone(),
1342 capabilities: registration.manifest.capabilities.clone(),
1343 };
1344 if self.capability_evaluator.record_hello(&cached_registration) {
1345 warn!(
1346 module_id = %cached_registration.module_id,
1347 "capability claims drifted from the cached manifest"
1348 );
1349 }
1350 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1351 self.enforce_capability_denies();
1352 }
1353 self.refresh_capability_requirements();
1354 }
1355
1356 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1360 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1361 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1362 self.supervisor.set_swap_promotion_observer(observer);
1363 }
1364
1365 pub fn refresh_capability_requirements(&self) {
1366 match self.runtime_capability_snapshot() {
1367 Ok((runtime, registrations)) => {
1368 log_requirement_events(
1369 self.capability_evaluator
1370 .evaluate_now(&runtime, ®istrations),
1371 );
1372 }
1373 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1374 }
1375 }
1376
1377 fn enforce_capability_denies(&self) {
1381 let registrations = match self.registry.list_registered_modules() {
1382 Ok(snapshot) => snapshot,
1383 Err(err) => {
1384 warn!(error = %err, "failed to read registrations for capability deny census");
1385 return;
1386 }
1387 };
1388 let manifests = registrations
1389 .into_iter()
1390 .map(|registration| {
1391 (
1392 registration.manifest.module_id.clone(),
1393 registration.manifest,
1394 )
1395 })
1396 .collect::<BTreeMap<_, _>>();
1397 let census = match self.forwarding.route_census(None) {
1398 Ok(census) => census,
1399 Err(err) => {
1400 warn!(error = %err, "failed to read route census for capability deny enforcement");
1401 return;
1402 }
1403 };
1404
1405 for (target_module_id, routes) in census {
1406 let Some(target_manifest) = manifests.get(&target_module_id) else {
1407 continue;
1408 };
1409 let mut closed_routes = Vec::new();
1410 let mut module_goodbyes = Vec::new();
1411 for route in routes {
1412 let Principal::Reserved {
1413 module_id: opening_module_id,
1414 } = &route.principal
1415 else {
1416 continue;
1417 };
1418 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1419 continue;
1420 };
1421 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1422 continue;
1423 };
1424
1425 match self.forwarding.release_client_route(
1426 route.goodbye_target.connection_id,
1427 route.goodbye_target.channel,
1428 route.goodbye_target.epoch,
1429 ) {
1430 Ok(RouteRelease::Removed(module_goodbye)) => {
1431 warn!(
1432 opening_module_id,
1433 target_module_id,
1434 capability,
1435 "force-closing route because an attested capability deny edge now matches"
1436 );
1437 closed_routes.push(route);
1438 module_goodbyes.push(module_goodbye);
1439 }
1440 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1441 Err(err) => warn!(
1442 opening_module_id,
1443 target_module_id,
1444 capability,
1445 error = %err,
1446 "failed to force-close capability-denied route"
1447 ),
1448 }
1449 }
1450
1451 if closed_routes.is_empty() {
1452 continue;
1453 }
1454 send_route_control_pushes(
1455 &self.forwarding,
1456 closed_routes,
1457 ClientControlPush::RouteClosed {
1458 module_id: target_module_id,
1459 channels: Vec::new(),
1460 reason: RouteCloseReason::CapabilityDenied,
1461 drained: false,
1462 abandoned: 0,
1463 excluded_subscriptions: 0,
1464 terminal: Some(false),
1465 },
1466 );
1467 self.emit_route_goodbyes(module_goodbyes);
1468 }
1469 }
1470
1471 fn not_ready_reason(
1477 &self,
1478 registration: &crate::registry::ModuleRegistration,
1479 ) -> Option<NotReadyReason> {
1480 if !registration.ready {
1481 return Some(NotReadyReason {
1482 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1483 capability: None,
1484 });
1485 }
1486 self.first_unprovided_required_capability(registration)
1487 .map(|capability| NotReadyReason {
1488 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1489 capability: Some(capability),
1490 })
1491 }
1492
1493 fn first_unprovided_required_capability(
1511 &self,
1512 registration: &crate::registry::ModuleRegistration,
1513 ) -> Option<String> {
1514 let required = registration
1515 .manifest
1516 .capabilities
1517 .iter()
1518 .flat_map(|declarations| declarations.requires.iter())
1519 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1520 .map(|requirement| requirement.capability.as_str())
1521 .collect::<BTreeSet<_>>();
1522 if required.is_empty() {
1523 return None;
1524 }
1525 let module_id = registration.manifest.module_id.as_str();
1526 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1527 if required
1528 .iter()
1529 .any(|capability| verdict(capability).is_none())
1530 {
1531 self.refresh_capability_requirements();
1532 }
1533 required
1534 .into_iter()
1535 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1536 .map(str::to_string)
1537 }
1538
1539 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1540 self.capability_evaluator
1541 .statuses()
1542 .into_iter()
1543 .map(capability_requirement_status)
1544 .collect()
1545 }
1546
1547 fn deregister_connection(
1554 &self,
1555 connection_id: ConnectionId,
1556 reason: RegistrationEndReason,
1557 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1558 self.registry
1559 .deregister_connection_with_reason(connection_id, reason)
1560 }
1561
1562 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1563 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1564 return None;
1565 }
1566 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1567 parse_client_control_request(&frame.body)
1568 else {
1569 return None;
1570 };
1571 Some(target_module_id(&target).to_string())
1572 }
1573
1574 pub(crate) fn route_open_capacity_refusal(
1575 &self,
1576 ctx: &RouteCtx,
1577 frame: &Frame,
1578 target_module_id: &str,
1579 in_flight: usize,
1580 limit: usize,
1581 ) -> Result<Frame, RouterError> {
1582 self.route_open_admission_refusal_frame(
1583 ctx,
1584 frame,
1585 target_module_id,
1586 "open_admission_full",
1587 (in_flight, limit),
1588 format!(
1589 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1590 ),
1591 )
1592 }
1593
1594 fn route_open_target_capacity_refusal(
1595 &self,
1596 ctx: &RouteCtx,
1597 frame: &Frame,
1598 target_module_id: &str,
1599 in_flight: usize,
1600 ) -> Result<Frame, RouterError> {
1601 self.route_open_admission_refusal_frame(
1602 ctx,
1603 frame,
1604 target_module_id,
1605 "target_binds_full",
1606 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1607 format!(
1608 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1609 ),
1610 )
1611 }
1612
1613 fn route_open_admission_refusal_frame(
1621 &self,
1622 ctx: &RouteCtx,
1623 frame: &Frame,
1624 target_module_id: &str,
1625 reason: &'static str,
1626 (in_flight, limit): (usize, usize),
1627 message: impl Into<String>,
1628 ) -> Result<Frame, RouterError> {
1629 let code = error_codes::TARGET_UNAVAILABLE;
1630 self.counters.increment_route_open_refused(code);
1631 info!(
1632 target: "control",
1633 code,
1634 reason,
1635 module_id = ?target_module_id,
1636 connection_id = ctx.connection_id.get(),
1637 in_flight,
1638 limit,
1639 "route.open refused"
1640 );
1641 control_error_frame(frame, code, message.into())
1642 }
1643
1644 #[cfg(test)]
1650 pub fn handle_control(
1651 &self,
1652 connection_id: ConnectionId,
1653 frame: Frame,
1654 ) -> Result<Vec<Frame>, RouterError> {
1655 match frame.header.ty {
1656 FrameType::Ping => Ok(vec![pong(&frame)?]),
1657 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1658 FrameType::Goodbye => self.handle_goodbye(connection_id),
1659 ty => Ok(vec![control_error_frame(
1660 &frame,
1661 "unsupported_control_frame",
1662 format!("unsupported channel-0 frame {ty:?}"),
1663 )?]),
1664 }
1665 }
1666
1667 pub async fn handle_control_frame(
1668 &self,
1669 ctx: &RouteCtx,
1670 frame: Frame,
1671 ) -> Result<Vec<Frame>, RouterError> {
1672 self.handle_control_frame_timed(ctx, frame, None).await
1673 }
1674
1675 pub(crate) fn record_caller_endpoints(
1676 &self,
1677 connection: ConnectionId,
1678 endpoints: Option<(SocketAddr, SocketAddr)>,
1679 ) {
1680 let mut callers = self
1681 .caller_endpoints
1682 .lock()
1683 .unwrap_or_else(|p| p.into_inner());
1684 callers.remove(&connection);
1685 if let Some(endpoints) = endpoints {
1686 callers.insert(connection, endpoints);
1687 }
1688 }
1689
1690 #[cfg(test)]
1691 pub(crate) fn with_caller_lookup_for_test(
1692 mut self,
1693 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1694 ) -> Self {
1695 self.caller_lookup = Some(Arc::new(lookup));
1696 self
1697 }
1698
1699 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1700 let endpoints = self
1701 .caller_endpoints
1702 .lock()
1703 .unwrap_or_else(|p| p.into_inner())
1704 .get(&connection)
1705 .copied();
1706 #[cfg(test)]
1707 let lookup = self.caller_lookup.clone();
1708 tokio::task::spawn_blocking(move || {
1709 #[cfg(test)]
1710 if let Some(lookup) = lookup {
1711 return lookup(endpoints);
1712 }
1713 endpoints
1714 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1715 .unwrap_or_default()
1716 })
1717 .await
1718 .unwrap_or_default()
1719 }
1720
1721 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1722 frame.header.channel == 0
1723 && frame.header.ty == FrameType::Request
1724 && self
1725 .forwarding
1726 .module_endpoint_for_connection(ctx.connection_id)
1727 .is_ok_and(|endpoint| endpoint.is_none())
1728 && parse_client_control_request(&frame.body)
1729 .ok()
1730 .is_some_and(|request| privileged_operation(&request).is_some())
1731 }
1732
1733 pub(crate) async fn handle_control_frame_timed(
1734 &self,
1735 ctx: &RouteCtx,
1736 frame: Frame,
1737 dispatch_started_at: Option<StdInstant>,
1738 ) -> Result<Vec<Frame>, RouterError> {
1739 match frame.header.ty {
1740 FrameType::Ping => Ok(vec![pong(&frame)?]),
1741 FrameType::Hello => {
1742 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1743 }
1744 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1745 FrameType::Cancel => {
1746 if self
1749 .forwarding
1750 .operator_confirms()
1751 .cancel(ctx.connection_id, frame.header.corr)
1752 || self
1753 .supervisor
1754 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1755 || self.cancel_activity_subscription(ctx, &frame)?
1756 {
1757 Ok(Vec::new())
1758 } else {
1759 Ok(vec![control_error_frame(
1760 &frame,
1761 "unknown_subscription",
1762 "no subscription has this correlation id",
1763 )?])
1764 }
1765 }
1766 FrameType::Request => {
1767 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1770 if op
1771 .as_ref()
1772 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1773 {
1774 return self.handle_activity_record(ctx, frame).await;
1775 }
1776 if op.as_ref().is_some_and(|probe| {
1777 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1778 }) {
1779 return self.handle_activity_read(ctx, frame).await;
1780 }
1781 if op
1782 .as_ref()
1783 .is_some_and(|probe| probe.op == "operator.confirm")
1784 {
1785 return self.handle_operator_confirm(ctx, frame);
1786 }
1787 if self
1788 .forwarding
1789 .module_endpoint_for_connection(ctx.connection_id)
1790 .map_err(RouterError::Forwarding)?
1791 .is_some()
1792 {
1793 if !is_known_module_request_op(&frame.body) {
1794 return Ok(vec![control_error_frame(
1795 &frame,
1796 "unsupported_control_frame",
1797 "module-originated channel-0 REQUEST is not supported",
1798 )?]);
1799 }
1800 let request = match parse_module_control_request_from_module(&frame.body) {
1801 Ok(request) => request,
1802 Err((err, ControlRequestBodyError::UnknownOp)) => {
1803 return Ok(vec![control_error_frame(
1804 &frame,
1805 "unsupported_control_frame",
1806 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1807 )?])
1808 }
1809 Err((err, ControlRequestBodyError::InvalidBody)) => {
1810 return Ok(vec![control_error_frame(
1811 &frame,
1812 "invalid_control_body",
1813 format!("malformed module control body: {err}"),
1814 )?])
1815 }
1816 };
1817 let op = module_control_request_op(&request);
1818 let corr = frame.header.corr;
1819 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1820 let result =
1821 self.handle_module_control_request(ctx.connection_id, frame, request);
1822 log_slow_control_dispatch(
1823 dispatch_started_at,
1824 op,
1825 ctx.connection_id,
1826 corr,
1827 None,
1828 );
1829 return result;
1830 }
1831
1832 if is_known_module_request_op(&frame.body) {
1833 return Ok(vec![control_error_frame(
1834 &frame,
1835 "not_registered",
1836 "catalog.update requires an active module registration owned by this connection",
1837 )?]);
1838 }
1839
1840 let request = match parse_client_control_request(&frame.body) {
1841 Ok(request) => request,
1842 Err((err, ControlRequestBodyError::UnknownOp)) => {
1843 return Ok(vec![control_error_frame(
1844 &frame,
1845 "unknown_control_op",
1846 format!("unknown client control op: {err}"),
1847 )?])
1848 }
1849 Err((err, ControlRequestBodyError::InvalidBody)) => {
1850 return Ok(vec![control_error_frame(
1851 &frame,
1852 "invalid_control_body",
1853 format!("malformed client control body: {err}"),
1854 )?])
1855 }
1856 };
1857 let op = client_control_request_op(&request);
1858 let corr = frame.header.corr;
1859 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1860 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1861 #[cfg(test)]
1862 if let Some(delay) = self.control_dispatch_delay {
1863 tokio::time::sleep(delay).await;
1864 }
1865 let result = self
1866 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1867 .await;
1868 if let Some(timings) = route_open_timings.as_mut() {
1869 timings.finish();
1870 }
1871 log_slow_control_dispatch(
1872 dispatch_started_at,
1873 op,
1874 ctx.connection_id,
1875 corr,
1876 route_open_timings.as_ref(),
1877 );
1878 result
1879 }
1880 FrameType::Push => {
1881 let Some(endpoint) = self
1882 .forwarding
1883 .module_endpoint_for_connection(ctx.connection_id)
1884 .map_err(RouterError::Forwarding)?
1885 else {
1886 return Ok(vec![control_error_frame(
1887 &frame,
1888 "unsupported_control_frame",
1889 "client-originated channel-0 PUSH is not supported",
1890 )?]);
1891 };
1892 self.handle_status_update(endpoint, frame)
1893 }
1894 FrameType::Response | FrameType::Error
1895 if self
1896 .forwarding
1897 .module_endpoint_for_connection(ctx.connection_id)
1898 .map_err(RouterError::Forwarding)?
1899 .is_some() =>
1900 {
1901 self.handle_module_relay_response(ctx.connection_id, frame)
1902 }
1903 ty => Ok(vec![control_error_frame(
1904 &frame,
1905 "unsupported_control_frame",
1906 format!("unsupported channel-0 frame {ty:?}"),
1907 )?]),
1908 }
1909 }
1910
1911 pub fn cleanup_connection(
1912 &self,
1913 connection_id: ConnectionId,
1914 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1915 self.caller_endpoints
1916 .lock()
1917 .unwrap_or_else(|p| p.into_inner())
1918 .remove(&connection_id);
1919 self.forwarding
1920 .operator_confirms()
1921 .connection_closed(connection_id);
1922 self.cleanup_connection_with_end_reason(
1923 connection_id,
1924 RegistrationEndReason::ConnectionClosed,
1925 )
1926 }
1927
1928 fn cleanup_connection_with_end_reason(
1929 &self,
1930 connection_id: ConnectionId,
1931 requested_reason: RegistrationEndReason,
1932 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1933 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1934 self.registry
1935 .get_module_by_connection(connection_id)?
1936 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
1937 .and_then(|module| module.registration_end_reason().ok().flatten())
1938 .unwrap_or(requested_reason)
1939 } else {
1940 requested_reason
1941 };
1942 let crash_closed = self
1943 .registry
1944 .get_module_by_connection(connection_id)?
1945 .and_then(|registration| {
1946 self.forwarding
1947 .module_endpoint_for_connection(connection_id)
1948 .ok()
1949 .flatten()
1950 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1951 .map(|routes| (registration.manifest.module_id, routes))
1952 });
1953 let crash_closed = crash_closed.map(|(module_id, routes)| {
1954 let terminal = match self.supervisor.get(&module_id) {
1955 None => false,
1956 Some(module) => match module.will_recover_after_connection_loss() {
1957 Ok(will_recover) => !will_recover,
1958 Err(err) => {
1959 warn!(
1960 %module_id,
1961 error = %err,
1962 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1963 );
1964 false
1965 }
1966 },
1967 };
1968 let (reason, drained) = match self
1972 .forwarding
1973 .daemon_drain_result_for_connection(connection_id)
1974 {
1975 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
1976 Ok(None) => (RouteCloseReason::Crash, false),
1977 Err(err) => {
1978 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1979 (RouteCloseReason::Crash, false)
1980 }
1981 };
1982 (module_id, routes, reason, drained, terminal)
1983 });
1984 let registrations = self.deregister_connection(connection_id, end_reason);
1985 let cleanup = if crash_closed.is_some() {
1986 self.forwarding.cleanup_connection_counted(connection_id)
1987 } else {
1988 self.forwarding
1994 .cleanup_connection(connection_id)
1995 .map(|released| crate::forwarding::ConnectionCleanup {
1996 released,
1997 abandoned_relays: 0,
1998 })
1999 };
2000 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2005 let abandoned = cleanup
2006 .as_ref()
2007 .map_or(0, |cleanup| cleanup.abandoned_relays);
2008 send_route_control_pushes(
2009 &self.forwarding,
2010 routes,
2011 ClientControlPush::RouteClosed {
2012 module_id,
2013 channels: Vec::new(),
2014 reason,
2015 drained,
2016 abandoned,
2017 excluded_subscriptions: 0,
2018 terminal: Some(terminal),
2019 },
2020 );
2021 }
2022 if let Ok(cleanup) = cleanup {
2023 self.emit_route_goodbyes(cleanup.released);
2024 }
2025 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2029 crate::supervise::notify_registration_release();
2030 self.capability_evaluator.wake_deadline_loop();
2031 self.refresh_capability_requirements();
2032 }
2033 self.supervisor.remove_spawn_subscribers(connection_id);
2034 self.activity_streams
2035 .lock()
2036 .unwrap_or_else(|p| p.into_inner())
2037 .retain(|(id, _), task| {
2038 if *id == connection_id {
2039 task.abort();
2040 false
2041 } else {
2042 true
2043 }
2044 });
2045 self.hello_launch_nonces
2048 .lock()
2049 .unwrap_or_else(|poisoned| poisoned.into_inner())
2050 .forget(connection_id);
2051 self.scopes
2052 .write()
2053 .unwrap_or_else(|poisoned| poisoned.into_inner())
2054 .release_connection(connection_id);
2055 registrations
2056 }
2057
2058 pub(crate) fn handle_route_goodbye(
2059 &self,
2060 connection_id: ConnectionId,
2061 route_channel: u16,
2062 route_epoch: u32,
2063 ) -> Result<bool, RouterError> {
2064 debug!(
2065 connection_id = connection_id.get(),
2066 route_channel, route_epoch, "handling route GOODBYE"
2067 );
2068 let RouteRelease::Removed(released_route) = self
2069 .forwarding
2070 .release_client_route(connection_id, route_channel, route_epoch)
2071 .map_err(RouterError::Forwarding)?
2072 else {
2073 return Ok(false);
2074 };
2075 self.emit_route_goodbyes(vec![released_route]);
2076 Ok(true)
2077 }
2078
2079 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2080 for released in released_routes {
2081 let frame = match Frame::build_with_version(
2082 released.negotiated_ver,
2083 FrameType::Goodbye,
2084 control_flags(),
2085 released.channel,
2086 released.epoch,
2087 0,
2088 Vec::new(),
2089 ) {
2090 Ok(frame) => frame,
2091 Err(err) => {
2092 warn!(
2093 route_channel = released.channel,
2094 error = %err,
2095 "failed to build route GOODBYE frame"
2096 );
2097 continue;
2098 }
2099 };
2100 if !released.close_on_delivery_failure() {
2101 crate::forwarding::send_module_route_goodbye(
2102 &self.counters,
2103 &released.sink,
2104 frame,
2105 released.module_id.as_deref(),
2106 "client route released",
2107 );
2108 continue;
2109 }
2110 if let Err(err) = released.sink.try_send(frame) {
2111 warn!(
2112 target_connection_id = released.connection_id.get(),
2113 route_channel = released.channel,
2114 error = %err,
2115 "route GOODBYE was not delivered to client; closing target connection"
2116 );
2117 if self
2118 .forwarding
2119 .escalate_client_delivery_failure(
2120 released.connection_id,
2121 released.channel,
2122 released.epoch,
2123 CloseReason::new(
2124 "route_goodbye_delivery_failed",
2125 format!(
2126 "failed to enqueue route GOODBYE for channel {}: {err}",
2127 released.channel
2128 ),
2129 ),
2130 crate::forwarding::UndeliveredFrame {
2131 module_id: released.module_id.as_deref(),
2132 sink: &released.sink,
2133 },
2134 )
2135 .unwrap_or(false)
2136 {
2137 self.counters.increment_goodbye_relay_client_failed();
2138 }
2139 }
2140 }
2141 }
2142
2143 fn send_abandoned_route_bind_goodbye(
2155 &self,
2156 module_sink: &crate::FrameSink,
2157 negotiated_ver: u8,
2158 module_channel: u16,
2159 module_epoch: u32,
2160 ) {
2161 let frame = match Frame::build_with_version(
2162 negotiated_ver,
2163 FrameType::Goodbye,
2164 control_flags(),
2165 module_channel,
2166 module_epoch,
2167 0,
2168 Vec::new(),
2169 ) {
2170 Ok(frame) => frame,
2171 Err(err) => {
2172 warn!(
2173 route_channel = module_channel,
2174 error = %err,
2175 "failed to build GOODBYE for abandoned route.bind"
2176 );
2177 return;
2178 }
2179 };
2180 crate::forwarding::send_module_route_goodbye(
2181 &self.counters,
2182 module_sink,
2183 frame,
2184 None,
2185 "abandoned route.bind",
2186 );
2187 }
2188
2189 fn handle_hello(
2190 &self,
2191 connection_id: ConnectionId,
2192 sink: Option<crate::FrameSink>,
2193 frame: Frame,
2194 ) -> Result<Vec<Frame>, RouterError> {
2195 debug!(
2196 connection_id = connection_id.get(),
2197 corr = frame.header.corr,
2198 "handling HELLO"
2199 );
2200 if self
2204 .registry
2205 .get_module_by_connection(connection_id)
2206 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2207 .is_some()
2208 {
2209 return Ok(vec![control_error_frame(
2210 &frame,
2211 "invalid_hello",
2212 "connection is already registered as a module",
2213 )?]);
2214 }
2215 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2216 Ok(value) => value,
2217 Err(err) => {
2218 return Ok(vec![control_error_frame(
2219 &frame,
2220 "invalid_hello",
2221 format!("malformed HELLO body: {err}"),
2222 )?])
2223 }
2224 };
2225 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2226 return Ok(vec![control_error_frame(
2227 &frame,
2228 "invalid_capability_grammar",
2229 err.to_string(),
2230 )?]);
2231 }
2232 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2233 return Ok(vec![control_error_frame(
2234 &frame,
2235 "invalid_manifest",
2236 err.to_string(),
2237 )?]);
2238 }
2239 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2240 return Ok(vec![control_error_frame(
2241 &frame,
2242 "invalid_event_declaration",
2243 err.to_string(),
2244 )?]);
2245 }
2246 if let Some(provenance) = hello_value
2247 .get("manifest")
2248 .and_then(|manifest| manifest.get("provenance"))
2249 {
2250 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2251 return Ok(vec![control_error_frame(
2252 &frame,
2253 "invalid_manifest",
2254 format!("malformed manifest provenance: {err}"),
2255 )?]);
2256 }
2257 }
2258 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2259 Ok(hello) => hello,
2260 Err(err) => {
2261 return Ok(vec![control_error_frame(
2262 &frame,
2263 "invalid_hello",
2264 format!("malformed HELLO body: {err}"),
2265 )?])
2266 }
2267 };
2268
2269 if hello.protocol_ver != hello.manifest.protocol_ver {
2270 return Ok(vec![control_error_frame(
2271 &frame,
2272 "invalid_manifest",
2273 format!(
2274 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2275 hello.protocol_ver, hello.manifest.protocol_ver
2276 ),
2277 )?]);
2278 }
2279
2280 if hello.manifest.module_id.trim().is_empty() {
2281 return Ok(vec![control_error_frame(
2282 &frame,
2283 "invalid_manifest",
2284 "manifest module_id must not be empty",
2285 )?]);
2286 }
2287
2288 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2289 Ok(negotiated_ver) => negotiated_ver,
2290 Err(message) => {
2291 return Ok(vec![control_error_frame(
2292 &frame,
2293 "version_unsupported",
2294 message,
2295 )?])
2296 }
2297 };
2298
2299 let swap_admission = self
2307 .supervisor
2308 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2309 if swap_admission == SwapHelloAdmission::Refused {
2310 warn!(
2311 module_id = %hello.manifest.module_id,
2312 connection_id = connection_id.get(),
2313 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2314 );
2315 return Ok(vec![control_error_frame(
2316 &frame,
2317 "swap_token_invalid",
2318 format!(
2319 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2320 hello.manifest.module_id
2321 ),
2322 )?]);
2323 }
2324 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2325
2326 if let Some(rejection) = (!swap_candidate)
2334 .then(|| {
2335 self.supervisor.reserved_hello_rejection(
2336 &hello.manifest.module_id,
2337 hello.launch_nonce.as_deref(),
2338 )
2339 })
2340 .flatten()
2341 {
2342 let message = match rejection {
2343 ReservedHelloRejection::Exact { module_id } => format!(
2344 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2345 ),
2346 ReservedHelloRejection::Prefix {
2347 prefix,
2348 owner_module_id,
2349 } => format!(
2350 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2351 hello.manifest.module_id
2352 ),
2353 };
2354 return Ok(vec![control_error_frame(
2355 &frame,
2356 "reserved_module",
2357 message,
2358 )?]);
2359 }
2360
2361 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2362 &hello.manifest.module_id,
2363 hello.manifest.capabilities.as_ref(),
2364 );
2365 if let Some(refusal) = reserved_capability_refusals.first() {
2366 let capability = refusal.capability.clone();
2367 let bound_module = refusal.claimants[0].clone();
2368 log_duplicate_claim_events(reserved_capability_refusals);
2369 return Ok(vec![control_error_frame(
2370 &frame,
2371 "reserved_capability",
2372 format!(
2373 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2374 capability, bound_module, hello.manifest.module_id
2375 ),
2376 )?]);
2377 }
2378
2379 if self
2382 .forwarding
2383 .connection_has_client_routes(connection_id)
2384 .map_err(RouterError::Forwarding)?
2385 {
2386 return Ok(vec![control_error_frame(
2387 &frame,
2388 "invalid_hello",
2389 "connection has open client routes and cannot also register as a module",
2390 )?]);
2391 }
2392
2393 self.hello_launch_nonces
2398 .lock()
2399 .unwrap_or_else(|poisoned| poisoned.into_inner())
2400 .record(connection_id, hello.launch_nonce.as_deref());
2401 let control_ops = effective_module_control_ops(hello.control_ops);
2402 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2405 if swap_candidate {
2406 return self.register_swap_candidate(
2407 connection_id,
2408 sink,
2409 &frame,
2410 hello.manifest,
2411 negotiated_ver,
2412 control_ops,
2413 hello_ack,
2414 );
2415 }
2416 let registration = match self.registry.register_unpublished_with_control_ops(
2423 hello.manifest,
2424 negotiated_ver,
2425 connection_id,
2426 control_ops,
2427 ) {
2428 Ok(registration) => registration,
2429 Err(RegistryError::DuplicateModuleId { module_id }) => {
2430 return Ok(vec![control_error_frame(
2431 &frame,
2432 "duplicate_module_id",
2433 format!(
2434 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2435 ),
2436 )?])
2437 }
2438 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2439 return Ok(vec![control_error_frame(
2440 &frame,
2441 "invalid_module_id",
2442 err.to_string(),
2443 )?])
2444 }
2445 Err(err) => {
2446 return Ok(vec![control_error_frame(
2447 &frame,
2448 "registry_error",
2449 err.to_string(),
2450 )?])
2451 }
2452 };
2453 #[cfg(test)]
2454 self.hold_between_registry_and_forwarding();
2455
2456 let reply = if let Some(sink) = sink {
2457 let concurrency = manifest_concurrency(®istration.manifest);
2473 if let Err(err) = self.forwarding.register_module_connection_acked(
2474 connection_id,
2475 registration.manifest.module_id.clone(),
2476 negotiated_ver,
2477 concurrency,
2478 sink,
2479 hello_ack,
2480 ) {
2481 if matches!(
2486 self.deregister_connection(
2487 connection_id,
2488 RegistrationEndReason::RegistrationFailed,
2489 ),
2490 Ok(r) if !r.is_empty()
2491 ) {
2492 crate::supervise::notify_registration_release();
2493 }
2494 return Ok(vec![control_error_frame(
2495 &frame,
2496 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2497 "invalid_hello"
2498 } else {
2499 forwarding_error_code(&err)
2500 },
2501 err.to_string(),
2502 )?]);
2503 }
2504 Vec::new()
2505 } else {
2506 vec![hello_ack]
2509 };
2510
2511 if !self
2516 .registry
2517 .publish_registration(connection_id)
2518 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2519 {
2520 debug!(
2521 module_id = %registration.manifest.module_id,
2522 connection_id = connection_id.get(),
2523 "registration was removed before it could be published"
2524 );
2525 }
2526
2527 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2536 info!(
2537 module_id = %registration.manifest.module_id,
2538 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2539 );
2540 }
2541
2542 self.apply_registration_capabilities(®istration);
2543
2544 self.supervisor
2545 .record_activity_registration(®istration.manifest.module_id);
2546
2547 info!(
2548 module_id = %registration.manifest.module_id,
2549 module_version = %registration.manifest.module_version,
2550 negotiated_ver,
2551 routable_provider = manifest_provides_routable_role(®istration.manifest),
2552 connection_id = connection_id.get(),
2553 "module registered"
2554 );
2555
2556 Ok(reply)
2557 }
2558
2559 #[allow(clippy::too_many_arguments)]
2576 fn register_swap_candidate(
2577 &self,
2578 connection_id: ConnectionId,
2579 sink: Option<crate::FrameSink>,
2580 frame: &Frame,
2581 manifest: ModuleManifest,
2582 negotiated_ver: u8,
2583 control_ops: Vec<String>,
2584 hello_ack: Frame,
2585 ) -> Result<Vec<Frame>, RouterError> {
2586 let module_id = manifest.module_id.clone();
2587 let registration = match self.registry.register_unpublished_candidate_with_control_ops(
2588 manifest,
2589 negotiated_ver,
2590 connection_id,
2591 control_ops,
2592 ) {
2593 Ok(registration) => registration,
2594 Err(RegistryError::DuplicateModuleId { module_id }) => {
2595 return Ok(vec![control_error_frame(
2596 frame,
2597 "duplicate_module_id",
2598 format!(
2599 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2600 ),
2601 )?])
2602 }
2603 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2604 return Ok(vec![control_error_frame(
2605 frame,
2606 "invalid_module_id",
2607 err.to_string(),
2608 )?])
2609 }
2610 Err(err) => {
2611 return Ok(vec![control_error_frame(
2612 frame,
2613 "registry_error",
2614 err.to_string(),
2615 )?])
2616 }
2617 };
2618 #[cfg(test)]
2619 self.hold_between_registry_and_forwarding();
2620 let reply = if let Some(sink) = sink {
2621 let concurrency = manifest_concurrency(®istration.manifest);
2625 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2626 connection_id,
2627 module_id.clone(),
2628 negotiated_ver,
2629 concurrency,
2630 sink,
2631 hello_ack,
2632 ) {
2633 if matches!(
2634 self.deregister_connection(
2635 connection_id,
2636 RegistrationEndReason::RegistrationFailed,
2637 ),
2638 Ok(r) if !r.is_empty()
2639 ) {
2640 crate::supervise::notify_registration_release();
2641 }
2642 return Ok(vec![control_error_frame(
2643 frame,
2644 forwarding_error_code(&err),
2645 err.to_string(),
2646 )?]);
2647 }
2648 Vec::new()
2649 } else {
2650 vec![hello_ack]
2651 };
2652 if !self
2653 .registry
2654 .publish_registration(connection_id)
2655 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2656 {
2657 debug!(
2658 module_id = %module_id,
2659 connection_id = connection_id.get(),
2660 "swap candidate registration was removed before it could be published"
2661 );
2662 }
2663 self.supervisor.mark_swap_candidate_admitted(&module_id);
2664 info!(
2665 module_id = %module_id,
2666 module_version = %registration.manifest.module_version,
2667 negotiated_ver,
2668 ready = registration.ready,
2669 connection_id = connection_id.get(),
2670 "swap candidate registered; not routable until cutover"
2671 );
2672 Ok(reply)
2673 }
2674
2675 fn build_hello_ack(
2676 &self,
2677 frame: &Frame,
2678 negotiated_ver: u8,
2679 module_id: &str,
2680 ) -> Result<Frame, RouterError> {
2681 let ack = ModuleHelloAckBody {
2682 negotiated_ver,
2683 subc_ops: module_subc_ops(),
2684 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2685 storage: self
2686 .storage_config
2687 .as_ref()
2688 .map(|cfg| cfg.descriptor_for(module_id)),
2689 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2690 };
2691 let body = serde_json::to_vec(&ack).map_err(|err| {
2692 RouterError::backend(
2693 0,
2694 frame.header.corr,
2695 format!("failed to encode HELLO_ACK: {err}"),
2696 )
2697 })?;
2698
2699 Frame::build_with_version(
2700 negotiated_ver,
2701 FrameType::HelloAck,
2702 control_flags(),
2703 0,
2704 0,
2705 frame.header.corr,
2706 body,
2707 )
2708 .map_err(RouterError::FrameBuild)
2709 }
2710
2711 async fn handle_client_control_request(
2712 &self,
2713 ctx: &RouteCtx,
2714 frame: Frame,
2715 request: ClientControlRequest,
2716 route_open_timings: &mut Option<RouteOpenTimings>,
2717 ) -> Result<Vec<Frame>, RouterError> {
2718 if let Some(operation) = privileged_operation(&request) {
2719 let caller = self.resolve_caller(ctx.connection_id).await;
2720 let op = client_control_request_op(&request);
2721 let text = prompt_text(operation, &caller);
2722 match self.privileged_operator_prompt {
2723 PrivilegedOperatorPrompt::Log => {
2724 info!(target: "subc_daemon::privileged_prompt", operation = op,
2725 target = ?operation, caller = %caller.description(), summary = %text,
2726 "privileged request would prompt the operator");
2727 }
2728 PrivilegedOperatorPrompt::Enforce => {
2729 let key =
2730 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2731 let outcome = self
2732 .forwarding
2733 .operator_confirms()
2734 .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2735 .await;
2736 if outcome.code != "confirmed" {
2737 return Ok(vec![outcome.frame(&frame)]);
2738 }
2739 }
2740 }
2741 }
2742 match request {
2743 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2744 ClientControlRequest::CatalogList { module_id } => {
2745 self.handle_catalog_list(frame, module_id)
2746 }
2747 ClientControlRequest::RouteOpen {
2748 target,
2749 identity,
2750 consumer_identity,
2751 consumer_capabilities,
2752 role_versions,
2753 admission_facts,
2754 scope,
2755 } => {
2756 self.handle_route_open(
2757 ctx,
2758 frame,
2759 RouteOpenRequest {
2760 target,
2761 identity,
2762 consumer_identity,
2763 consumer_capabilities,
2764 role_versions,
2765 admission_facts,
2766 scope,
2767 },
2768 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2769 )
2770 .await
2771 }
2772 ClientControlRequest::RoutePoll {
2773 route_channel,
2774 route_epoch,
2775 kind,
2776 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2777 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2778 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2779 self.handle_supervisor_spawn_snapshot(frame)
2780 }
2781 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2782 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2783 }
2784 ClientControlRequest::SupervisorRestart {
2785 module_id,
2786 drain_timeout_ms,
2787 } => {
2788 self.log_supervisor_request_received(
2789 ctx,
2790 frame.header.corr,
2791 ops::SUPERVISOR_RESTART,
2792 Some(&module_id),
2793 None,
2794 )?;
2795 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2796 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2797 .await
2798 }
2799 ClientControlRequest::SupervisorSwap {
2800 module_id,
2801 ready_timeout_ms,
2802 } => {
2803 self.log_supervisor_request_received(
2804 ctx,
2805 frame.header.corr,
2806 ops::SUPERVISOR_SWAP,
2807 Some(&module_id),
2808 None,
2809 )?;
2810 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2811 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2812 .await
2813 }
2814 ClientControlRequest::SupervisorReload { module_id } => {
2815 self.log_supervisor_request_received(
2816 ctx,
2817 frame.header.corr,
2818 ops::SUPERVISOR_RELOAD,
2819 Some(&module_id),
2820 None,
2821 )?;
2822 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2823 self.handle_supervisor_reload(frame, module_id, requester)
2824 .await
2825 }
2826 ClientControlRequest::SupervisorRescan { preview } => {
2827 if !preview {
2828 self.log_supervisor_request_received(
2829 ctx,
2830 frame.header.corr,
2831 ops::SUPERVISOR_RESCAN,
2832 None,
2833 None,
2834 )?;
2835 }
2836 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2837 self.handle_supervisor_rescan(frame, preview, requester)
2838 .await
2839 }
2840 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2841 self.log_supervisor_request_received(
2842 ctx,
2843 frame.header.corr,
2844 ops::SUPERVISOR_RELEASE_RESERVED,
2845 Some(&module_id),
2846 None,
2847 )?;
2848 self.handle_supervisor_release_reserved(frame, module_id)
2849 .await
2850 }
2851 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2852 self.log_supervisor_request_received(
2853 ctx,
2854 frame.header.corr,
2855 ops::SUPERVISOR_SET_ENABLED,
2856 Some(&module_id),
2857 Some(enabled),
2858 )?;
2859 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2860 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2861 .await
2862 }
2863 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2864 self.handle_supervisor_health_probe(frame, module_id).await
2865 }
2866 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2867 ClientControlRequest::SupervisorRoutes { module_id } => {
2868 self.handle_supervisor_routes(frame, module_id)
2869 }
2870 ClientControlRequest::SupervisorProvenance { module_id } => {
2871 self.handle_supervisor_provenance(frame, module_id).await
2872 }
2873 ClientControlRequest::SupervisorStderrTail {
2874 module_id,
2875 max_lines,
2876 max_bytes,
2877 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2878 ClientControlRequest::SupervisorTerminals { module_id } => {
2879 self.handle_supervisor_terminals(frame, module_id).await
2880 }
2881 }
2882 }
2883
2884 fn log_supervisor_request_received(
2885 &self,
2886 ctx: &RouteCtx,
2887 corr: u64,
2888 op: &'static str,
2889 module_id: Option<&str>,
2890 enabled: Option<bool>,
2891 ) -> Result<(), RouterError> {
2892 let caller = self.supervisor_requester(ctx, corr)?;
2893 let caller = principal_label(&caller);
2894 info!(
2897 target: "control",
2898 op,
2899 module_id,
2900 enabled,
2901 connection_id = ctx.connection_id.get(),
2902 caller = %caller,
2903 "supervisor request received"
2904 );
2905 Ok(())
2906 }
2907
2908 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
2909 Ok(self
2910 .registry
2911 .get_module_by_connection(ctx.connection_id)
2912 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2913 .map(|registration| Principal::Reserved {
2914 module_id: registration.manifest.module_id,
2915 })
2916 .unwrap_or(Principal::Direct))
2917 }
2918
2919 fn handle_module_control_request(
2920 &self,
2921 connection_id: ConnectionId,
2922 frame: Frame,
2923 request: ModuleControlRequestFromModule,
2924 ) -> Result<Vec<Frame>, RouterError> {
2925 match request {
2926 ModuleControlRequestFromModule::CatalogUpdate {
2927 provides,
2928 capabilities,
2929 ready,
2930 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2931 ModuleControlRequestFromModule::LiveRoots {} => {
2932 let registered = self
2933 .registry
2934 .get_module_by_connection(connection_id)
2935 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2936 let Some(registration) = registered else {
2937 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2938 };
2939 let response = self
2940 .forwarding
2941 .live_roots(®istration.manifest.module_id)
2942 .map_err(RouterError::Forwarding)?;
2943 Ok(vec![control_response_body_frame(
2944 &frame,
2945 &response,
2946 "ModuleControlResponseToModule::LiveRoots",
2947 )?])
2948 }
2949 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2950 self.handle_scope_sync(connection_id, frame, generation, scopes)
2951 }
2952 ModuleControlRequestFromModule::ScopeApply {
2953 generation,
2954 upsert,
2955 end,
2956 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
2957 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2958 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2959 }
2960 }
2961 }
2962
2963 fn handle_operator_confirm(
2964 &self,
2965 ctx: &RouteCtx,
2966 frame: Frame,
2967 ) -> Result<Vec<Frame>, RouterError> {
2968 use crate::operator_confirm::{audit, Outcome};
2969 let registration = self
2970 .registry
2971 .get_module_by_connection(ctx.connection_id)
2972 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2973 let Some(registration) = registration else {
2974 let outcome = Outcome::refusal("not_registered");
2975 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2976 return Ok(vec![outcome.frame(&frame)]);
2977 };
2978 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
2979 Ok(request) => request,
2980 Err(_) => {
2981 let outcome = Outcome::refusal("invalid_control_body");
2982 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2983 return Ok(vec![outcome.frame(&frame)]);
2984 }
2985 };
2986 let module_id = registration.manifest.module_id;
2987 let nonce = self
2991 .hello_launch_nonces
2992 .lock()
2993 .unwrap_or_else(|p| p.into_inner())
2994 .nonce(ctx.connection_id)
2995 .map(str::to_owned);
2996 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
2997 self.supervisor
2998 .spawned_consumer_authorized(&module_id, nonce)
2999 });
3000 let confirms = self.forwarding.operator_confirms();
3001 self.forwarding
3002 .with_operator_route(
3003 ctx.connection_id,
3004 request.route_channel,
3005 request.route_epoch,
3006 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3007 )
3008 .map_err(RouterError::Forwarding)
3009 }
3010
3011 fn handle_scope_sync(
3015 &self,
3016 connection_id: ConnectionId,
3017 frame: Frame,
3018 generation: u64,
3019 scopes: Vec<ScopeRecord>,
3020 ) -> Result<Vec<Frame>, RouterError> {
3021 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3022 }
3023
3024 fn handle_scope_change(
3025 &self,
3026 connection_id: ConnectionId,
3027 frame: Frame,
3028 generation: u64,
3029 scopes: Vec<ScopeRecord>,
3030 end: Option<Vec<ScopeEnd>>,
3031 ) -> Result<Vec<Frame>, RouterError> {
3032 let is_apply = end.is_some();
3033 let Some(registration) = self
3034 .registry
3035 .get_module_by_connection(connection_id)
3036 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3037 else {
3038 return Ok(vec![control_error_frame(
3039 &frame,
3040 "not_registered",
3041 "scope.sync and scope.apply require an active module registration owned by this connection",
3042 )?]);
3043 };
3044 let owner = registration.manifest.module_id;
3045 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3046 let is_current_launch = |connection: ConnectionId| {
3047 self.hello_launch_nonces
3048 .lock()
3049 .unwrap_or_else(|poisoned| poisoned.into_inner())
3050 .presented(connection, current_nonce.as_deref())
3051 };
3052 let mut table = self
3057 .scopes
3058 .write()
3059 .unwrap_or_else(|poisoned| poisoned.into_inner());
3060 let outcome = match end {
3061 Some(end) => table.apply(
3062 &owner,
3063 connection_id,
3064 is_current_launch,
3065 generation,
3066 scopes,
3067 end,
3068 ),
3069 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3070 };
3071 let drained = match &outcome {
3072 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3073 Err(_) => Vec::new(),
3074 };
3075 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3076 drop(table);
3077 if has_deadlines {
3078 self.scope_deadline_added.notify_one();
3079 }
3080 match outcome {
3081 Ok(applied) => {
3082 let counts = ScopeOutcomeCounts::of(&applied.results);
3083 info!(
3084 owner = %owner,
3085 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3086 generation,
3087 records = applied.results.len(),
3088 created = counts.created,
3089 replaced = counts.replaced,
3090 updated = counts.updated,
3091 unchanged = counts.unchanged,
3092 refused = counts.refused,
3093 ended = applied.ended.len(),
3094 tag_changes = applied.tag_changes.len(),
3095 routes_closed = drained.len(),
3096 "scope change accepted"
3097 );
3098 for refused in applied
3104 .results
3105 .iter()
3106 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3107 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3108 {
3109 warn!(
3110 owner = %owner,
3111 generation,
3112 scope_ref = %refused.scope_ref,
3113 scope_epoch = refused.scope_epoch,
3114 code = refused.code.as_deref().unwrap_or(""),
3115 "scope record refused"
3116 );
3117 }
3118 self.close_scope_drained_routes(drained);
3119 let response = if is_apply {
3120 ModuleControlResponseToModule::ScopeApply {
3121 generation,
3122 results: applied.results,
3123 end_results: applied.end_results,
3124 ended: applied.ended,
3125 }
3126 } else {
3127 ModuleControlResponseToModule::ScopeSync {
3128 generation,
3129 results: applied.results,
3130 ended: applied.ended,
3131 }
3132 };
3133 Ok(vec![control_response_body_frame(
3134 &frame,
3135 &response,
3136 "scope change response",
3137 )?])
3138 }
3139 Err(refusal) => {
3140 info!(
3141 owner = %owner,
3142 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3143 generation,
3144 code = refusal.code,
3145 "scope change refused"
3146 );
3147 Ok(vec![control_error_frame(
3148 &frame,
3149 refusal.code,
3150 refusal.message,
3151 )?])
3152 }
3153 }
3154 }
3155
3156 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3162 if drained.is_empty() {
3163 return;
3164 }
3165 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3166 BTreeMap::new();
3167 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3168 for route in drained {
3169 warn!(
3170 module_id = %route.module_id,
3171 reason = ?route.reason,
3172 client_connection_id = route.client.connection_id.get(),
3173 route_channel = route.client.channel,
3174 "closing route because its scope changed"
3175 );
3176 pushes
3177 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3178 .or_insert_with(|| (route.reason, Vec::new()))
3179 .1
3180 .push(EndpointRoute {
3181 goodbye_target: route.client.clone(),
3182 principal: Principal::Unverified,
3183 bound_at: Instant::now(),
3184 draining: false,
3185 drain_reason: None,
3186 });
3187 goodbyes.push(route.module);
3188 goodbyes.push(route.client);
3189 }
3190 for ((module_id, _), (reason, routes)) in pushes {
3191 send_route_control_pushes(
3192 &self.forwarding,
3193 routes,
3194 ClientControlPush::RouteClosed {
3195 module_id,
3196 channels: Vec::new(),
3197 reason,
3198 drained: false,
3199 abandoned: 0,
3200 excluded_subscriptions: 0,
3201 terminal: Some(false),
3202 },
3203 );
3204 }
3205 self.emit_route_goodbyes(goodbyes);
3206 }
3207
3208 fn handle_scope_describe(
3211 &self,
3212 connection_id: ConnectionId,
3213 frame: Frame,
3214 owner: Principal,
3215 scope_ref: String,
3216 ) -> Result<Vec<Frame>, RouterError> {
3217 let registered = self
3218 .registry
3219 .get_module_by_connection(connection_id)
3220 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3221 if registered.is_none() {
3222 return Ok(vec![control_error_frame(
3223 &frame,
3224 "not_registered",
3225 "scope.describe requires an active module registration owned by this connection",
3226 )?]);
3227 }
3228 let description = self
3229 .scopes
3230 .read()
3231 .unwrap_or_else(|poisoned| poisoned.into_inner())
3232 .describe(&owner, &scope_ref);
3233 let owner_configured = match &owner {
3234 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3239 _ => false,
3240 };
3241 let response = ModuleControlResponseToModule::ScopeDescribe {
3242 status: description.status,
3243 scope_epoch: description.scope_epoch,
3244 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3245 owner_synced: description.owner_synced,
3246 owner_configured,
3247 scope: description.stamp,
3248 };
3249 Ok(vec![control_response_body_frame(
3250 &frame,
3251 &response,
3252 "ModuleControlResponseToModule::ScopeDescribe",
3253 )?])
3254 }
3255
3256 fn handle_catalog_update(
3257 &self,
3258 connection_id: ConnectionId,
3259 frame: Frame,
3260 provides: Vec<ProviderRole>,
3261 capabilities: Option<CapabilityDeclarations>,
3262 ready: Option<bool>,
3263 ) -> Result<Vec<Frame>, RouterError> {
3264 self.refresh_capability_requirements();
3265 let Some(registration) = self
3266 .registry
3267 .get_module_by_connection(connection_id)
3268 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3269 else {
3270 return Ok(vec![control_error_frame(
3271 &frame,
3272 "not_registered",
3273 "catalog.update requires an active module registration owned by this connection",
3274 )?]);
3275 };
3276
3277 if let Some(message) =
3278 catalog_update_frozen_field_message(®istration.manifest, &provides)
3279 {
3280 return Ok(vec![control_error_frame(
3281 &frame,
3282 "catalog_update_frozen_field",
3283 message,
3284 )?]);
3285 }
3286
3287 let mut candidate = registration.manifest.clone();
3288 candidate.provides = provides.clone();
3289 candidate.capabilities = capabilities
3290 .clone()
3291 .or_else(|| registration.manifest.capabilities.clone());
3292 if let Err(err) = candidate.validate_capability_grammar() {
3293 return Ok(vec![control_error_frame(
3294 &frame,
3295 "invalid_capability_grammar",
3296 err.to_string(),
3297 )?]);
3298 }
3299
3300 let mut conflicts = self
3303 .capability_evaluator
3304 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3305 if let Some(conflict) = conflicts.first() {
3306 let message = format!(
3307 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3308 conflict.capability, conflict.claimants[0], candidate.module_id
3309 );
3310 for conflict in &mut conflicts {
3311 conflict.source = DuplicateClaimSource::CatalogUpdate;
3312 }
3313 log_duplicate_claim_events(conflicts);
3314 return Ok(vec![control_error_frame(
3315 &frame,
3316 "reserved_capability",
3317 message,
3318 )?]);
3319 }
3320
3321 let updated = self
3322 .registry
3323 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3324 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3325 if updated.is_none() {
3326 return Ok(vec![control_error_frame(
3327 &frame,
3328 "not_registered",
3329 "catalog.update requires an active module registration owned by this connection",
3330 )?]);
3331 }
3332 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3333 log_duplicate_claim_events(
3334 self.capability_evaluator
3335 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3336 );
3337 }
3338 if capability_census_trigger(
3339 registration.manifest.capabilities.as_ref(),
3340 updated
3341 .as_ref()
3342 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3343 ) {
3344 self.enforce_capability_denies();
3345 }
3346 self.refresh_capability_requirements();
3347
3348 let response = ModuleControlResponseToModule::CatalogUpdate {};
3349 control_response_body_frame(
3350 &frame,
3351 &response,
3352 "ModuleControlResponseToModule::CatalogUpdate",
3353 )
3354 .map(|frame| vec![frame])
3355 }
3356
3357 pub(crate) fn with_activity_ledger_config(
3358 mut self,
3359 run_dir: &Path,
3360 incarnation: String,
3361 config: crate::activity::ActivityConfig,
3362 ) -> Self {
3363 self.activity = Some(crate::activity::ActivityLedger::open_config(
3364 run_dir.join("activity.jsonl"),
3365 incarnation,
3366 config,
3367 ));
3368 self.supervisor.set_activity(self.activity.clone());
3369 self
3370 }
3371
3372 pub(crate) async fn configure_activity(
3373 &self,
3374 config: crate::activity::ActivityConfig,
3375 ) -> Result<(), &'static str> {
3376 match &self.activity {
3377 Some(ledger) => ledger.configure(config).await,
3378 None => Ok(()),
3379 }
3380 }
3381
3382 fn cancel_activity_subscription(
3383 &self,
3384 ctx: &RouteCtx,
3385 frame: &Frame,
3386 ) -> Result<bool, RouterError> {
3387 let task = self
3388 .activity_streams
3389 .lock()
3390 .unwrap_or_else(|p| p.into_inner())
3391 .remove(&(ctx.connection_id, frame.header.corr));
3392 let Some(task) = task else {
3393 return Ok(false);
3394 };
3395 task.abort();
3396 let end = Frame::build_with_version(
3397 response_version(frame),
3398 FrameType::StreamEnd,
3399 control_flags(),
3400 0,
3401 0,
3402 frame.header.corr,
3403 Vec::new(),
3404 )
3405 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3406 let sink = ctx.egress.clone();
3407 tokio::spawn(async move {
3408 let _ = sink.send(end).await;
3409 });
3410 Ok(true)
3411 }
3412
3413 async fn handle_activity_read(
3414 &self,
3415 ctx: &RouteCtx,
3416 frame: Frame,
3417 ) -> Result<Vec<Frame>, RouterError> {
3418 use subc_control::{
3419 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3420 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3421 };
3422 #[derive(Deserialize)]
3423 struct Identity {
3424 #[serde(default)]
3425 consumer_identity: Option<ConsumerIdentity>,
3426 }
3427 let principal = match self
3431 .forwarding
3432 .module_id_for_connection(ctx.connection_id)
3433 .map_err(RouterError::Forwarding)?
3434 {
3435 Some(module_id) => Ok(Principal::Reserved { module_id }),
3436 None => {
3437 let identity: Identity = match serde_json::from_slice(&frame.body) {
3438 Ok(identity) => identity,
3439 Err(_) => {
3440 return Ok(vec![control_error_frame(
3441 &frame,
3442 ACTIVITY_MALFORMED,
3443 "malformed activity read",
3444 )?])
3445 }
3446 };
3447 self.route_open_principal(&frame, identity.consumer_identity)?
3448 }
3449 };
3450 let admitted = matches!(principal, Ok(Principal::Direct))
3451 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3452 if !admitted {
3453 return Ok(vec![control_error_frame(
3454 &frame,
3455 ACTIVITY_READ_FORBIDDEN,
3456 "activity reads require an operator or trusted mirror principal",
3457 )?]);
3458 }
3459 let Some(ledger) = &self.activity else {
3460 return Ok(vec![control_error_frame(
3461 &frame,
3462 ACTIVITY_UNAVAILABLE,
3463 "activity ledger unavailable",
3464 )?]);
3465 };
3466 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3467 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3468 {
3469 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3470 Ok(request) => ledger.query(request).await,
3471 Err(_) => Err(ACTIVITY_MALFORMED),
3472 };
3473 return match result {
3474 Ok(reply) => Ok(vec![control_response_body_frame(
3475 &frame,
3476 &reply,
3477 ops::ACTIVITY_QUERY,
3478 )?]),
3479 Err(code) => Ok(vec![control_error_frame(
3480 &frame,
3481 code,
3482 "activity query refused",
3483 )?]),
3484 };
3485 }
3486 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3487 Ok(request) => request,
3488 Err(_) => {
3489 return Ok(vec![control_error_frame(
3490 &frame,
3491 ACTIVITY_MALFORMED,
3492 "malformed activity subscription",
3493 )?])
3494 }
3495 };
3496 let mut subscription = match ledger.subscribe(request.since).await {
3497 Ok(subscription) => subscription,
3498 Err(code) => {
3499 return Ok(vec![control_error_frame(
3500 &frame,
3501 code,
3502 "activity subscription refused",
3503 )?])
3504 }
3505 };
3506 let key = (ctx.connection_id, frame.header.corr);
3507 let sink = ctx.egress.clone();
3508 let mut streams = self
3509 .activity_streams
3510 .lock()
3511 .unwrap_or_else(|p| p.into_inner());
3512 streams.retain(|_, task| !task.is_finished());
3513 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3514 return Ok(vec![control_error_frame(
3515 &frame,
3516 "duplicate_subscription",
3517 "activity subscription correlation id already in use",
3518 )?]);
3519 }
3520 let task = tokio::spawn(async move {
3521 loop {
3522 let entry = if let Some(entry) = subscription.replay.pop_front() {
3523 Some(entry)
3524 } else {
3525 subscription.receiver.recv().await
3526 };
3527 let Some(entry) = entry else {
3528 break;
3529 };
3530 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3531 return;
3532 };
3533 let Ok(next) = Frame::build_with_version(
3534 response_version(&frame),
3535 FrameType::StreamData,
3536 control_flags(),
3537 0,
3538 0,
3539 frame.header.corr,
3540 body,
3541 ) else {
3542 return;
3543 };
3544 if sink.send(next).await.is_err() {
3545 return;
3546 }
3547 }
3548 if let Some(code) = subscription.terminal_code() {
3551 if let Ok(end) =
3552 control_error_frame(&frame, code, "activity subscriber fell behind")
3553 {
3554 let _ = sink.send(end).await;
3555 }
3556 }
3557 });
3558 streams.insert(key, task.abort_handle());
3559 Ok(Vec::new())
3560 }
3561
3562 async fn handle_activity_record(
3563 &self,
3564 ctx: &RouteCtx,
3565 frame: Frame,
3566 ) -> Result<Vec<Frame>, RouterError> {
3567 use subc_control::{
3568 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3569 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3570 };
3571 let result = async {
3572 let module = self
3573 .forwarding
3574 .module_id_for_connection(ctx.connection_id)
3575 .ok()
3576 .flatten()
3577 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3578 let request: ActivityRecordRequest =
3579 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3580 crate::activity::validate(&module, &request)?;
3581 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3582 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3583 let at_ms = std::time::SystemTime::now()
3584 .duration_since(std::time::UNIX_EPOCH)
3585 .unwrap_or_default()
3586 .as_millis() as u64;
3587 ledger.record(module, request, actor, at_ms).await
3588 }
3589 .await;
3590 match result {
3591 Ok(entry) => Ok(vec![control_response_body_frame(
3592 &frame,
3593 &ActivityRecordResponse { seq: entry.seq },
3594 "activity.record",
3595 )?]),
3596 Err(code) => Ok(vec![control_error_frame(
3597 &frame,
3598 code,
3599 "activity recording refused",
3600 )?]),
3601 }
3602 }
3603
3604 fn activity_actor(
3605 &self,
3606 connection: ConnectionId,
3607 module: &str,
3608 on: Option<&subc_control::ActivityOn>,
3609 ) -> Result<subc_control::ActivityActor, &'static str> {
3610 use crate::forwarding::{DataRoute, DataRouteState};
3611 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3612 let principal = |id: &str| Principal::Reserved {
3613 module_id: id.to_owned(),
3614 };
3615 let mut actor = ActivityActor {
3616 principal: principal(module),
3617 scope_owner: None,
3618 agent_id: None,
3619 run_id: None,
3620 flow_id: None,
3621 project_root: None,
3622 };
3623 let scope = match on {
3624 None => return Ok(actor),
3625 Some(ActivityOn::Scope(scope)) => {
3626 let selector = subc_protocol::scope::ScopeSelector::new(
3627 principal(module),
3628 scope.scope_ref.clone(),
3629 Some(scope.epoch),
3630 );
3631 self.scopes
3632 .read()
3633 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3634 .attribute(&principal(module), module, &selector, true)
3635 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3636 }
3637 Some(ActivityOn::Route(route)) => {
3638 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3639 .forwarding
3640 .lookup_data_route(connection, route.channel, route.epoch)
3641 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3642 else {
3643 return Err(ACTIVITY_ACTOR_UNKNOWN);
3644 };
3645 if binding.module_id != module {
3646 return Err(ACTIVITY_ACTOR_UNKNOWN);
3647 }
3648 actor.principal = binding.principal.clone();
3649 actor.project_root = binding
3650 .project_root
3651 .as_ref()
3652 .map(|root| root.as_path().to_path_buf());
3653 let Some(bound) = &binding.scope else {
3654 return Ok(actor);
3655 };
3656 let owner = principal(&bound.owner);
3657 let selector = subc_protocol::scope::ScopeSelector::new(
3658 owner.clone(),
3659 bound.scope_ref.clone(),
3660 Some(bound.tag.scope_epoch),
3661 );
3662 let admitted = self
3663 .scopes
3664 .read()
3665 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3666 .attribute(&owner, module, &selector, true)
3667 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3668 if admitted.tag != bound.tag {
3669 return Err(ACTIVITY_ACTOR_UNKNOWN);
3670 }
3671 admitted
3672 }
3673 };
3674 actor.scope_owner = Some(scope.stamp.owner);
3675 actor.agent_id = scope.stamp.attributes.agent_id;
3676 actor.run_id = scope.stamp.attributes.run_id;
3677 actor.flow_id = scope.stamp.attributes.flow_id;
3678 Ok(actor)
3679 }
3680
3681 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3682 self.refresh_capability_requirements();
3683 let mut counters = self.counters.snapshot();
3689 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3690 obj.insert(
3691 "activity_unparseable_lines".into(),
3692 ledger.read_errors().into(),
3693 );
3694 obj.insert(
3695 "activity_lifecycle_append_failures".into(),
3696 ledger.lifecycle_failures().into(),
3697 );
3698 obj.insert(
3699 "activity_started_digest_skips".into(),
3700 ledger.lifecycle_digest_skips().into(),
3701 );
3702 }
3703 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3704 self.forwarding.client_route_concentration(),
3705 counters.as_object_mut(),
3706 ) {
3707 obj.insert(
3708 "client_connections_with_routes".into(),
3709 connections_with_routes.into(),
3710 );
3711 obj.insert("max_routes_on_one_connection".into(), max.into());
3712 }
3713 if let (Some(open_breakers), Some(obj)) = (
3719 self.route_bind_breakers.open_snapshot(),
3720 counters.as_object_mut(),
3721 ) {
3722 obj.insert("route_bind_breakers_open".into(), open_breakers);
3723 }
3724 let response = ClientControlResponse::ServerDescribe {
3725 protocol_ver: PROTOCOL_VERSION,
3726 subc_ops: subc_ops(),
3727 capabilities: self.subc_capabilities.as_ref().to_vec(),
3728 connected_clients: self.connected_clients.count(),
3729 counters: Some(counters),
3730 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3731 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3732 capability_requirements: self.capability_requirement_statuses(),
3733 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3734 };
3735 Ok(vec![control_response_body_frame(
3736 &frame,
3737 &response,
3738 "ClientControlResponse::ServerDescribe",
3739 )?])
3740 }
3741
3742 fn handle_catalog_list(
3743 &self,
3744 frame: Frame,
3745 module_id: Option<String>,
3746 ) -> Result<Vec<Frame>, RouterError> {
3747 let (generation, modules) = self.registry.list_modules().map_err(|err| {
3748 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3749 })?;
3750 let entries = modules
3751 .into_iter()
3752 .filter(|registration| {
3753 module_id
3754 .as_deref()
3755 .map(|wanted| registration.manifest.module_id == wanted)
3756 .unwrap_or(true)
3757 })
3758 .map(|registration| {
3759 let not_ready = self.not_ready_reason(®istration);
3760 let roles = registration.manifest.provides;
3761 CatalogEntry::new(
3762 registration.manifest.module_id,
3763 roles,
3764 registration.control_ops,
3765 )
3766 .with_ready(not_ready.is_none())
3767 .with_not_ready(not_ready)
3768 .with_module_version(Some(registration.manifest.module_version))
3769 .with_capabilities(registration.manifest.capabilities)
3770 .with_self_signals(registration.manifest.self_signals)
3771 .with_events(registration.manifest.events)
3772 })
3773 .collect();
3774 let response = ClientControlResponse::CatalogList {
3775 generation,
3776 modules: entries,
3777 subc_ops: subc_ops(),
3778 };
3779 Ok(vec![control_response_body_frame(
3780 &frame,
3781 &response,
3782 "ClientControlResponse::CatalogList",
3783 )?])
3784 }
3785
3786 fn route_open_principal(
3787 &self,
3788 frame: &Frame,
3789 consumer_identity: Option<ConsumerIdentity>,
3790 ) -> Result<Result<Principal, Frame>, RouterError> {
3791 let Some(consumer_identity) = consumer_identity else {
3792 return Ok(Ok(Principal::Direct));
3793 };
3794
3795 if self.supervisor.spawned_consumer_authorized(
3796 &consumer_identity.module_id,
3797 &consumer_identity.launch_nonce,
3798 ) {
3799 return Ok(Ok(Principal::Reserved {
3800 module_id: consumer_identity.module_id,
3801 }));
3802 }
3803
3804 Ok(Err(control_error_frame(
3805 frame,
3806 "bad_consumer_identity",
3807 format!(
3808 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3809 consumer_identity.module_id
3810 ),
3811 )?))
3812 }
3813
3814 fn route_open_refusal_frame(
3821 &self,
3822 ctx: &RouteCtx,
3823 frame: &Frame,
3824 module_id: &str,
3825 reason: &'static str,
3826 code: &'static str,
3827 message: impl Into<String>,
3828 ) -> Result<Frame, RouterError> {
3829 self.observe_route_open_refusal(ctx, module_id, reason, code);
3830 control_error_frame(frame, code, message.into())
3831 }
3832
3833 fn route_open_breaker_refusal_frame(
3851 &self,
3852 ctx: &RouteCtx,
3853 frame: &Frame,
3854 module_id: &str,
3855 consecutive_timeouts: u32,
3856 retry_in: Duration,
3857 probe_in_flight: bool,
3858 ) -> Result<Frame, RouterError> {
3859 self.counters
3860 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
3861 debug!(
3862 target: "control",
3863 code = "module_timeout",
3864 module_id = ?module_id,
3865 connection_id = ctx.connection_id.get(),
3866 consecutive_timeouts,
3867 retry_in_ms = retry_in.as_millis() as u64,
3868 probe_in_flight,
3869 "route.open refused by open bind-relay breaker"
3870 );
3871 let detail = if probe_in_flight {
3878 "one test route is already being tried; retry once it settles".to_string()
3879 } else {
3880 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
3881 };
3882 control_error_frame(
3883 frame,
3884 "module_timeout",
3885 format!(
3886 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
3887 timed out in a row); {detail}; its established routes are unaffected"
3888 ),
3889 )
3890 }
3891
3892 fn observe_route_open_refusal(
3905 &self,
3906 ctx: &RouteCtx,
3907 module_id: &str,
3908 reason: &'static str,
3909 code: &'static str,
3910 ) {
3911 self.counters.increment_route_open_refused(code);
3912 info!(
3913 target: "control",
3914 code,
3915 reason,
3916 module_id = ?module_id,
3917 connection_id = ctx.connection_id.get(),
3918 "route.open refused"
3919 );
3920 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
3921 self.route_outages.record_not_serving(module_id, reason);
3922 }
3923 }
3924
3925 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
3970 self.route_outages.record_accepted(module_id);
3971 self.counters.increment_route_open_accepted(principal);
3972 info!(
3973 target: "control",
3974 principal,
3975 module_id,
3976 connection_id = ctx.connection_id.get(),
3977 "route.open accepted"
3978 );
3979 }
3980
3981 fn supervised_absent_route_open_refusal_frame(
3982 &self,
3983 ctx: &RouteCtx,
3984 frame: &Frame,
3985 module_id: &str,
3986 code: &'static str,
3987 status: &crate::supervise::ModuleStatus,
3988 ) -> Result<Frame, RouterError> {
3989 self.counters.increment_route_open_refused(code);
3990 info!(
3991 target: "control",
3992 code,
3993 reason = "supervised_not_registered",
3994 module_id = ?module_id,
3995 connection_id = ctx.connection_id.get(),
3996 state = %status.state,
3997 enabled = status.enabled,
3998 live = status.live,
3999 "route.open refused"
4000 );
4001 self.route_outages
4005 .record_not_serving(module_id, "supervised_not_registered");
4006 let message = if let Some(pid) = status.waiting_for_previous_pid {
4007 format!("waiting: previous instance (pid {pid}) is still running but hidden; it cannot be identified")
4008 } else {
4009 format!(
4010 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4011 status.state, status.enabled, status.live
4012 )
4013 };
4014 control_error_frame(frame, code, message)
4015 }
4016
4017 async fn handle_route_open(
4018 &self,
4019 ctx: &RouteCtx,
4020 frame: Frame,
4021 request: RouteOpenRequest,
4022 timings: &mut RouteOpenTimings,
4023 ) -> Result<Vec<Frame>, RouterError> {
4024 let RouteOpenRequest {
4025 target,
4026 mut identity,
4027 consumer_identity,
4028 consumer_capabilities,
4029 role_versions,
4030 admission_facts,
4031 scope,
4032 } = request;
4033 let target_module_id = target_module_id(&target).to_string();
4034 timings.module_id.clone_from(&target_module_id);
4035 if self
4036 .registry
4037 .get_module_by_connection(ctx.connection_id)
4038 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4039 .is_some()
4040 {
4041 return Ok(vec![control_error_frame(
4042 &frame,
4043 "invalid_request",
4044 "module connections cannot open client routes",
4045 )?]);
4046 }
4047 debug!(
4048 connection_id = ctx.connection_id.get(),
4049 corr = frame.header.corr,
4050 module_id = %target_module_id,
4051 "handling route.open"
4052 );
4053
4054 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4060 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4061 self.observe_route_open_refusal(
4062 ctx,
4063 &target_module_id,
4064 "invalid_role_versions",
4065 error_codes::INVALID_REQUEST,
4066 );
4067 return Ok(vec![control_error_body_frame(
4068 &frame,
4069 ErrorBody {
4070 code: error_codes::INVALID_REQUEST.to_string(),
4071 message: error.to_string(),
4072 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4073 },
4074 )?]);
4075 }
4076
4077 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4078 if status.waiting_for_previous_pid.is_some() {
4079 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4080 ctx,
4081 &frame,
4082 &target_module_id,
4083 "module_warming",
4084 &status,
4085 )?]);
4086 }
4087 }
4088
4089 let Some(registration) = self
4108 .registry
4109 .get_module(&target_module_id)
4110 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4111 else {
4112 if let Some((status, warming)) =
4113 self.supervisor_status(&target_module_id, frame.header.corr)?
4114 {
4115 if status.protocol == ModuleProtocol::None {
4122 return Ok(vec![self.route_open_refusal_frame(
4123 ctx,
4124 &frame,
4125 &target_module_id,
4126 "protocol_none",
4127 error_codes::MODULE_NO_PROTOCOL,
4128 format!(
4129 "module_id '{target_module_id}' is declared protocol: none; \
4130 it speaks no subc wire and serves no routes"
4131 ),
4132 )?]);
4133 }
4134 let code = if warming {
4135 "module_warming"
4136 } else {
4137 "target_unavailable"
4138 };
4139 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4140 ctx,
4141 &frame,
4142 &target_module_id,
4143 code,
4144 &status,
4145 )?]);
4146 }
4147 if let Some(removed_ago_ms) =
4148 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4149 {
4150 return Ok(vec![self.route_open_refusal_frame(
4151 ctx,
4152 &frame,
4153 &target_module_id,
4154 "removed",
4155 error_codes::MODULE_REMOVED,
4156 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4157 )?]);
4158 }
4159 return Ok(vec![self.route_open_refusal_frame(
4160 ctx,
4161 &frame,
4162 &target_module_id,
4163 "not_registered",
4164 error_codes::UNKNOWN_MODULE,
4165 format!("module_id '{target_module_id}' is not registered"),
4166 )?]);
4167 };
4168
4169 if !registration.ready {
4173 self.counters
4174 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4175 info!(
4176 target: "control",
4177 code = error_codes::MODULE_WARMING,
4178 module_id = ?target_module_id,
4179 connection_id = ctx.connection_id.get(),
4180 reason = "declared_not_ready",
4181 "route.open refused"
4182 );
4183 self.route_outages
4186 .record_not_serving(&target_module_id, "declared_not_ready");
4187 return Ok(vec![control_error_body_frame(
4188 &frame,
4189 ErrorBody {
4190 code: error_codes::MODULE_WARMING.to_string(),
4191 message: format!(
4192 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4193 ),
4194 detail: Some(serde_json::json!({
4195 "reason": "declared_not_ready"
4196 })),
4197 },
4198 )?]);
4199 }
4200
4201 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4220 self.counters
4221 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4222 info!(
4223 target: "control",
4224 code = error_codes::MODULE_WARMING,
4225 module_id = ?target_module_id,
4226 connection_id = ctx.connection_id.get(),
4227 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4228 capability = %capability,
4229 "route.open refused"
4230 );
4231 return Ok(vec![control_error_body_frame(
4232 &frame,
4233 ErrorBody {
4234 code: error_codes::MODULE_WARMING.to_string(),
4235 message: format!(
4236 "module_id '{target_module_id}' requires capability '{capability}', \
4237 which no registered module provides; retry"
4238 ),
4239 detail: Some(serde_json::json!({
4240 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4241 "capability": capability,
4242 })),
4243 },
4244 )?]);
4245 }
4246
4247 if !target_has_required_role(&target, ®istration.manifest.provides) {
4248 return Ok(vec![self.route_open_refusal_frame(
4249 ctx,
4250 &frame,
4251 &target_module_id,
4252 "role_not_provided",
4253 "target_unavailable",
4254 format!("module_id '{target_module_id}' does not provide the requested target"),
4255 )?]);
4256 }
4257
4258 if registration.state != ChannelState::Active {
4259 return Ok(vec![self.route_open_refusal_frame(
4260 ctx,
4261 &frame,
4262 &target_module_id,
4263 "registration_not_active",
4264 "target_unavailable",
4265 format!("module_id '{target_module_id}' is not active"),
4266 )?]);
4267 }
4268
4269 if self
4270 .forwarding
4271 .module_is_draining(&target_module_id)
4272 .map_err(RouterError::Forwarding)?
4273 {
4274 return Ok(vec![self.route_open_refusal_frame(
4275 ctx,
4276 &frame,
4277 &target_module_id,
4278 "reloading",
4279 "module_reloading",
4280 format!("module_id '{target_module_id}' is reloading"),
4281 )?]);
4282 }
4283
4284 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4285 process_liveness.process_live(&target_module_id) == Some(false)
4286 }) {
4287 if process_liveness.process_replacing(&target_module_id) {
4294 return Ok(vec![self.route_open_refusal_frame(
4295 ctx,
4296 &frame,
4297 &target_module_id,
4298 "reloading",
4299 "module_reloading",
4300 format!("module_id '{target_module_id}' is reloading"),
4301 )?]);
4302 }
4303 return Ok(vec![self.route_open_refusal_frame(
4304 ctx,
4305 &frame,
4306 &target_module_id,
4307 "supervisor_not_live",
4308 "target_unavailable",
4309 format!("module_id '{target_module_id}' is not live"),
4310 )?]);
4311 }
4312
4313 if !self
4314 .forwarding
4315 .has_live_module_connection(&target_module_id)
4316 .map_err(RouterError::Forwarding)?
4317 {
4318 return Ok(vec![self.route_open_refusal_frame(
4319 ctx,
4320 &frame,
4321 &target_module_id,
4322 "no_forwarding_connection",
4323 "target_unavailable",
4324 format!("module_id '{target_module_id}' has no live forwarding connection"),
4325 )?]);
4326 }
4327
4328 if let Some(error) =
4329 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4330 {
4331 self.observe_route_open_refusal(
4332 ctx,
4333 &target_module_id,
4334 "op_not_allowed",
4335 "op_not_allowed",
4336 );
4337 return Ok(vec![error]);
4338 }
4339
4340 let principal = match self.route_open_principal(&frame, consumer_identity)? {
4341 Ok(principal) => principal,
4342 Err(error) => {
4343 self.observe_route_open_refusal(
4344 ctx,
4345 &target_module_id,
4346 "bad_consumer_identity",
4347 "bad_consumer_identity",
4348 );
4349 return Ok(vec![error]);
4350 }
4351 };
4352 timings.principal = principal_label(&principal);
4353
4354 if let Principal::Reserved {
4358 module_id: opening_module_id,
4359 } = &principal
4360 {
4361 if let Some(opening_registration) = self
4362 .registry
4363 .get_module(opening_module_id)
4364 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4365 {
4366 if let Some(capability) =
4367 denied_capability(&opening_registration.manifest, ®istration.manifest)
4368 {
4369 warn!(
4370 opening_module_id,
4371 target_module_id,
4372 capability,
4373 "refusing route.open because an attested capability deny edge matches"
4374 );
4375 return Ok(vec![self.route_open_refusal_frame(
4376 ctx,
4377 &frame,
4378 &target_module_id,
4379 "capability_deny_edge",
4380 "capability_forbidden",
4381 format!(
4382 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4383 ),
4384 )?]);
4385 }
4386 }
4387 }
4388
4389 if admission_facts.is_some() {
4390 let carrier_matches = matches!(
4391 &principal,
4392 Principal::Reserved { module_id }
4393 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4394 );
4395 if !carrier_matches {
4396 return Ok(vec![self.route_open_refusal_frame(
4397 ctx,
4398 &frame,
4399 &target_module_id,
4400 "admission_facts_carrier_not_permitted",
4401 "admission_facts_not_permitted",
4402 "admission facts may only be carried by the configured reserved module",
4403 )?]);
4404 }
4405
4406 let target_allowed = self
4407 .admission_facts_targets
4408 .as_ref()
4409 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4410 if !target_allowed {
4411 return Ok(vec![self.route_open_refusal_frame(
4412 ctx,
4413 &frame,
4414 &target_module_id,
4415 "admission_facts_target_not_listed",
4416 "admission_facts_target_not_allowed",
4417 format!(
4418 "admission facts are not permitted for target module_id '{target_module_id}'"
4419 ),
4420 )?]);
4421 }
4422
4423 }
4427
4428 let (bound_scope, mut scope_stamp) = match &scope {
4434 None => (None, None),
4435 Some(selector) => {
4436 let owner_configured = match &selector.owner {
4437 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4443 _ => false,
4444 };
4445 let mut scopes = self
4446 .scopes
4447 .write()
4448 .unwrap_or_else(|poisoned| poisoned.into_inner());
4449 let admitted = scopes
4450 .admit(&principal, &target_module_id, selector, owner_configured)
4451 .and_then(|admission| {
4452 crate::scopes::check_target_flow_support(
4453 &admission.stamp,
4454 &target_module_id,
4455 registration.manifest.capabilities.as_ref(),
4456 )?;
4457 crate::scopes::check_target_agent_run_support(
4458 &admission.stamp,
4459 &target_module_id,
4460 registration.manifest.capabilities.as_ref(),
4461 )?;
4462 let bound = scopes.reserve_bind(
4463 &admission,
4464 &principal,
4465 ctx.connection_id,
4466 selector.hold_presence,
4467 );
4468 Ok((admission, bound))
4469 });
4470 match admitted {
4471 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4472 Err(refusal) => {
4473 return Ok(vec![self.route_open_refusal_frame(
4474 ctx,
4475 &frame,
4476 &target_module_id,
4477 refusal.code,
4478 refusal.code,
4479 refusal.message,
4480 )?]);
4481 }
4482 }
4483 }
4484 };
4485
4486 timings.enter(RouteOpenPhase::ProjectRoot);
4508 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4509 Ok(project_root) => project_root,
4510 Err(err) => {
4511 return Ok(vec![control_error_frame(
4512 &frame,
4513 "invalid_project_root",
4514 err.to_string(),
4515 )?])
4516 }
4517 };
4518 timings.enter(RouteOpenPhase::Admission);
4519 identity.project_root = project_root.as_path().to_path_buf();
4520
4521 let _concurrency_guard = match self
4532 .route_bind_concurrency
4533 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4534 {
4535 Ok(guard) => guard,
4536 Err(in_flight) => {
4537 return Ok(vec![self.route_open_target_capacity_refusal(
4538 ctx,
4539 &frame,
4540 &target_module_id,
4541 in_flight,
4542 )?]);
4543 }
4544 };
4545
4546 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4549 RouteBindAdmission::Admitted { guard, probe } => {
4550 if probe {
4551 info!(
4552 module_id = %target_module_id,
4553 connection_id = ctx.connection_id.get(),
4554 "route.bind breaker half-open: admitting one probe"
4555 );
4556 }
4557 guard
4558 }
4559 RouteBindAdmission::Refused {
4560 consecutive_timeouts,
4561 retry_in,
4562 probe_in_flight,
4563 } => {
4564 return Ok(vec![self.route_open_breaker_refusal_frame(
4565 ctx,
4566 &frame,
4567 &target_module_id,
4568 consecutive_timeouts,
4569 retry_in,
4570 probe_in_flight,
4571 )?]);
4572 }
4573 };
4574
4575 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4580 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4581 timings.enter(RouteOpenPhase::EgressReservation);
4582 let pending = match self
4583 .forwarding
4584 .begin_route_bind_relay_for(
4585 ctx.connection_id,
4586 ctx.egress.clone(),
4587 response_version(&frame),
4588 frame.header.corr,
4589 &target_module_id,
4590 principal.clone(),
4591 bound_scope,
4592 Some(project_root),
4593 relay_deadline,
4594 )
4595 .await
4596 {
4597 Ok(pending) => pending,
4598 Err(err) => {
4599 return Ok(vec![self.route_open_refusal_frame(
4600 ctx,
4601 &frame,
4602 &target_module_id,
4603 "relay_reservation_failed",
4604 forwarding_error_code(&err),
4605 err.to_string(),
4606 )?])
4607 }
4608 };
4609 timings.enter(RouteOpenPhase::Admission);
4610 let crate::forwarding::PendingRouteBindRelay {
4611 endpoint,
4612 module_sink,
4613 negotiated_ver,
4614 client_channel,
4615 client_epoch,
4616 module_channel,
4617 module_epoch,
4618 corr: relay_corr,
4619 receiver,
4620 } = pending;
4621 let mut reservation =
4622 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4623
4624 let relay_registration = self
4629 .registry
4630 .get_module_by_connection(endpoint.connection_id)
4631 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4632 let relay_capabilities = relay_registration
4633 .as_ref()
4634 .and_then(|registration| registration.manifest.capabilities.as_ref());
4635 if let Some(selector) = &scope {
4636 if let Err(refusal) = self
4639 .scopes
4640 .read()
4641 .unwrap_or_else(|poisoned| poisoned.into_inner())
4642 .check_bind_proof(&principal, selector)
4643 {
4644 reservation.release_and_disarm();
4645 return Ok(vec![self.route_open_refusal_frame(
4646 ctx,
4647 &frame,
4648 &target_module_id,
4649 refusal.code,
4650 refusal.code,
4651 refusal.message,
4652 )?]);
4653 }
4654 }
4655 if let Some(stamp) = scope_stamp
4656 .as_ref()
4657 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4658 {
4659 if let Err(refusal) = crate::scopes::check_target_flow_support(
4660 stamp,
4661 &target_module_id,
4662 relay_capabilities,
4663 )
4664 .and_then(|()| {
4665 crate::scopes::check_target_agent_run_support(
4666 stamp,
4667 &target_module_id,
4668 relay_capabilities,
4669 )
4670 }) {
4671 reservation.release_and_disarm();
4672 return Ok(vec![self.route_open_refusal_frame(
4673 ctx,
4674 &frame,
4675 &target_module_id,
4676 refusal.code,
4677 refusal.code,
4678 refusal.message,
4679 )?]);
4680 }
4681 }
4682
4683 let call_mode = if relay_capabilities
4686 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4687 {
4688 Some(derive_call_mode(scope_stamp.as_ref()))
4689 } else {
4690 if let Some(stamp) = scope_stamp.as_mut() {
4691 stamp.expires_at_ms = None;
4692 }
4693 None
4694 };
4695
4696 debug!(
4697 connection_id = ctx.connection_id.get(),
4698 client_channel,
4699 client_epoch,
4700 module_channel,
4701 module_epoch,
4702 "reserved route handle pair"
4703 );
4704 let principal_label = principal_label(&principal);
4707 let relay = ModuleControlRequest::RouteBind {
4708 route_channel: module_channel,
4709 epoch: module_epoch,
4710 target,
4711 identity,
4712 principal: Some(principal),
4713 consumer_capabilities,
4714 role_versions,
4715 admission_facts,
4716 scope: scope_stamp,
4717 call_mode,
4718 };
4719 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4720 RouterError::backend(
4721 0,
4722 frame.header.corr,
4723 format!("failed to encode route.bind request: {err}"),
4724 )
4725 })?;
4726 let relay_frame = Frame::build_with_version(
4727 negotiated_ver,
4728 FrameType::Request,
4729 control_flags(),
4730 0,
4731 0,
4732 relay_corr,
4733 relay_body,
4734 )
4735 .map_err(RouterError::FrameBuild)?;
4736
4737 timings.enter(RouteOpenPhase::BindSend);
4738 let sent = module_sink.send(relay_frame).await;
4739 timings.enter(RouteOpenPhase::Admission);
4740 if let Err(err) = sent {
4741 reservation.release_and_disarm();
4742 return Ok(vec![self.route_open_refusal_frame(
4743 ctx,
4744 &frame,
4745 &target_module_id,
4746 "relay_send_failed",
4747 "target_unavailable",
4748 err.to_string(),
4749 )?]);
4750 }
4751
4752 #[cfg(test)]
4753 if let Some(hold) = &self.route_bind_relay_sent_hold {
4754 hold.wait().await;
4758 hold.wait().await;
4759 }
4760
4761 if self
4766 .forwarding
4767 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4768 .map_err(RouterError::Forwarding)?
4769 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4770 {
4771 self.send_abandoned_route_bind_goodbye(
4772 &module_sink,
4773 negotiated_ver,
4774 module_channel,
4775 module_epoch,
4776 );
4777 }
4778
4779 timings.enter(RouteOpenPhase::BindAckWait);
4780 let outcome = timeout_at(relay_deadline, receiver).await;
4781 timings.enter(RouteOpenPhase::Admission);
4782 match outcome {
4783 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4784 reservation.disarm();
4785 if breaker.record_accepted() {
4786 info!(
4787 module_id = %target_module_id,
4788 "route.bind breaker closed: the probe was accepted"
4789 );
4790 }
4791 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
4792 Ok(Vec::new())
4793 }
4794 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4795 reservation.release_and_disarm();
4796 breaker.record_inconclusive();
4800 let scope_code = match body.code.as_str() {
4804 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4805 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4806 _ => None,
4807 };
4808 if let Some(code) = scope_code {
4809 self.observe_route_open_refusal(
4810 ctx,
4811 &target_module_id,
4812 "scope_changed_before_commit",
4813 code,
4814 );
4815 return Ok(vec![control_error_body_frame(&frame, body)?]);
4816 }
4817 self.counters
4818 .increment_route_open_refused("module_rejected");
4819 info!(
4820 target: "control",
4821 code = "module_rejected",
4822 module_code = ?body.code,
4823 module_id = ?target_module_id,
4824 connection_id = ctx.connection_id.get(),
4825 "route.open refused"
4826 );
4827 Ok(vec![control_error_body_frame(&frame, body)?])
4828 }
4829 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
4830 reservation.release_and_disarm();
4831 breaker.record_inconclusive();
4832 tracing::warn!(
4839 module_id = %target_module_id,
4840 "route.bind relay abandoned: {message}"
4841 );
4842 Ok(vec![self.route_open_refusal_frame(
4843 ctx,
4844 &frame,
4845 &target_module_id,
4846 "relay_abandoned",
4847 "target_unavailable",
4848 message,
4849 )?])
4850 }
4851 Ok(Err(_)) => {
4852 reservation.release_and_disarm();
4853 breaker.record_inconclusive();
4854 Ok(vec![self.route_open_refusal_frame(
4855 ctx,
4856 &frame,
4857 &target_module_id,
4858 "relay_waiter_canceled",
4859 "target_unavailable",
4860 "route.bind relay waiter was canceled before the module responded",
4861 )?])
4862 }
4863 Err(_) => {
4864 reservation.release_and_disarm();
4865 if let Some(opened) = breaker.record_timeout(
4869 self.route_bind_breaker_threshold,
4870 self.route_bind_breaker_cooldown,
4871 ) {
4872 warn!(
4873 module_id = %target_module_id,
4874 consecutive_timeouts = opened.consecutive_timeouts,
4875 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
4876 reopened_after_probe = opened.reopened_after_probe,
4877 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
4878 );
4879 }
4880 tracing::warn!(
4888 module_id = %target_module_id,
4889 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
4890 "route.bind relay timed out: module did not ack within budget"
4891 );
4892 Ok(vec![self.route_open_refusal_frame(
4893 ctx,
4894 &frame,
4895 &target_module_id,
4896 "relay_timed_out",
4897 "module_timeout",
4898 format!(
4899 "module_id '{target_module_id}' did not answer route.bind within {:?}",
4900 route_bind_relay_timeout
4901 ),
4902 )?])
4903 }
4904 }
4905 }
4906
4907 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4908 let response = ClientControlResponse::SupervisorSpawnSnapshot {
4909 snapshot: self.supervisor.spawn_snapshot(),
4910 };
4911 Ok(vec![control_response_body_frame(
4912 &frame,
4913 &response,
4914 "ClientControlResponse::SupervisorSpawnSnapshot",
4915 )?])
4916 }
4917
4918 fn handle_supervisor_spawn_subscribe(
4919 &self,
4920 ctx: &RouteCtx,
4921 frame: Frame,
4922 since: Option<SpawnCursor>,
4923 ) -> Result<Vec<Frame>, RouterError> {
4924 match self.supervisor.subscribe_spawns(
4925 ctx.connection_id,
4926 frame.header.corr,
4927 response_version(&frame),
4928 since,
4929 ctx.egress.clone(),
4930 ) {
4931 Ok(()) => Ok(Vec::new()),
4932 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
4933 Ok(vec![control_error_body_frame(
4934 &frame,
4935 ErrorBody {
4936 code: "spawn_cursor_incarnation_mismatch".to_string(),
4937 message: "spawn cursor belongs to a different daemon incarnation"
4938 .to_string(),
4939 detail: Some(serde_json::json!({
4940 "current_daemon_incarnation": current
4941 })),
4942 },
4943 )?])
4944 }
4945 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
4946 &frame,
4947 ErrorBody {
4948 code: "spawn_cursor_too_old".to_string(),
4949 message: "spawn cursor predates the retained event ring".to_string(),
4950 detail: Some(serde_json::json!({
4951 "oldest_retained_cursor": oldest
4952 })),
4953 },
4954 )?]),
4955 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
4956 0,
4957 frame.header.corr,
4958 format!("failed to open supervisor spawn subscription: {error}"),
4959 )),
4960 }
4961 }
4962
4963 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4964 let generation = self
4965 .registry
4966 .generation()
4967 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4968 let mut modules = Vec::new();
4969 for module in self.supervisor.list() {
4970 let status = module.status_for_control("list").map_err(|err| {
4971 RouterError::backend(
4972 0,
4973 frame.header.corr,
4974 format!("failed to read supervisor status: {err}"),
4975 )
4976 })?;
4977 let (configured, _) = module.configuration().map_err(|err| {
4978 RouterError::backend(
4979 0,
4980 frame.header.corr,
4981 format!("failed to read module configuration: {err}"),
4982 )
4983 })?;
4984 let image = module.running_image_agreement().await;
4986 let resources = Some(module.child_resource_usage());
4989 let pending_reload = Some(reload_verdict(
4990 &configured.program,
4991 status.spawned_from.as_deref(),
4992 image,
4993 ));
4994 modules.push(
4997 SupervisorEntry::new(
4998 status.module_id,
4999 status.state.to_string(),
5000 status.enabled,
5001 status.live,
5002 status.health.status,
5003 )
5004 .with_launch_nonce_env(Some(!cfg!(unix)))
5005 .with_launch_nonce_source(status.launch_nonce_source)
5006 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5007 .with_protocol(status.protocol)
5008 .with_pending_reload(pending_reload)
5009 .with_last_probe_ms(status.health.last_probe_ms)
5010 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5011 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5012 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5013 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5014 .with_restart_count(Some(status.restart_count))
5015 .with_max_restarts(Some(status.max_restarts))
5016 .with_lifetime_restarts(Some(status.lifetime_restarts))
5017 .with_spawn_generation(Some(status.spawn_generation))
5018 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5019 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5020 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5021 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5022 .with_resources(resources),
5023 );
5024 }
5025 let response = ClientControlResponse::SupervisorList {
5026 generation,
5027 modules,
5028 };
5029 Ok(vec![control_response_body_frame(
5030 &frame,
5031 &response,
5032 "ClientControlResponse::SupervisorList",
5033 )?])
5034 }
5035
5036 fn handle_supervisor_stderr_tail(
5037 &self,
5038 frame: Frame,
5039 module_id: String,
5040 max_lines: Option<u32>,
5041 max_bytes: Option<u32>,
5042 ) -> Result<Vec<Frame>, RouterError> {
5043 let Some(module) = self.supervisor.get(&module_id) else {
5044 return Ok(vec![control_error_frame(
5045 &frame,
5046 "unknown_module",
5047 format!("module_id '{module_id}' is not supervised"),
5048 )?]);
5049 };
5050
5051 let snapshot = module.stderr_tail(
5052 max_lines.map(|value| value as usize),
5053 max_bytes.map(|value| value as usize),
5054 );
5055
5056 let response = ClientControlResponse::SupervisorStderrTail {
5057 module_id,
5058 tail: StderrTail {
5059 capture: match snapshot.capture {
5060 CaptureState::Captured => StderrCaptureState::Captured,
5061 CaptureState::Incomplete { reason } => {
5062 StderrCaptureState::Incomplete { reason }
5063 }
5064 CaptureState::NotCaptured { reason } => {
5065 StderrCaptureState::NotCaptured { reason }
5066 }
5067 },
5068 entries: snapshot
5069 .entries
5070 .into_iter()
5071 .map(|entry| match entry {
5072 TailEntry::Line {
5073 text,
5074 truncated,
5075 at_ms,
5076 } => StderrTailEntry::Line {
5077 text,
5078 truncated,
5079 at_ms,
5080 },
5081 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5082 })
5083 .collect(),
5084 dropped_lines: snapshot.dropped_lines,
5085 },
5086 };
5087 Ok(vec![control_response_body_frame(
5088 &frame,
5089 &response,
5090 "ClientControlResponse::SupervisorStderrTail",
5091 )?])
5092 }
5093
5094 async fn handle_supervisor_terminals(
5095 &self,
5096 frame: Frame,
5097 module_id: String,
5098 ) -> Result<Vec<Frame>, RouterError> {
5099 let Some(module) = self.supervisor.get(&module_id) else {
5100 return Ok(vec![control_error_frame(
5101 &frame,
5102 "unknown_module",
5103 format!("module_id '{module_id}' is not supervised"),
5104 )?]);
5105 };
5106
5107 let terminals = module
5110 .read_durable_terminal_history()
5111 .await
5112 .map_err(|error| {
5113 RouterError::backend(
5114 0,
5115 frame.header.corr,
5116 format!("failed to read terminal history: {error}"),
5117 )
5118 })?;
5119 let response = ClientControlResponse::SupervisorTerminals {
5120 module_id,
5121 terminals,
5122 };
5123 Ok(vec![control_response_body_frame(
5124 &frame,
5125 &response,
5126 "ClientControlResponse::SupervisorTerminals",
5127 )?])
5128 }
5129
5130 fn handle_supervisor_routes(
5131 &self,
5132 frame: Frame,
5133 module_id: Option<String>,
5134 ) -> Result<Vec<Frame>, RouterError> {
5135 let modules = self
5136 .forwarding
5137 .route_census(module_id.as_deref())
5138 .map_err(RouterError::Forwarding)?
5139 .into_iter()
5140 .map(|(module_id, routes)| SupervisorRouteModule {
5141 module_id,
5142 routes: routes
5143 .into_iter()
5144 .map(|route| SupervisorRoute {
5145 consumer: match route.principal {
5146 Principal::Reserved { module_id } => {
5147 SupervisorRouteConsumer::Reserved { module_id }
5148 }
5149 Principal::Direct | Principal::Unverified => {
5150 SupervisorRouteConsumer::Direct {
5151 connection_id: route.goodbye_target.connection_id.get(),
5152 }
5153 }
5154 },
5155 age_ms: Instant::now()
5156 .saturating_duration_since(route.bound_at)
5157 .as_millis()
5158 .try_into()
5159 .unwrap_or(u64::MAX),
5160 draining: route.draining,
5161 drain_reason: route.drain_reason,
5162 })
5163 .collect(),
5164 })
5165 .collect();
5166 let response = ClientControlResponse::SupervisorRoutes { modules };
5167 Ok(vec![control_response_body_frame(
5168 &frame,
5169 &response,
5170 "ClientControlResponse::SupervisorRoutes",
5171 )?])
5172 }
5173
5174 async fn handle_supervisor_provenance(
5175 &self,
5176 frame: Frame,
5177 module_id: Option<String>,
5178 ) -> Result<Vec<Frame>, RouterError> {
5179 let mut selected = if let Some(module_id) = module_id {
5180 let Some(module) = self.supervisor.get(&module_id) else {
5181 return Ok(vec![control_error_frame(
5182 &frame,
5183 "unknown_module",
5184 format!("module_id '{module_id}' is not supervised"),
5185 )?]);
5186 };
5187 vec![module]
5188 } else {
5189 self.supervisor.list()
5190 };
5191
5192 let mut modules = Vec::with_capacity(selected.len());
5193 for module in selected.drain(..) {
5194 let (status, observed_image) = module
5195 .status_and_running_image_agreement()
5196 .await
5197 .map_err(|err| {
5198 RouterError::backend(
5199 0,
5200 frame.header.corr,
5201 format!("failed to read supervisor status: {err}"),
5202 )
5203 })?;
5204 let module_declared = self
5205 .registry
5206 .get_module(&status.module_id)
5207 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5208 .and_then(|registration| registration.manifest.provenance)
5209 .map(|build| ModuleDeclaredProvenance::Reported { build })
5210 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5211 #[cfg(test)]
5212 let running_image = match &self.provenance_probe_override {
5213 Some(result) => result.clone(),
5214 None => observed_image,
5215 };
5216 #[cfg(not(test))]
5217 let running_image = observed_image;
5218 modules.push(SupervisorModuleProvenance {
5219 module_id: status.module_id,
5220 module_declared,
5221 daemon_observed: SupervisorObservedProcess {
5222 launch_nonce_source: status.launch_nonce_source,
5223 pid: status.pid,
5224 spawned_at_ms: status.spawned_at_ms,
5225 spawned_from: status.spawned_from,
5226 running_image,
5227 },
5228 });
5229 }
5230 let daemon = SupervisorDaemonProvenance {
5231 daemon_build: self.daemon_provenance.build.clone(),
5232 daemon_observed: DaemonObservedProcess {
5233 pid: self.daemon_provenance.pid,
5234 started_at_ms: self
5235 .daemon_provenance
5236 .start_clock
5237 .map(|clock| clock.started_at_ms())
5238 .or(self.daemon_provenance.started_at_ms),
5239 running_image: self
5240 .daemon_provenance
5241 .probe
5242 .observe(
5243 self.daemon_provenance.pid,
5244 self.daemon_provenance.executable_path.as_deref(),
5245 self.daemon_provenance.executable_identity.clone(),
5246 self.daemon_provenance.process_start_time,
5247 )
5248 .await,
5249 },
5250 };
5251 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5252 Ok(vec![control_response_body_frame(
5253 &frame,
5254 &response,
5255 "ClientControlResponse::SupervisorProvenance",
5256 )?])
5257 }
5258
5259 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5260 self.refresh_capability_requirements();
5261 let generation = self
5262 .registry
5263 .generation()
5264 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5265 let modules = self
5266 .supervisor
5267 .list()
5268 .into_iter()
5269 .map(|module| {
5270 let status = module.status_for_control("health").map_err(|err| {
5271 RouterError::backend(
5272 0,
5273 frame.header.corr,
5274 format!("failed to read supervisor health: {err}"),
5275 )
5276 })?;
5277 let module_id = status.module_id;
5278 let capability_detail = self
5279 .capability_evaluator
5280 .required_problem_detail(&module_id);
5281 Ok(SupervisorHealthEntry {
5282 module_id,
5283 status: status.health.status,
5284 detail: append_capability_problem_detail(
5285 status.health.detail,
5286 capability_detail,
5287 ),
5288 metrics: status.health.metrics,
5289 consecutive_failures: status.health.consecutive_failures,
5290 late_answer_count: status.health.late_answer_count,
5291 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5292 last_action: status.health.last_action,
5293 last_action_ms: status.health.last_action_ms,
5294 last_probe_ms: status.health.last_probe_ms,
5295 })
5296 })
5297 .collect::<Result<Vec<_>, RouterError>>()?;
5298 let response = ClientControlResponse::SupervisorHealth {
5299 generation,
5300 modules,
5301 };
5302 Ok(vec![control_response_body_frame(
5303 &frame,
5304 &response,
5305 "ClientControlResponse::SupervisorHealth",
5306 )?])
5307 }
5308
5309 async fn handle_supervisor_restart(
5310 &self,
5311 frame: Frame,
5312 module_id: String,
5313 drain_timeout_ms: Option<u64>,
5314 requester: Principal,
5315 ) -> Result<Vec<Frame>, RouterError> {
5316 let operation_lock = self.supervisor.operation_lock();
5317 let _operation_guard = operation_lock.lock().await;
5318 let Some(module) = self.supervisor.get(&module_id) else {
5319 return Ok(vec![control_error_frame(
5320 &frame,
5321 "unknown_module",
5322 format!("module_id '{module_id}' is not supervised"),
5323 )?]);
5324 };
5325
5326 self.route_outages.mark_operator_action(&module_id);
5327 if let Err(err) = module
5328 .restart_requested(drain_timeout_ms, Some(requester))
5329 .await
5330 {
5331 self.route_outages
5332 .operator_action_ended_unrefused(&module_id);
5333 let (code, message) = match err {
5334 crate::supervise::SuperviseError::Disabled { .. } => {
5335 ("module_disabled", err.to_string())
5336 }
5337 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5338 ("swap_in_progress", err.to_string())
5339 }
5340 _ => (
5341 "target_unavailable",
5342 format!("failed to restart module_id '{module_id}': {err}"),
5343 ),
5344 };
5345 return Ok(vec![control_error_frame(&frame, code, message)?]);
5346 }
5347
5348 let response = ClientControlResponse::SupervisorAck {
5349 module_id,
5350 applied: true,
5351 };
5352 Ok(vec![control_response_body_frame(
5353 &frame,
5354 &response,
5355 "ClientControlResponse::SupervisorAck",
5356 )?])
5357 }
5358
5359 async fn handle_supervisor_swap(
5363 &self,
5364 frame: Frame,
5365 module_id: String,
5366 ready_timeout_ms: Option<u64>,
5367 requester: Principal,
5368 ) -> Result<Vec<Frame>, RouterError> {
5369 let module = {
5376 let operation_lock = self.supervisor.operation_lock();
5377 let _operation_guard = operation_lock.lock().await;
5378 self.supervisor.get(&module_id)
5379 };
5380 let Some(module) = module else {
5381 return Ok(vec![control_error_frame(
5382 &frame,
5383 "unknown_module",
5384 format!("module_id '{module_id}' is not supervised"),
5385 )?]);
5386 };
5387
5388 self.route_outages.mark_operator_action(&module_id);
5389 if let Err(err) = module
5390 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5391 .await
5392 {
5393 self.route_outages
5394 .operator_action_ended_unrefused(&module_id);
5395 use crate::supervise::SuperviseError;
5396 let message = err.to_string();
5397 let error = match err {
5398 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5399 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5400 code: "swap_refused".to_string(),
5401 message,
5402 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5403 },
5404 SuperviseError::SwapFailed {
5405 arm,
5406 candidate_exit,
5407 ..
5408 } => ErrorBody {
5409 code: "swap_failed".to_string(),
5410 message,
5411 detail: Some(serde_json::json!({
5412 "arm": arm.as_str(),
5413 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5414 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5415 })),
5416 },
5417 _ => ErrorBody::new(
5418 "target_unavailable",
5419 format!("failed to swap module_id '{module_id}': {message}"),
5420 ),
5421 };
5422 return Ok(vec![control_error_body_frame(&frame, error)?]);
5423 }
5424 self.route_outages
5428 .operator_action_ended_unrefused(&module_id);
5429
5430 let response = ClientControlResponse::SupervisorAck {
5431 module_id,
5432 applied: true,
5433 };
5434 Ok(vec![control_response_body_frame(
5435 &frame,
5436 &response,
5437 "ClientControlResponse::SupervisorAck",
5438 )?])
5439 }
5440
5441 async fn handle_supervisor_reload(
5442 &self,
5443 frame: Frame,
5444 module_id: String,
5445 requester: Principal,
5446 ) -> Result<Vec<Frame>, RouterError> {
5447 let operation_lock = self.supervisor.operation_lock();
5448 let _operation_guard = operation_lock.lock().await;
5449 let Some(module) = self.supervisor.get(&module_id) else {
5450 return Ok(vec![control_error_frame(
5451 &frame,
5452 "unknown_module",
5453 format!("module_id '{module_id}' is not supervised"),
5454 )?]);
5455 };
5456
5457 self.route_outages.mark_operator_action(&module_id);
5458 if let Err(err) = module.reload_requested(Some(requester)).await {
5459 self.route_outages
5460 .operator_action_ended_unrefused(&module_id);
5461 let (code, message) = match err {
5462 crate::supervise::SuperviseError::Disabled { .. } => {
5463 ("module_disabled", err.to_string())
5464 }
5465 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5466 ("swap_in_progress", err.to_string())
5467 }
5468 _ => (
5469 "reload_failed",
5470 format!("failed to reload module_id '{module_id}': {err}"),
5471 ),
5472 };
5473 return Ok(vec![control_error_frame(&frame, code, message)?]);
5474 }
5475
5476 let response = ClientControlResponse::SupervisorAck {
5477 module_id,
5478 applied: true,
5479 };
5480 Ok(vec![control_response_body_frame(
5481 &frame,
5482 &response,
5483 "ClientControlResponse::SupervisorAck",
5484 )?])
5485 }
5486
5487 async fn handle_supervisor_rescan(
5488 &self,
5489 frame: Frame,
5490 preview: bool,
5491 requester: Principal,
5492 ) -> Result<Vec<Frame>, RouterError> {
5493 let Some(context) = self.rescan.clone() else {
5494 return Ok(vec![control_error_frame(
5495 &frame,
5496 "rescan_unavailable",
5497 "the daemon was not started with a reloadable config path".to_string(),
5498 )?]);
5499 };
5500
5501 let operation_lock = self.supervisor.operation_lock();
5502 let _operation_guard = operation_lock.lock().await;
5503 let loaded = match crate::daemon_config::load_with_activity(&context.config_path) {
5504 Ok(config) => config,
5505 Err(err) => {
5506 return Ok(vec![control_error_frame(
5507 &frame,
5508 "invalid_daemon_config",
5509 format!("supervisor rescan rejected daemon config: {err}"),
5510 )?])
5511 }
5512 };
5513 let Some((config, activity_config)) = loaded else {
5521 return Ok(vec![control_error_frame(
5522 &frame,
5523 "invalid_daemon_config",
5524 format!(
5525 "daemon config not found at {}; refusing to rescan (an absent config would \
5526 retire every supervised module)",
5527 context.config_path.display()
5528 ),
5529 )?]);
5530 };
5531 let (
5532 configured_port,
5533 storage_config,
5534 admission_facts_carrier_module_id,
5535 admission_facts_targets,
5536 scope_authority_owners,
5537 privileged_operator_prompt,
5538 modules,
5539 reserved_capabilities,
5540 ) = (
5541 config.port,
5542 config.storage,
5543 config.admission_facts_carrier_module_id,
5544 config.admission_facts_targets,
5545 config.scope_authority_owners,
5546 config.privileged_operator_prompt,
5547 config.modules,
5548 config.reserved_capabilities,
5549 );
5550
5551 let mut restart_required = Vec::new();
5560 for section in RestartRequiredSection::ALL {
5561 let changed = match section {
5562 RestartRequiredSection::Port => configured_port != context.configured_port,
5563 RestartRequiredSection::Storage => storage_config != context.storage_config,
5564 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5565 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5566 }
5567 RestartRequiredSection::AdmissionFactsTargets => {
5568 admission_facts_targets != context.admission_facts_targets
5569 }
5570 RestartRequiredSection::ScopeAuthorityOwners => {
5571 scope_authority_owners != context.scope_authority_owners
5572 }
5573 RestartRequiredSection::PrivilegedOperatorPrompt => {
5574 privileged_operator_prompt != context.privileged_operator_prompt
5575 }
5576 };
5577 if changed {
5578 restart_required.push(section.label().to_string());
5579 }
5580 }
5581 if !restart_required.is_empty() {
5582 warn!(
5583 config_path = %context.config_path.display(),
5584 sections = %restart_required.join(", "),
5585 "daemon config changed outside the modules section; restart the daemon to apply those changes"
5586 );
5587 }
5588
5589 for configured in &modules {
5590 if let Err(err) = validate_spec(&configured.module_spec()) {
5591 return Ok(vec![control_error_frame(
5592 &frame,
5593 "invalid_daemon_config",
5594 format!("supervisor rescan rejected daemon config: {err}"),
5595 )?]);
5596 }
5597 }
5598
5599 let configured_capabilities = modules
5600 .iter()
5601 .map(|module| (module.module_id.clone(), module.enabled))
5602 .collect::<Vec<_>>();
5603 let preview_capability_warnings = if preview {
5604 let (_, registrations) = self.runtime_capability_snapshot()?;
5605 let current_modules = self
5606 .supervisor
5607 .list()
5608 .into_iter()
5609 .map(|module| module.module_id().to_string())
5610 .collect::<BTreeSet<_>>();
5611 let resulting_modules = configured_capabilities.clone();
5612 let removed = current_modules
5613 .into_iter()
5614 .filter(|module_id| {
5615 !resulting_modules
5616 .iter()
5617 .any(|(configured_id, _)| configured_id == module_id)
5618 })
5619 .collect::<Vec<_>>();
5620 self.capability_evaluator.preview_removal_warnings(
5621 resulting_modules,
5622 &removed,
5623 ®istrations,
5624 )
5625 } else {
5626 Vec::new()
5627 };
5628 let result = match self
5629 .reconcile_supervised_modules(&context.supervisor, modules, preview, requester.clone())
5630 .await
5631 {
5632 Ok(result) => result,
5633 Err(message) => {
5634 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5635 }
5636 };
5637 if !preview {
5638 if let Some(ledger) = &self.activity {
5639 ledger.module_rescanned(&result.added, &result.removed, requester);
5640 }
5641 if let Err(code) = self.configure_activity(activity_config).await {
5642 warn!(
5643 code,
5644 "activity retention could not be applied during rescan"
5645 );
5646 }
5647 self.capability_evaluator
5648 .configure(configured_capabilities, reserved_capabilities);
5649 self.capability_evaluator.wake_deadline_loop();
5650 self.refresh_capability_requirements();
5651 }
5652 let mut result = result;
5653 result.restart_required = restart_required;
5654 result.capability_warnings = preview_capability_warnings;
5655 let response = ClientControlResponse::SupervisorRescan { result };
5656 Ok(vec![control_response_body_frame(
5657 &frame,
5658 &response,
5659 "ClientControlResponse::SupervisorRescan",
5660 )?])
5661 }
5662
5663 async fn handle_supervisor_release_reserved(
5664 &self,
5665 frame: Frame,
5666 module_id: String,
5667 ) -> Result<Vec<Frame>, RouterError> {
5668 let Some(context) = self.rescan.clone() else {
5669 return Ok(vec![control_error_frame(
5670 &frame,
5671 "release_unavailable",
5672 "reserved-id release requires a daemon started with a reloadable config path",
5673 )?]);
5674 };
5675 let operation_lock = self.supervisor.operation_lock();
5676 let _operation_guard = operation_lock.lock().await;
5677 let loaded = match crate::daemon_config::load(&context.config_path) {
5678 Ok(Some(config)) => config,
5679 Ok(None) => {
5680 return Ok(vec![control_error_frame(
5681 &frame,
5682 "invalid_daemon_config",
5683 format!(
5684 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5685 context.config_path.display()
5686 ),
5687 )?])
5688 }
5689 Err(err) => {
5690 return Ok(vec![control_error_frame(
5691 &frame,
5692 "invalid_daemon_config",
5693 format!("unable to verify reserved-id release against daemon config: {err}"),
5694 )?])
5695 }
5696 };
5697 if loaded
5698 .modules
5699 .iter()
5700 .any(|configured| configured.module_id == module_id)
5701 {
5702 return Ok(vec![control_error_frame(
5703 &frame,
5704 "reserved_module_configured",
5705 format!(
5706 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5707 ),
5708 )?]);
5709 }
5710 if !self.supervisor.release_retained_reserved_gate(&module_id) {
5711 return Ok(vec![control_error_frame(
5712 &frame,
5713 "reserved_gate_not_retained",
5714 format!(
5715 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
5716 ),
5717 )?]);
5718 }
5719
5720 let response = ClientControlResponse::SupervisorAck {
5721 module_id,
5722 applied: true,
5723 };
5724 Ok(vec![control_response_body_frame(
5725 &frame,
5726 &response,
5727 "ClientControlResponse::SupervisorAck",
5728 )?])
5729 }
5730
5731 async fn reconcile_supervised_modules(
5740 &self,
5741 supervisor: &Supervisor,
5742 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
5743 preview: bool,
5744 requester: Principal,
5745 ) -> Result<SupervisorRescanResult, String> {
5746 let mut current = BTreeMap::new();
5747 for module in self.supervisor.list() {
5748 let (spec, health) = module.configuration().map_err(|err| {
5749 format!(
5750 "failed to read configuration for module_id '{}': {err}",
5751 module.module_id()
5752 )
5753 })?;
5754 let enabled = module
5755 .status()
5756 .map_err(|err| {
5757 format!(
5758 "failed to read status for module_id '{}': {err}",
5759 module.module_id()
5760 )
5761 })?
5762 .enabled;
5763 current.insert(
5764 module.module_id().to_string(),
5765 (module, spec, health, enabled),
5766 );
5767 }
5768 let configured = configured_modules
5769 .into_iter()
5770 .map(|module| (module.module_id.clone(), module))
5771 .collect::<BTreeMap<_, _>>();
5772
5773 let added = configured
5774 .keys()
5775 .filter(|module_id| !current.contains_key(*module_id))
5776 .cloned()
5777 .collect::<Vec<_>>();
5778 let removed = current
5779 .keys()
5780 .filter(|module_id| !configured.contains_key(*module_id))
5781 .cloned()
5782 .collect::<Vec<_>>();
5783 let mut changed_pending_reload = Vec::new();
5784 let mut configuration_changes = BTreeSet::new();
5785 let mut enabled_changes = BTreeSet::new();
5786 let mut unchanged = 0_u32;
5787
5788 for (module_id, configured_module) in &configured {
5789 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5790 else {
5791 continue;
5792 };
5793 let launch_changed = *current_spec != configured_module.module_spec();
5797 let configuration_changed =
5798 launch_changed || *current_health != configured_module.health;
5799 let enabled_changed = *current_enabled != configured_module.enabled;
5800 if configuration_changed {
5801 configuration_changes.insert(module_id.clone());
5802 }
5803 if launch_changed {
5804 changed_pending_reload.push(module_id.clone());
5805 }
5806 if enabled_changed {
5807 enabled_changes.insert(module_id.clone());
5808 }
5809 if !configuration_changed && !enabled_changed {
5810 unchanged = unchanged.saturating_add(1);
5811 }
5812 }
5813
5814 if preview {
5820 return Ok(SupervisorRescanResult {
5821 added,
5822 removed,
5823 changed_pending_reload,
5824 enabled_changes: enabled_changes.iter().cloned().collect(),
5825 unchanged,
5826 preview: true,
5827 restart_required: Vec::new(),
5831 capability_warnings: Vec::new(),
5832 });
5833 }
5834
5835 for module_id in &removed {
5836 let module = ¤t
5837 .get(module_id)
5838 .expect("removed module came from current supervisor state")
5839 .0;
5840 module
5841 .retire_requested(Some(requester.clone()))
5842 .await
5843 .map_err(|err| {
5844 format!("failed to retire module_id '{module_id}' during rescan: {err}")
5845 })?;
5846 self.supervisor.record_rescan_removal(module_id);
5880 self.supervisor.retire(module_id);
5881 self.route_outages.forget(module_id);
5882 }
5883
5884 for module_id in configured.keys() {
5885 let Some((module, _, _, _)) = current.get(module_id) else {
5886 continue;
5887 };
5888 let configured_module = configured
5889 .get(module_id)
5890 .expect("configured module id came from configured map");
5891 if configuration_changes.contains(module_id) {
5892 module
5893 .update_configuration(
5894 configured_module.module_spec(),
5895 configured_module.health.clone(),
5896 configured_module.drain_timeout_ms,
5897 )
5898 .await
5899 .map_err(|err| {
5900 format!(
5901 "failed to update module_id '{module_id}' configuration during rescan: {err}"
5902 )
5903 })?;
5904 }
5905 if enabled_changes.contains(module_id) {
5906 self.route_outages.mark_operator_action(module_id);
5909 module
5910 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
5911 .await
5912 .map_err(|err| {
5913 self.route_outages.operator_action_ended_unrefused(module_id);
5914 format!(
5915 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
5916 configured_module.enabled
5917 )
5918 })?;
5919 }
5920 }
5921
5922 for module_id in &added {
5923 let configured_module = configured
5924 .get(module_id)
5925 .expect("added module id came from configured map");
5926 supervisor
5927 .supervise_configured_requested(
5928 configured_module.module_spec(),
5929 configured_module.enabled,
5930 configured_module.health.clone(),
5931 configured_module.drain_timeout_ms,
5932 configured_module.restart,
5933 Some(requester.clone()),
5934 )
5935 .map_err(|err| {
5936 format!("failed to add module_id '{module_id}' during rescan: {err}")
5937 })?;
5938 }
5939
5940 Ok(SupervisorRescanResult {
5941 added,
5942 removed,
5943 changed_pending_reload,
5944 enabled_changes: enabled_changes.iter().cloned().collect(),
5945 unchanged,
5946 preview: false,
5947 restart_required: Vec::new(),
5950 capability_warnings: Vec::new(),
5951 })
5952 }
5953
5954 async fn handle_supervisor_set_enabled(
5955 &self,
5956 frame: Frame,
5957 module_id: String,
5958 enabled: bool,
5959 requester: Principal,
5960 ) -> Result<Vec<Frame>, RouterError> {
5961 let operation_lock = self.supervisor.operation_lock();
5962 let _operation_guard = operation_lock.lock().await;
5963 let Some(module) = self.supervisor.get(&module_id) else {
5964 return Ok(vec![control_error_frame(
5965 &frame,
5966 "unknown_module",
5967 format!("module_id '{module_id}' is not supervised"),
5968 )?]);
5969 };
5970
5971 self.route_outages.mark_operator_action(&module_id);
5974 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
5975 Ok(applied) => applied,
5976 Err(err) => {
5977 self.route_outages
5978 .operator_action_ended_unrefused(&module_id);
5979 return Ok(vec![control_error_frame(
5980 &frame,
5981 "target_unavailable",
5982 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
5983 )?]);
5984 }
5985 };
5986 if !applied {
5987 self.route_outages
5990 .operator_action_ended_unrefused(&module_id);
5991 }
5992
5993 self.capability_evaluator.wake_deadline_loop();
5994 self.refresh_capability_requirements();
5995 let response = ClientControlResponse::SupervisorAck { module_id, applied };
5996 Ok(vec![control_response_body_frame(
5997 &frame,
5998 &response,
5999 "ClientControlResponse::SupervisorAck",
6000 )?])
6001 }
6002
6003 async fn handle_supervisor_health_probe(
6004 &self,
6005 frame: Frame,
6006 module_id: String,
6007 ) -> Result<Vec<Frame>, RouterError> {
6008 self.refresh_capability_requirements();
6009 let Some(registration) = self
6010 .registry
6011 .get_module(&module_id)
6012 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6013 else {
6014 return Ok(vec![control_error_frame(
6015 &frame,
6016 "unknown_module",
6017 format!("module_id '{module_id}' is not registered"),
6018 )?]);
6019 };
6020
6021 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6034 {
6035 return Ok(vec![control_error_frame(
6036 &frame,
6037 "health_not_advertised",
6038 format!("module_id '{module_id}' did not advertise health.check"),
6039 )?]);
6040 }
6041
6042 let deadline = Instant::now() + self.health_probe_timeout;
6043 let pending = match self.forwarding.begin_module_control_rpc_for(
6044 &module_id,
6045 MODULE_CONTROL_OP_HEALTH_CHECK,
6046 deadline,
6047 ) {
6048 Ok(pending) => pending,
6049 Err(err) => {
6050 return Ok(vec![control_error_frame(
6051 &frame,
6052 forwarding_error_code(&err),
6053 err.to_string(),
6054 )?])
6055 }
6056 };
6057
6058 let PendingModuleControlRpc {
6059 endpoint,
6060 module_sink,
6061 negotiated_ver,
6062 corr: probe_corr,
6063 receiver,
6064 } = pending;
6065 let mut guard =
6066 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6067 let probe_body =
6068 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6069 RouterError::backend(
6070 0,
6071 frame.header.corr,
6072 format!("failed to encode health.check request: {err}"),
6073 )
6074 })?;
6075 let probe_frame = Frame::build_with_version(
6076 negotiated_ver,
6077 FrameType::Request,
6078 control_flags(),
6079 0,
6080 0,
6081 probe_corr,
6082 probe_body,
6083 )
6084 .map_err(RouterError::FrameBuild)?;
6085
6086 if let Err(err) = module_sink.send(probe_frame).await {
6087 return Ok(vec![control_error_frame(
6088 &frame,
6089 "target_unavailable",
6090 err.to_string(),
6091 )?]);
6092 }
6093
6094 match timeout_at(deadline, receiver).await {
6095 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6096 guard.disarm();
6097 let Some(report) = response.health_report() else {
6098 return Ok(vec![control_error_frame(
6099 &frame,
6100 "invalid_control_body",
6101 "health.check RPC returned a non-health response",
6102 )?]);
6103 };
6104 let HealthReport {
6109 status,
6110 detail,
6111 metrics,
6112 } = report;
6113 let capability_detail = self
6114 .capability_evaluator
6115 .required_problem_detail(&module_id);
6116 let response = ClientControlResponse::SupervisorHealthProbe {
6117 module_id,
6118 status,
6119 detail: append_capability_problem_detail(detail, capability_detail),
6120 metrics,
6121 };
6122 Ok(vec![control_response_body_frame(
6123 &frame,
6124 &response,
6125 "ClientControlResponse::SupervisorHealthProbe",
6126 )?])
6127 }
6128 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6129 guard.disarm();
6130 Ok(vec![control_error_body_frame(&frame, body)?])
6131 }
6132 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6133 guard.disarm();
6134 Ok(vec![control_error_frame(
6135 &frame,
6136 "target_unavailable",
6137 message,
6138 )?])
6139 }
6140 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6141 guard.disarm();
6142 Ok(vec![control_error_frame(
6143 &frame,
6144 "invalid_control_body",
6145 message,
6146 )?])
6147 }
6148 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6149 guard.disarm();
6150 Ok(vec![control_error_frame(
6151 &frame,
6152 "invalid_control_body",
6153 format!("expected module-control op '{expected}', got '{actual}'"),
6154 )?])
6155 }
6156 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6157 guard.disarm();
6158 Ok(vec![control_error_frame(
6159 &frame,
6160 "module_timeout",
6161 format!(
6162 "module_id '{module_id}' answered health.check after {:?}",
6163 self.health_probe_timeout
6164 ),
6165 )?])
6166 }
6167 Ok(Err(_)) => Ok(vec![control_error_frame(
6168 &frame,
6169 "target_unavailable",
6170 "health.check waiter was canceled before the module responded",
6171 )?]),
6172 Err(_) => Ok(vec![control_error_frame(
6173 &frame,
6174 "module_timeout",
6175 format!(
6176 "module_id '{module_id}' did not answer health.check within {:?}",
6177 self.health_probe_timeout
6178 ),
6179 )?]),
6180 }
6181 }
6182
6183 fn supervisor_status(
6184 &self,
6185 module_id: &str,
6186 corr: u64,
6187 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6188 self.supervisor
6189 .get(module_id)
6190 .map(|module| {
6191 let warming = module.is_warming_for_control("status").map_err(|err| {
6192 RouterError::backend(
6193 0,
6194 corr,
6195 format!(
6196 "failed to read supervisor warming state for module_id '{module_id}': {err}"
6197 ),
6198 )
6199 })?;
6200 module.status_for_control("status").map_err(|err| {
6201 RouterError::backend(
6202 0,
6203 corr,
6204 format!(
6205 "failed to read supervisor status for module_id '{module_id}': {err}"
6206 ),
6207 )
6208 }).map(|status| (status, warming))
6209 })
6210 .transpose()
6211 }
6212
6213 fn guard_module_control_op(
6214 &self,
6215 frame: &Frame,
6216 module_id: &str,
6217 op: &str,
6218 ) -> Result<Option<Frame>, RouterError> {
6219 if self.module_grants_op(module_id, op, frame.header.corr)? {
6220 return Ok(None);
6221 }
6222
6223 Ok(Some(control_error_frame(
6224 frame,
6225 "op_not_allowed",
6226 format!("module_id '{module_id}' did not grant control op '{op}'"),
6227 )?))
6228 }
6229
6230 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6231 let Some(registration) = self
6232 .registry
6233 .get_module(module_id)
6234 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6235 else {
6236 return Ok(false);
6237 };
6238 Ok(module_registration_grants_op(®istration.control_ops, op))
6239 }
6240
6241 fn handle_status_update(
6242 &self,
6243 endpoint: ModuleEndpointId,
6244 frame: Frame,
6245 ) -> Result<Vec<Frame>, RouterError> {
6246 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6247 Ok(update) => update,
6248 Err(err) => {
6249 if is_known_module_push_op(&frame.body) {
6254 return Ok(vec![control_error_frame(
6255 &frame,
6256 "invalid_control_body",
6257 format!("malformed module control push body: {err}"),
6258 )?]);
6259 }
6260 return Ok(Vec::new());
6261 }
6262 };
6263
6264 match update {
6265 ModuleControlPush::RouteStatus {
6266 route_channel,
6267 route_epoch,
6268 status,
6269 } => {
6270 self.forwarding
6271 .cache_status(endpoint, route_channel, route_epoch, status)
6272 .map_err(RouterError::Forwarding)?;
6273 }
6274 }
6275 Ok(Vec::new())
6276 }
6277
6278 fn handle_route_poll(
6279 &self,
6280 ctx: &RouteCtx,
6281 frame: Frame,
6282 route_channel: u16,
6283 route_epoch: u32,
6284 kind: PollKind,
6285 ) -> Result<Vec<Frame>, RouterError> {
6286 let snapshot = self
6287 .forwarding
6288 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6289 .map_err(RouterError::Forwarding)?;
6290 let response = match (kind, snapshot) {
6291 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6292 ClientControlResponse::RoutePoll {
6293 route_channel,
6294 route_epoch,
6295 status,
6296 live: None,
6297 }
6298 }
6299 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6300 route_channel,
6301 route_epoch,
6302 status: None,
6303 live: None,
6304 },
6305 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6306 let live = self
6332 .process_liveness
6333 .as_ref()
6334 .and_then(|source| source.process_live(&module_id))
6335 .unwrap_or(true);
6336 ClientControlResponse::RoutePoll {
6337 route_channel,
6338 route_epoch,
6339 status: None,
6340 live: Some(live),
6341 }
6342 }
6343 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6344 route_channel,
6345 route_epoch,
6346 status: None,
6347 live: Some(false),
6348 },
6349 };
6350
6351 Ok(vec![control_response_body_frame(
6352 &frame,
6353 &response,
6354 "ClientControlResponse::RoutePoll",
6355 )?])
6356 }
6357
6358 pub(crate) fn observe_module_control_completion(
6359 &self,
6360 completion: ModuleControlRpcCompletion,
6361 ) -> bool {
6362 match completion {
6363 ModuleControlRpcCompletion::Unknown => false,
6364 ModuleControlRpcCompletion::Settled => true,
6365 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6366 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6367 info!(
6368 module_id = %module_id,
6369 latency_ms,
6370 "late health.check answer proves the module is alive"
6371 );
6372 match self
6373 .supervisor
6374 .record_late_health_answer(&module_id, latency_ms)
6375 {
6376 Ok(true) => {}
6377 Ok(false) => debug!(
6378 module_id = %module_id,
6379 latency_ms,
6380 "late health.check answer has no active supervisor snapshot"
6381 ),
6382 Err(err) => warn!(
6383 module_id = %module_id,
6384 latency_ms,
6385 error = %err,
6386 "failed to record late health.check answer"
6387 ),
6388 }
6389 true
6390 }
6391 }
6392 }
6393
6394 fn refuse_to_end_module_connection_for_a_client(
6415 &self,
6416 module_connection_id: ConnectionId,
6417 corr: u64,
6418 err: ForwardingError,
6419 ) -> Result<(), RouterError> {
6420 if let ForwardingError::ConnectionClosing { connection_id } = err {
6421 if connection_id != module_connection_id {
6422 warn!(
6423 module_connection_id = module_connection_id.get(),
6424 client_connection_id = connection_id.get(),
6425 corr,
6426 "dropping a route.bind response for a closing client; the module connection keeps serving"
6427 );
6428 return Ok(());
6429 }
6430 }
6431 Err(RouterError::Forwarding(err))
6432 }
6433
6434 fn handle_module_relay_response(
6435 &self,
6436 connection_id: ConnectionId,
6437 frame: Frame,
6438 ) -> Result<Vec<Frame>, RouterError> {
6439 let mut secondary_error = None;
6440 let outcome = match frame.header.ty {
6441 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6442 Ok(probe) if probe.op == "route.bind" => {
6443 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6444 Ok(ModuleControlResponse::RouteBindAck {}) => {
6445 RouteBindRelayOutcome::Accepted
6446 }
6447 Ok(other) => {
6448 let message =
6449 format!("route.bind response carried unexpected body: {other:?}");
6450 secondary_error = Some(control_error_frame(
6451 &frame,
6452 "invalid_control_body",
6453 message.clone(),
6454 )?);
6455 RouteBindRelayOutcome::ModuleGone(message)
6456 }
6457 Err(err) => {
6458 let message = format!("malformed route.bind response body: {err}");
6459 secondary_error = Some(control_error_frame(
6460 &frame,
6461 "invalid_control_body",
6462 message.clone(),
6463 )?);
6464 RouteBindRelayOutcome::ModuleGone(message)
6465 }
6466 }
6467 }
6468 Ok(probe) => {
6469 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6470 {
6471 Ok(response) => ModuleControlRpcOutcome::Response(response),
6472 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6473 "malformed {} response body: {err}",
6474 probe.op
6475 )),
6476 };
6477 let completion = self
6478 .forwarding
6479 .complete_module_control_rpc(
6480 connection_id,
6481 frame.header.corr,
6482 Some(&probe.op),
6483 outcome,
6484 )
6485 .map_err(RouterError::Forwarding)?;
6486 if !self.observe_module_control_completion(completion) {
6487 debug!(
6488 connection_id = connection_id.get(),
6489 corr = frame.header.corr,
6490 op = %probe.op,
6491 "dropping late or unknown module-control RPC response"
6492 );
6493 }
6494 return Ok(Vec::new());
6495 }
6496 Err(err) => {
6497 if let Some(expected_op) = self
6498 .forwarding
6499 .pending_module_control_op(connection_id, frame.header.corr)
6500 .map_err(RouterError::Forwarding)?
6501 {
6502 let completion = self
6503 .forwarding
6504 .complete_module_control_rpc(
6505 connection_id,
6506 frame.header.corr,
6507 None,
6508 ModuleControlRpcOutcome::MalformedResponse(format!(
6509 "malformed {expected_op} response body: {err}"
6510 )),
6511 )
6512 .map_err(RouterError::Forwarding)?;
6513 if !self.observe_module_control_completion(completion) {
6514 debug!(
6515 connection_id = connection_id.get(),
6516 corr = frame.header.corr,
6517 "dropping late malformed module-control RPC response"
6518 );
6519 }
6520 return Ok(Vec::new());
6521 }
6522 let message = format!("malformed route.bind response body: {err}");
6523 secondary_error = Some(control_error_frame(
6524 &frame,
6525 "invalid_control_body",
6526 message.clone(),
6527 )?);
6528 RouteBindRelayOutcome::ModuleGone(message)
6529 }
6530 },
6531 FrameType::Error => {
6532 if self
6533 .forwarding
6534 .pending_module_control_op(connection_id, frame.header.corr)
6535 .map_err(RouterError::Forwarding)?
6536 .is_some()
6537 {
6538 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6539 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
6540 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6541 "malformed module-control ERROR body: {err}"
6542 )),
6543 };
6544 let completion = self
6545 .forwarding
6546 .complete_module_control_rpc(
6547 connection_id,
6548 frame.header.corr,
6549 None,
6550 outcome,
6551 )
6552 .map_err(RouterError::Forwarding)?;
6553 if !self.observe_module_control_completion(completion) {
6554 debug!(
6555 connection_id = connection_id.get(),
6556 corr = frame.header.corr,
6557 "dropping late or unknown module-control RPC error"
6558 );
6559 }
6560 return Ok(Vec::new());
6561 }
6562 match serde_json::from_slice::<ErrorBody>(&frame.body) {
6563 Ok(body) => RouteBindRelayOutcome::Rejected(body),
6564 Err(err) => {
6565 let message = format!("malformed route.bind ERROR body: {err}");
6566 secondary_error = Some(control_error_frame(
6567 &frame,
6568 "invalid_control_body",
6569 message.clone(),
6570 )?);
6571 RouteBindRelayOutcome::ModuleGone(message)
6572 }
6573 }
6574 }
6575 ty => {
6576 return Ok(vec![control_error_frame(
6577 &frame,
6578 "unsupported_control_frame",
6579 format!("unsupported module channel-0 frame {ty:?}"),
6580 )?])
6581 }
6582 };
6583
6584 let settled =
6585 self.forwarding
6586 .complete_pending_relay(connection_id, frame.header.corr, outcome);
6587 let completion = match settled {
6588 Ok(completion) => completion,
6589 Err(err) => {
6590 self.refuse_to_end_module_connection_for_a_client(
6591 connection_id,
6592 frame.header.corr,
6593 err,
6594 )?;
6595 return Ok(secondary_error.into_iter().collect());
6596 }
6597 };
6598 if let Some(target) = completion.abandoned.as_ref() {
6599 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6600 }
6601 if !completion.settled {
6602 debug!(
6603 connection_id = connection_id.get(),
6604 corr = frame.header.corr,
6605 frame_type = ?frame.header.ty,
6606 "dropping late or unknown route.bind relay response"
6607 );
6608 }
6609 Ok(secondary_error.into_iter().collect())
6610 }
6611
6612 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6613 debug!(connection_id = connection_id.get(), "handling GOODBYE");
6614 self.cleanup_connection_with_end_reason(
6618 connection_id,
6619 RegistrationEndReason::ExplicitGoodbye,
6620 )
6621 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6622 Ok(Vec::new())
6623 }
6624}
6625
6626impl Default for ControlHandler {
6627 fn default() -> Self {
6628 Self::new(Arc::new(Registry::default()))
6629 }
6630}
6631
6632impl crate::supervise::SwapPromotionObserver for ControlHandler {
6633 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6634 self.apply_registration_capabilities(registration);
6635 }
6636}
6637
6638fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6639 CapabilityRequirementStatus {
6640 consumer: status.consumer,
6641 capability: status.capability,
6642 need: match status.need {
6643 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6644 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6645 },
6646 verdict: status.verdict.as_str().to_string(),
6647 episode_seq: status.episode_seq,
6648 config_satisfiable: status.config_satisfiable,
6649 runtime_available: status.runtime_available,
6650 detail: status.detail,
6651 }
6652}
6653
6654fn append_capability_problem_detail(
6655 detail: Option<String>,
6656 capability_detail: Option<String>,
6657) -> Option<String> {
6658 match (detail, capability_detail) {
6659 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
6660 (Some(detail), None) => Some(detail),
6661 (None, Some(capability_detail)) => Some(capability_detail),
6662 (None, None) => None,
6663 }
6664}
6665
6666fn subc_ops() -> Vec<String> {
6667 SUBC_CONTROL_OPS
6668 .iter()
6669 .map(|op| (*op).to_string())
6670 .collect()
6671}
6672
6673fn module_subc_ops() -> Vec<String> {
6674 SUBC_CONTROL_OPS
6675 .iter()
6676 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
6677 .map(|op| (*op).to_string())
6678 .collect()
6679}
6680
6681#[cfg(test)]
6682fn module_baseline_control_ops() -> Vec<String> {
6683 MODULE_BASELINE_CONTROL_OPS
6684 .iter()
6685 .map(|op| (*op).to_string())
6686 .collect()
6687}
6688
6689fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
6690 let mut seen = HashSet::new();
6691 let mut effective = Vec::new();
6692 for op in MODULE_BASELINE_CONTROL_OPS {
6693 if seen.insert((*op).to_string()) {
6694 effective.push((*op).to_string());
6695 }
6696 }
6697 for op in declared.unwrap_or_default() {
6698 if seen.insert(op.clone()) {
6699 effective.push(op);
6700 }
6701 }
6702 effective
6703}
6704
6705fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
6706 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
6707}
6708
6709fn target_module_id(target: &RouteTarget) -> &str {
6710 match target {
6711 RouteTarget::ToolProvider { module_id }
6712 | RouteTarget::ManagementSurface { module_id }
6713 | RouteTarget::InternalService { module_id, .. } => module_id,
6714 }
6715}
6716
6717fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
6718 roles.iter().any(|role| match (target, role) {
6719 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
6720 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
6721 (
6722 RouteTarget::InternalService { service_id, .. },
6723 ProviderRole::InternalService {
6724 service_id: provided,
6725 ..
6726 },
6727 ) => service_id == provided,
6728 _ => false,
6729 })
6730}
6731
6732fn is_routable_role(role: &ProviderRole) -> bool {
6733 matches!(
6734 role,
6735 ProviderRole::ToolProvider { .. }
6736 | ProviderRole::ManagementSurface { .. }
6737 | ProviderRole::InternalService { .. }
6738 )
6739}
6740
6741#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6742enum ControlRequestBodyError {
6743 UnknownOp,
6744 InvalidBody,
6745}
6746
6747#[derive(Debug, Deserialize)]
6748struct ControlOpProbe {
6749 op: String,
6750}
6751
6752const MODULE_PUSH_OPS: &[&str] = &["route.status"];
6755
6756fn is_known_module_push_op(body: &[u8]) -> bool {
6757 serde_json::from_slice::<ControlOpProbe>(body)
6758 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
6759 .unwrap_or(false)
6760}
6761
6762fn is_known_module_request_op(body: &[u8]) -> bool {
6763 serde_json::from_slice::<ControlOpProbe>(body)
6764 .map(|probe| is_module_to_subc_op(&probe.op))
6765 .unwrap_or(false)
6766}
6767
6768fn is_module_to_subc_op(op: &str) -> bool {
6769 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
6770}
6771
6772fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
6773 debug!(
6774 op = %op,
6775 connection_id = connection_id.get(),
6776 corr,
6777 "control dispatch"
6778 );
6779}
6780
6781fn log_slow_control_dispatch(
6782 dispatch_started_at: Option<StdInstant>,
6783 op: &'static str,
6784 connection_id: ConnectionId,
6785 corr: u64,
6786 route_open_timings: Option<&RouteOpenTimings>,
6787) {
6788 let Some(dispatch_started_at) = dispatch_started_at else {
6789 return;
6790 };
6791 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
6793 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
6794 if let Some(timings) = route_open_timings {
6795 warn!(
6796 op = %op,
6797 connection_id = connection_id.get(),
6798 corr,
6799 elapsed_ms = elapsed.as_millis() as u64,
6800 module_id = ?timings.module_id,
6801 principal = %timings.principal,
6802 admission_ms = timings.admission.as_millis() as u64,
6803 project_root_ms = timings.project_root.as_millis() as u64,
6804 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
6805 bind_send_ms = timings.bind_send.as_millis() as u64,
6806 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
6807 "slow control dispatch"
6808 );
6809 return;
6810 }
6811 warn!(
6812 op = %op,
6813 connection_id = connection_id.get(),
6814 corr,
6815 elapsed_ms = elapsed.as_millis() as u64,
6816 "slow control dispatch"
6817 );
6818 }
6819}
6820
6821#[cfg(test)]
6823type TestCallerLookup =
6824 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
6825
6826fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
6828 match request {
6829 ClientControlRequest::SupervisorRestart { module_id, .. } => {
6830 Some(PrivilegedOperation::Restart { module_id })
6831 }
6832 ClientControlRequest::SupervisorSwap { module_id, .. } => {
6833 Some(PrivilegedOperation::Swap { module_id })
6834 }
6835 ClientControlRequest::SupervisorReload { module_id } => {
6836 Some(PrivilegedOperation::Reload { module_id })
6837 }
6838 ClientControlRequest::SupervisorRescan { preview } => {
6839 if *preview {
6840 None
6841 } else {
6842 Some(PrivilegedOperation::Rescan)
6843 }
6844 }
6845 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
6846 Some(PrivilegedOperation::SetEnabled {
6847 module_id,
6848 enabled: *enabled,
6849 })
6850 }
6851 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
6852 Some(PrivilegedOperation::ReleaseReserved { module_id })
6853 }
6854 ClientControlRequest::ServerDescribe {}
6855 | ClientControlRequest::CatalogList { .. }
6856 | ClientControlRequest::RouteOpen { .. }
6857 | ClientControlRequest::RoutePoll { .. }
6858 | ClientControlRequest::SupervisorList {}
6859 | ClientControlRequest::SupervisorHealth {}
6860 | ClientControlRequest::SupervisorHealthProbe { .. }
6861 | ClientControlRequest::SupervisorRoutes { .. }
6862 | ClientControlRequest::SupervisorProvenance { .. }
6863 | ClientControlRequest::SupervisorStderrTail { .. }
6864 | ClientControlRequest::SupervisorTerminals { .. }
6865 | ClientControlRequest::SupervisorSpawnSnapshot {}
6866 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
6867 }
6868}
6869
6870fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
6871 match request {
6872 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
6873 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
6874 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
6875 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
6876 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
6877 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
6878 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
6879 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
6880 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
6881 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
6882 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
6883 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
6884 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
6885 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
6886 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
6887 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
6888 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
6889 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
6890 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
6891 }
6892}
6893
6894fn principal_label(principal: &Principal) -> String {
6895 match principal {
6896 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
6897 Principal::Direct => "direct".to_string(),
6898 other => format!("{other:?}"),
6899 }
6900}
6901
6902fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
6903 match request {
6904 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
6905 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
6906 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
6907 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
6908 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
6909 }
6910}
6911
6912fn parse_client_control_request(
6913 body: &[u8],
6914) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
6915 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
6916 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6917 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
6918 ControlRequestBodyError::InvalidBody
6919 }
6920 Ok(_) => ControlRequestBodyError::UnknownOp,
6921 Err(_) => ControlRequestBodyError::InvalidBody,
6922 };
6923 (err, classification)
6924 })
6925}
6926
6927fn parse_module_control_request_from_module(
6928 body: &[u8],
6929) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
6930 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
6931 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6932 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
6933 Ok(_) => ControlRequestBodyError::UnknownOp,
6934 Err(_) => ControlRequestBodyError::InvalidBody,
6935 };
6936 (err, classification)
6937 })
6938}
6939
6940#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
6941enum ProviderRoleKind {
6942 ToolProvider,
6943 PipelineStage,
6944 ManagementSurface,
6945 InternalService,
6946}
6947
6948fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
6949 match role {
6950 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
6951 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
6952 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
6953 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
6954 }
6955}
6956
6957fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
6958 roles.iter().map(provider_role_kind).collect()
6959}
6960
6961const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
6964
6965#[derive(Debug, Default, PartialEq, Eq)]
6967struct ScopeOutcomeCounts {
6968 created: usize,
6969 replaced: usize,
6970 updated: usize,
6971 unchanged: usize,
6972 refused: usize,
6973}
6974
6975impl ScopeOutcomeCounts {
6976 fn of(results: &[ScopeRecordResult]) -> Self {
6977 let mut counts = Self::default();
6978 for result in results {
6979 let slot = match result.outcome {
6980 ScopeRecordOutcome::Created => &mut counts.created,
6981 ScopeRecordOutcome::Replaced => &mut counts.replaced,
6982 ScopeRecordOutcome::Updated => &mut counts.updated,
6983 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
6984 ScopeRecordOutcome::Refused => &mut counts.refused,
6985 };
6986 *slot += 1;
6987 }
6988 counts
6989 }
6990}
6991
6992#[cfg(test)]
6993mod scope_outcome_count_tests {
6994 use super::*;
6995
6996 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
6997 ScopeRecordResult {
6998 scope_ref: "r".to_string(),
6999 scope_epoch: 1,
7000 outcome,
7001 code: None,
7002 message: None,
7003 version: None,
7004 parent_state: None,
7005 }
7006 }
7007
7008 #[test]
7011 fn every_outcome_is_counted_in_its_own_field() {
7012 let results = [
7013 result(ScopeRecordOutcome::Created),
7014 result(ScopeRecordOutcome::Created),
7015 result(ScopeRecordOutcome::Replaced),
7016 result(ScopeRecordOutcome::Updated),
7017 result(ScopeRecordOutcome::Unchanged),
7018 result(ScopeRecordOutcome::Refused),
7019 result(ScopeRecordOutcome::Refused),
7020 result(ScopeRecordOutcome::Refused),
7021 ];
7022 assert_eq!(
7023 ScopeOutcomeCounts::of(&results),
7024 ScopeOutcomeCounts {
7025 created: 2,
7026 replaced: 1,
7027 updated: 1,
7028 unchanged: 1,
7029 refused: 3,
7030 }
7031 );
7032 }
7033}
7034
7035fn capability_census_trigger(
7039 old: Option<&CapabilityDeclarations>,
7040 new: Option<&CapabilityDeclarations>,
7041) -> bool {
7042 let old_provides = old
7043 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7044 .unwrap_or_default();
7045 let old_denies = old
7046 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7047 .unwrap_or_default();
7048 let new = new.cloned().unwrap_or(CapabilityDeclarations {
7049 provides: Vec::new(),
7050 requires: Vec::new(),
7051 must_never_reach: Vec::new(),
7052 });
7053
7054 new.provides
7055 .iter()
7056 .any(|capability| !old_provides.contains(capability))
7057 || new
7058 .must_never_reach
7059 .iter()
7060 .any(|capability| !old_denies.contains(capability))
7061}
7062
7063fn denied_capability<'a>(
7066 opening_manifest: &'a ModuleManifest,
7067 target_manifest: &ModuleManifest,
7068) -> Option<&'a str> {
7069 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7070 let target_capabilities = target_manifest.capabilities.as_ref()?;
7071 opening_capabilities
7072 .must_never_reach
7073 .iter()
7074 .find(|denied| {
7075 target_capabilities
7076 .provides
7077 .iter()
7078 .any(|provided| provided == *denied)
7079 })
7080 .map(String::as_str)
7081}
7082
7083fn catalog_update_frozen_field_message(
7084 registered: &ModuleManifest,
7085 provides: &[ProviderRole],
7086) -> Option<String> {
7087 let old_has_provides = !registered.provides.is_empty();
7088 let new_has_provides = !provides.is_empty();
7089 if old_has_provides != new_has_provides {
7090 return Some(format!(
7091 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7092 registered.module_id
7093 ));
7094 }
7095
7096 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
7097 return Some(format!(
7098 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7099 registered.module_id
7100 ));
7101 }
7102
7103 let registered_concurrency = manifest_concurrency(registered);
7104 let mut candidate = registered.clone();
7105 candidate.provides = provides.to_vec();
7106 let candidate_concurrency = manifest_concurrency(&candidate);
7107 if candidate_concurrency != registered_concurrency {
7108 return Some(format!(
7109 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7110 registered.module_id, registered_concurrency, candidate_concurrency
7111 ));
7112 }
7113
7114 None
7117}
7118
7119fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7120 manifest.provides.iter().any(is_routable_role)
7121}
7122
7123fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7129 manifest
7130 .provides
7131 .iter()
7132 .find_map(|provider| match provider {
7133 ProviderRole::ToolProvider { concurrency, .. }
7134 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7135 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7136 })
7137 .unwrap_or(Concurrency::ModuleManaged)
7138}
7139
7140fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7147 let has_management_surface = manifest
7148 .provides
7149 .iter()
7150 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7151 if !has_management_surface {
7152 return false;
7153 }
7154 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7155 return false;
7156 };
7157 let Some(provides) = raw
7158 .get("manifest")
7159 .and_then(|manifest| manifest.get("provides"))
7160 .and_then(serde_json::Value::as_array)
7161 else {
7162 return false;
7163 };
7164 provides.iter().any(|role| {
7169 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7170 && role.get("concurrency").is_none()
7171 })
7172}
7173
7174fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7175 if peer_version != PROTOCOL_VERSION {
7176 return Err(format!(
7177 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7178 ));
7179 }
7180 Ok(PROTOCOL_VERSION)
7181}
7182
7183fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7184 Frame::build_with_version(
7185 response_version(frame),
7186 FrameType::Pong,
7187 frame.header.flags,
7188 0,
7189 0,
7190 frame.header.corr,
7191 Vec::new(),
7192 )
7193 .map_err(RouterError::FrameBuild)
7194}
7195
7196pub(crate) fn control_error_frame(
7197 frame: &Frame,
7198 code: &'static str,
7199 message: impl Into<String>,
7200) -> Result<Frame, RouterError> {
7201 control_error_body_frame(
7202 frame,
7203 ErrorBody {
7204 code: code.to_string(),
7205 message: message.into(),
7206 detail: None,
7207 },
7208 )
7209}
7210
7211fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7212 let body = serde_json::to_vec(&error).map_err(|err| {
7213 RouterError::backend(
7214 0,
7215 frame.header.corr,
7216 format!("failed to encode control ERROR: {err}"),
7217 )
7218 })?;
7219
7220 Frame::build_with_version(
7221 response_version(frame),
7222 FrameType::Error,
7223 control_flags(),
7224 0,
7225 0,
7226 frame.header.corr,
7227 body,
7228 )
7229 .map_err(RouterError::FrameBuild)
7230}
7231
7232fn control_response_body_frame<T: Serialize>(
7233 frame: &Frame,
7234 reply: &T,
7235 label: &'static str,
7236) -> Result<Frame, RouterError> {
7237 let body = serde_json::to_vec(reply).map_err(|err| {
7238 RouterError::backend(
7239 0,
7240 frame.header.corr,
7241 format!("failed to encode {label}: {err}"),
7242 )
7243 })?;
7244
7245 Frame::build_with_version(
7246 response_version(frame),
7247 FrameType::Response,
7248 control_flags(),
7249 0,
7250 0,
7251 frame.header.corr,
7252 body,
7253 )
7254 .map_err(RouterError::FrameBuild)
7255}
7256
7257fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7287 match err {
7288 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7289 ForwardingError::NoModuleConnection => "target_unavailable",
7290 ForwardingError::ModuleReloading { .. } => "module_reloading",
7291 ForwardingError::ClientRouteChannelExhausted { .. }
7292 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7293 ForwardingError::StaleModuleEndpoint
7294 | ForwardingError::UnknownReservation { .. }
7295 | ForwardingError::ConnectionClosing { .. }
7296 | ForwardingError::ClientEgressClosed { .. }
7297 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7298 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7301 ForwardingError::RelayCorrelationExhausted
7302 | ForwardingError::RouteOpenBuild(_)
7303 | ForwardingError::Poisoned => "forwarding_error",
7304 }
7305}
7306
7307fn response_version(frame: &Frame) -> u8 {
7308 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7309 frame.header.ver
7310 } else {
7311 PROTOCOL_VERSION
7312 }
7313}
7314
7315fn control_flags() -> Flags {
7316 Flags::new(false, Priority::Passive, false)
7317}
7318
7319fn send_goodbye_target_best_effort(
7324 counters: &DaemonCounters,
7325 target: &GoodbyeTarget,
7326 context: &'static str,
7327) {
7328 let Ok(frame) = Frame::build_with_version(
7329 target.negotiated_ver,
7330 FrameType::Goodbye,
7331 control_flags(),
7332 target.channel,
7333 target.epoch,
7334 0,
7335 Vec::new(),
7336 ) else {
7337 return;
7338 };
7339 crate::forwarding::send_module_route_goodbye(
7340 counters,
7341 &target.sink,
7342 frame,
7343 target.module_id.as_deref(),
7344 context,
7345 );
7346}
7347
7348pub(crate) fn send_route_control_pushes(
7349 forwarding: &ForwardingTable,
7350 routes: Vec<EndpointRoute>,
7351 push: ClientControlPush,
7352) {
7353 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7354 for route in routes {
7355 let target = route.goodbye_target;
7356 if let Some((existing, channels)) = targets
7357 .iter_mut()
7358 .find(|(existing, _)| existing.connection_id == target.connection_id)
7359 {
7360 debug_assert_eq!(
7361 existing.negotiated_ver, target.negotiated_ver,
7362 "one connection cannot negotiate multiple frame versions"
7363 );
7364 if !channels.contains(&target.channel) {
7365 channels.push(target.channel);
7366 }
7367 continue;
7368 }
7369 let channel = target.channel;
7370 targets.push((target, vec![channel]));
7371 }
7372 for (target, mut channels) in targets {
7373 channels.sort_unstable();
7374 let mut push = push.clone();
7375 match &mut push {
7376 ClientControlPush::RouteClosing {
7377 channels: covered, ..
7378 }
7379 | ClientControlPush::RouteClosed {
7380 channels: covered, ..
7381 } => *covered = channels,
7382 }
7383 let body = match serde_json::to_vec(&push) {
7384 Ok(body) => body,
7385 Err(err) => {
7386 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7387 continue;
7388 }
7389 };
7390 let frame = match Frame::build_with_version(
7391 target.negotiated_ver,
7392 FrameType::Push,
7393 control_flags(),
7394 0,
7395 0,
7396 0,
7397 body.clone(),
7398 ) {
7399 Ok(frame) => frame,
7400 Err(err) => {
7401 warn!(
7402 route_channel = target.channel,
7403 error = %err,
7404 "failed to build route lifecycle control PUSH frame"
7405 );
7406 continue;
7407 }
7408 };
7409 if let Err(err) = target.sink.try_send(frame) {
7410 if target.close_on_delivery_failure() {
7411 warn!(
7412 target_connection_id = target.connection_id.get(),
7413 route_channel = target.channel,
7414 error = %err,
7415 "route lifecycle control PUSH was not delivered to client; closing target connection"
7416 );
7417 let _ = forwarding.escalate_client_delivery_failure(
7418 target.connection_id,
7419 target.channel,
7420 target.epoch,
7421 CloseReason::new(
7422 "route_lifecycle_push_delivery_failed",
7423 format!(
7424 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7425 target.channel
7426 ),
7427 ),
7428 crate::forwarding::UndeliveredFrame {
7429 module_id: target.module_id.as_deref(),
7430 sink: &target.sink,
7431 },
7432 );
7433 }
7434 }
7435 }
7436}
7437
7438#[cfg(test)]
7439mod tests {
7440 mod activity_read {
7441 include!("activity_read_tests.rs");
7442 }
7443
7444 mod lifecycle_activity {
7445 include!("control_lifecycle_tests.rs");
7446 }
7447 #[cfg(unix)]
7448 #[tokio::test]
7449 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
7450 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
7451 let path = dir.join("subc.jsonc");
7452 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
7453 "program":"/bin/sleep","args":["60"],"protocol":"none",
7454 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
7455 }}}).to_string()).unwrap();
7456 let mut configured = crate::daemon_config::load(&path)
7457 .unwrap()
7458 .unwrap()
7459 .modules
7460 .pop()
7461 .unwrap();
7462 let registry = std::sync::Arc::new(crate::Registry::default());
7463 let handle = crate::SupervisorHandle::new();
7464 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
7465 .with_handle(handle.clone());
7466 let module = supervisor
7467 .supervise_configured_with_health(
7468 configured.module_spec(),
7469 true,
7470 configured.health.clone(),
7471 None,
7472 configured.restart,
7473 )
7474 .unwrap();
7475 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
7476 let before = module.status().unwrap().pid;
7477 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
7478 configured.health.cadence = std::time::Duration::from_secs(3600);
7479 let health_only = handler
7480 .reconcile_supervised_modules(
7481 &supervisor,
7482 vec![configured.clone()],
7483 false,
7484 Principal::Direct,
7485 )
7486 .await
7487 .unwrap();
7488 assert!(
7489 health_only.changed_pending_reload.is_empty(),
7490 "health policy is already applied live"
7491 );
7492 assert_eq!(module.status().unwrap().pid, before);
7493 assert_eq!(
7494 module.configuration().unwrap().1.http,
7495 configured.health.http
7496 );
7497 configured.args = vec!["61".into()];
7498 let launch = handler
7499 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
7500 .await
7501 .unwrap();
7502 assert_eq!(launch.changed_pending_reload, ["stock"]);
7503 assert_eq!(
7504 module.status().unwrap().pid,
7505 before,
7506 "a launch edit is stored until reload"
7507 );
7508 module.drain().await.unwrap();
7509 }
7510 use cortexkit_test_support::ScratchDir;
7511 use std::{
7512 collections::BTreeMap,
7513 fmt,
7514 path::PathBuf,
7515 sync::{Arc, Mutex},
7516 time::Duration,
7517 };
7518
7519 use serde_json::{json, Value};
7520 use subc_protocol::{
7521 manifest::{
7522 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
7523 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
7524 },
7525 session::HealthStatus,
7526 FrameType,
7527 };
7528
7529 use super::*;
7530 use crate::{
7531 forwarding::{DataRoute, DataRouteState},
7532 registry::ChannelState,
7533 router::FrameSink,
7534 stderr_tail::DEFAULT_MAX_LINE_BYTES,
7535 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
7536 RouteCtx, Router,
7537 };
7538 use tokio::{
7539 sync::mpsc,
7540 time::{sleep, Instant},
7541 };
7542 use tracing::{
7543 field::{Field, Visit},
7544 Event, Subscriber,
7545 };
7546 use tracing_subscriber::{layer::Context, prelude::*, Layer};
7547
7548 fn fake_aft_stub_path() -> PathBuf {
7567 let mut path = std::env::current_exe().expect("current_exe available in tests");
7568 path.pop(); path.pop(); path.push(if cfg!(windows) {
7571 "fake-aft-stub.exe"
7572 } else {
7573 "fake-aft-stub"
7574 });
7575 assert!(
7576 path.exists(),
7577 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
7578 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
7579 path.display()
7580 );
7581 path
7582 }
7583
7584 fn client_retries(code: &str) -> bool {
7589 subc_protocol::error_codes::is_retryable_route_open(code)
7590 }
7591
7592 #[test]
7603 fn retryability_of_forwarding_codes_matches_the_failure() {
7604 let transient = [
7607 ForwardingError::NoModuleConnection,
7608 ForwardingError::ModuleReloading {
7609 module_id: "m".into(),
7610 },
7611 ForwardingError::StaleModuleEndpoint,
7612 ForwardingError::UnknownReservation {
7613 client_channel: 1,
7614 module_channel: 1,
7615 },
7616 ForwardingError::ConnectionClosing {
7617 connection_id: ConnectionId::new(1),
7618 },
7619 ForwardingError::ClientEgressClosed {
7620 connection_id: ConnectionId::new(1),
7621 },
7622 ForwardingError::ModuleEgressUnavailable {
7623 connection_id: ConnectionId::new(1),
7624 },
7625 ];
7626 for err in transient {
7627 let code = forwarding_error_code(&err);
7628 assert!(
7629 client_retries(code),
7630 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
7631 );
7632 }
7633
7634 let permanent = [
7639 ForwardingError::ConnectionRoleConflict {
7640 connection_id: ConnectionId::new(1),
7641 },
7642 ForwardingError::ClientRouteChannelExhausted {
7643 connection_id: ConnectionId::new(1),
7644 },
7645 ForwardingError::ModuleRouteChannelExhausted {
7646 endpoint: ModuleEndpointId {
7647 connection_id: ConnectionId::new(1),
7648 generation: 1,
7649 },
7650 },
7651 ForwardingError::RelayCorrelationExhausted,
7652 ForwardingError::RouteOpenBuild("x".into()),
7653 ForwardingError::Poisoned,
7654 ];
7655 for err in permanent {
7656 let code = forwarding_error_code(&err);
7657 assert!(
7658 !client_retries(code),
7659 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
7660 );
7661 }
7662 }
7663
7664 #[tokio::test]
7692 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
7693 let handler = ControlHandler::default();
7694 let frame =
7695 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
7696
7697 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
7701 assert_eq!(
7702 stamped,
7703 Principal::Direct,
7704 "a caller that proved nothing must not be stamped as a supervised module"
7705 );
7706
7707 let forged = handler
7712 .route_open_principal(
7713 &frame,
7714 Some(ConsumerIdentity {
7715 module_id: "aft".to_string(),
7716 launch_nonce: "not-a-real-nonce".to_string(),
7717 }),
7718 )
7719 .unwrap();
7720 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
7721 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
7722 }
7723
7724 #[test]
7741 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
7742 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
7743 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
7744 let ClientControlRequest::RouteOpen {
7745 consumer_identity, ..
7746 } = parsed
7747 else {
7748 panic!("route.open body must parse as RouteOpen");
7749 };
7750 assert_eq!(
7751 consumer_identity,
7752 Some(ConsumerIdentity {
7753 module_id: "aft".to_string(),
7754 launch_nonce: "n".to_string(),
7755 }),
7756 "the wire field name must reach the value route_open_principal reads"
7757 );
7758 }
7759
7760 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
7761 ModuleManifest::builder(module_id, "0.1.0")
7762 .protocol_ver(protocol_ver)
7763 .provides(vec![ProviderRole::ToolProvider {
7764 tools: vec![Tool {
7765 name: "read".to_string(),
7766 description: None,
7767 execution_mode: ExecutionMode::Pure,
7768 schema: json!({"type": "object"}),
7769 }],
7770 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
7771 concurrency: Concurrency::ModuleManaged,
7772 emits_push: true,
7773 sub_supervises: true,
7774 }])
7775 .build()
7776 }
7777
7778 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
7779 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
7780 }
7781
7782 fn hello_frame_with_control_ops(
7783 module_id: &str,
7784 protocol_ver: u8,
7785 corr: u64,
7786 control_ops: Option<Vec<String>>,
7787 ) -> Frame {
7788 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
7789 }
7790
7791 fn hello_frame_with_nonce(
7792 module_id: &str,
7793 protocol_ver: u8,
7794 corr: u64,
7795 launch_nonce: Option<&str>,
7796 ) -> Frame {
7797 hello_frame_full(
7798 module_id,
7799 protocol_ver,
7800 corr,
7801 None,
7802 launch_nonce.map(ToOwned::to_owned),
7803 )
7804 }
7805
7806 fn hello_frame_full(
7807 module_id: &str,
7808 protocol_ver: u8,
7809 corr: u64,
7810 control_ops: Option<Vec<String>>,
7811 launch_nonce: Option<String>,
7812 ) -> Frame {
7813 let body = serde_json::to_vec(&ModuleHelloBody {
7814 manifest: manifest(module_id, protocol_ver),
7815 protocol_ver,
7816 control_ops,
7817 launch_nonce,
7818 })
7819 .unwrap();
7820 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7821 }
7822
7823 fn non_routable_hello_frame_with_control_ops(
7824 module_id: &str,
7825 corr: u64,
7826 control_ops: Option<Vec<String>>,
7827 ) -> Frame {
7828 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
7829 manifest.provides.clear();
7830 let body = serde_json::to_vec(&ModuleHelloBody {
7831 manifest,
7832 protocol_ver: PROTOCOL_VERSION,
7833 control_ops,
7834 launch_nonce: None,
7835 })
7836 .unwrap();
7837 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7838 }
7839
7840 fn capability_grammar_hello_frame(
7841 capabilities: Value,
7842 runtime_computed: Option<Value>,
7843 corr: u64,
7844 ) -> Frame {
7845 let mut body = serde_json::to_value(ModuleHelloBody {
7846 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
7847 protocol_ver: PROTOCOL_VERSION,
7848 control_ops: None,
7849 launch_nonce: None,
7850 })
7851 .expect("HELLO body serializes");
7852 body["manifest"]["capabilities"] = capabilities;
7853 if let Some(runtime_computed) = runtime_computed {
7854 body["runtime_computed"] = runtime_computed;
7855 }
7856 Frame::build(
7857 FrameType::Hello,
7858 control_flags(),
7859 0,
7860 0,
7861 corr,
7862 serde_json::to_vec(&body).expect("HELLO body reserializes"),
7863 )
7864 .expect("HELLO frame builds")
7865 }
7866
7867 fn channel_request(channel: u16, corr: u64) -> Frame {
7868 Frame::build(
7869 FrameType::Request,
7870 Flags::new(true, Priority::Interactive, false),
7871 channel,
7872 0,
7873 corr,
7874 b"opaque".to_vec(),
7875 )
7876 .unwrap()
7877 }
7878
7879 fn route_ctx(
7880 connection_id: ConnectionId,
7881 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
7882 let (tx, rx) = mpsc::channel(8);
7883 (
7884 RouteCtx {
7885 connection_id,
7886 egress: FrameSink::new(tx),
7887 },
7888 rx,
7889 )
7890 }
7891
7892 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
7893 serde_json::from_slice(&frame.body).unwrap()
7894 }
7895
7896 async fn hello_via_sink(
7901 handler: &ControlHandler,
7902 ctx: &RouteCtx,
7903 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7904 hello: Frame,
7905 ) -> Frame {
7906 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
7907 assert!(
7908 replies.is_empty(),
7909 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
7910 );
7911 let ack = rx
7912 .try_recv()
7913 .expect("HELLO_ACK is queued on the module sink")
7914 .frame;
7915 assert_eq!(ack.header.ty, FrameType::HelloAck);
7916 ack
7917 }
7918
7919 fn parse_error(frame: &Frame) -> Value {
7920 serde_json::from_slice(&frame.body).unwrap()
7921 }
7922
7923 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
7924 serde_json::from_slice(&frame.body).unwrap()
7925 }
7926
7927 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
7928 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
7929 route_channel,
7930 route_epoch: 0,
7931 kind,
7932 })
7933 .unwrap();
7934 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7935 }
7936
7937 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
7938 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
7939 module_id: module_id.to_string(),
7940 })
7941 .unwrap();
7942 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7943 }
7944
7945 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
7946 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
7947 }
7948
7949 fn route_open_frame_with_consumer_capabilities(
7950 corr: u64,
7951 module_id: &str,
7952 project_root: ScratchDir,
7953 consumer_capabilities: Option<Vec<String>>,
7954 ) -> Frame {
7955 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7956 target: RouteTarget::ToolProvider {
7957 module_id: module_id.to_string(),
7958 },
7959 identity: BindIdentity::new(
7960 project_root.path().to_path_buf(),
7961 "unit".to_string(),
7962 "session".to_string(),
7963 ),
7964 consumer_identity: None,
7965 consumer_capabilities,
7966 role_versions: None,
7967 admission_facts: None,
7968 scope: None,
7969 })
7970 .unwrap();
7971 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7972 }
7973
7974 fn route_open_frame_with_role_versions(
7975 corr: u64,
7976 module_id: &str,
7977 project_root: ScratchDir,
7978 role_versions: Option<BTreeMap<String, String>>,
7979 ) -> Frame {
7980 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7981 target: RouteTarget::ToolProvider {
7982 module_id: module_id.to_string(),
7983 },
7984 identity: BindIdentity::new(
7985 project_root.path().to_path_buf(),
7986 "unit".to_string(),
7987 format!("session-{corr}"),
7988 ),
7989 consumer_identity: None,
7990 consumer_capabilities: None,
7991 role_versions,
7992 admission_facts: None,
7993 scope: None,
7994 })
7995 .unwrap();
7996 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7997 }
7998
7999 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8000 entries
8001 .iter()
8002 .map(|(role, version)| (role.to_string(), version.to_string()))
8003 .collect()
8004 }
8005
8006 fn route_open_frame_with_admission_facts(
8007 corr: u64,
8008 module_id: &str,
8009 project_root: ScratchDir,
8010 consumer_identity: Option<subc_control::ConsumerIdentity>,
8011 facts: Option<Value>,
8012 ) -> Frame {
8013 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8014 target: RouteTarget::ToolProvider {
8015 module_id: module_id.to_string(),
8016 },
8017 identity: BindIdentity::new(
8018 project_root.path().to_path_buf(),
8019 "unit".to_string(),
8020 format!("session-{corr}"),
8021 ),
8022 consumer_identity,
8023 consumer_capabilities: None,
8024 role_versions: None,
8025 admission_facts: facts,
8026 scope: None,
8027 })
8028 .unwrap();
8029 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8030 }
8031
8032 #[derive(Clone, Default)]
8033 struct EventCapture {
8034 events: Arc<Mutex<Vec<CapturedEvent>>>,
8035 }
8036
8037 #[derive(Clone, Debug)]
8038 struct CapturedEvent {
8039 target: String,
8040 level: tracing::Level,
8041 fields: BTreeMap<String, String>,
8042 }
8043
8044 impl EventCapture {
8045 fn events(&self) -> Vec<CapturedEvent> {
8046 self.events.lock().unwrap().clone()
8047 }
8048 }
8049
8050 impl<S> Layer<S> for EventCapture
8051 where
8052 S: Subscriber,
8053 {
8054 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8055 let mut visitor = EventFieldVisitor::default();
8056 event.record(&mut visitor);
8057 self.events.lock().unwrap().push(CapturedEvent {
8058 target: event.metadata().target().to_string(),
8059 level: *event.metadata().level(),
8060 fields: visitor.fields,
8061 });
8062 }
8063 }
8064
8065 #[derive(Default)]
8066 struct EventFieldVisitor {
8067 fields: BTreeMap<String, String>,
8068 }
8069
8070 impl Visit for EventFieldVisitor {
8071 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8072 self.fields
8073 .insert(field.name().to_string(), format!("{value:?}"));
8074 }
8075 }
8076
8077 fn health_response(corr: u64, status: HealthStatus) -> Frame {
8078 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8079 status,
8080 detail: Some("warming".to_string()),
8081 metrics: Some(json!({"queue_depth": 3})),
8082 })
8083 .unwrap();
8084 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8085 }
8086
8087 fn route_bind_ack(corr: u64) -> Frame {
8088 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8089 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8090 }
8091
8092 fn unique_project_root(label: &str) -> ScratchDir {
8093 ScratchDir::new(label)
8094 }
8095
8096 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8097 match parse_route_poll(frame) {
8098 ClientControlResponse::RoutePoll {
8099 status: None,
8100 live: Some(live),
8101 ..
8102 } => assert_eq!(live, expected_live),
8103 other => panic!("unexpected route.poll response: {other:?}"),
8104 }
8105 }
8106
8107 fn bind_liveness_route(
8108 registry: &Registry,
8109 forwarding: &ForwardingTable,
8110 module_id: &str,
8111 ) -> (RouteCtx, u16, u32) {
8112 let module_connection = ConnectionId::new(101);
8113 let client_connection = ConnectionId::new(202);
8114 let registration = registry
8115 .register_with_control_ops(
8116 manifest(module_id, PROTOCOL_VERSION),
8117 PROTOCOL_VERSION,
8118 module_connection,
8119 module_baseline_control_ops(),
8120 )
8121 .unwrap();
8122 let (module_tx, _module_rx) = mpsc::channel(8);
8123 let endpoint = forwarding
8124 .register_module_connection(
8125 module_connection,
8126 module_id.to_string(),
8127 PROTOCOL_VERSION,
8128 manifest_concurrency(®istration.manifest),
8129 FrameSink::new(module_tx),
8130 )
8131 .unwrap();
8132 let (client_ctx, _client_rx) = route_ctx(client_connection);
8133 let pending = forwarding
8134 .begin_route_bind_relay_for_test(
8135 client_connection,
8136 client_ctx.egress.clone(),
8137 1,
8138 module_id,
8139 )
8140 .unwrap();
8141 assert_eq!(pending.endpoint, endpoint);
8142 let route_channel = pending.client_channel;
8143 let route_epoch = pending.client_epoch;
8144 forwarding
8145 .complete_pending_relay(
8146 module_connection,
8147 pending.corr,
8148 RouteBindRelayOutcome::Accepted,
8149 )
8150 .unwrap();
8151 (client_ctx, route_channel, route_epoch)
8152 }
8153
8154 struct FakeProcessLiveness {
8155 live: Option<bool>,
8156 }
8157
8158 impl ModuleProcessLiveness for FakeProcessLiveness {
8159 fn process_live(&self, _module_id: &str) -> Option<bool> {
8160 self.live
8161 }
8162 }
8163
8164 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8165 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8166 {
8167 let registry = Arc::new(Registry::default());
8168 let supervisor_handle = SupervisorHandle::new();
8169 let supervisor = Supervisor::new_for_test(
8170 Arc::clone(®istry),
8171 RestartPolicy::new(1, Duration::from_millis(10)),
8172 )
8173 .with_handle(supervisor_handle.clone());
8174 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8175 let module = supervisor
8176 .spawn(ModuleSpec {
8177 module_id: "stderr-tail-wire".to_string(),
8178 program: fake_aft_stub_path(),
8179 args: Vec::new(),
8180 env: vec![
8181 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8182 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8183 ],
8184 reserved: false,
8185 reserved_prefixes: Vec::new(),
8186 protocol: ModuleProtocol::Subc,
8187 overlap: Default::default(),
8188 })
8189 .unwrap();
8190
8191 let deadline = Instant::now() + Duration::from_secs(5);
8192 loop {
8193 let tail = module.stderr_tail(None, None);
8194 if tail
8195 .entries
8196 .iter()
8197 .any(|entry| matches!(entry, TailEntry::ProcessStart))
8198 && tail.entries.iter().any(|entry| {
8199 matches!(
8200 entry,
8201 TailEntry::Line {
8202 truncated: true,
8203 ..
8204 }
8205 )
8206 })
8207 {
8208 break;
8209 }
8210 assert!(
8211 Instant::now() < deadline,
8212 "module did not produce a truncated line and restart boundary: {tail:?}"
8213 );
8214 sleep(Duration::from_millis(10)).await;
8215 }
8216
8217 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8218 let request = ClientControlRequest::SupervisorStderrTail {
8219 module_id: "stderr-tail-wire".to_string(),
8220 max_lines: None,
8221 max_bytes: None,
8222 };
8223 let frame = Frame::build(
8224 FrameType::Request,
8225 control_flags(),
8226 0,
8227 0,
8228 1,
8229 serde_json::to_vec(&request).unwrap(),
8230 )
8231 .unwrap();
8232 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8233 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8234 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8235 serde_json::from_slice(&responses[0].body).unwrap()
8236 else {
8237 panic!("expected supervisor.stderr_tail response");
8238 };
8239
8240 assert!(
8241 tail.entries
8242 .iter()
8243 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8244 "the control response lost the restart boundary"
8245 );
8246 let Some(StderrTailEntry::Line {
8247 text,
8248 truncated,
8249 at_ms,
8250 }) = tail.entries.iter().find(|entry| {
8251 matches!(
8252 entry,
8253 StderrTailEntry::Line {
8254 truncated: true,
8255 ..
8256 }
8257 )
8258 })
8259 else {
8260 panic!("the control response lost the truncated line");
8261 };
8262 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8263 assert!(*truncated);
8264 assert!(
8265 at_ms.is_some(),
8266 "the control response lost the line's capture time"
8267 );
8268 }
8269
8270 #[tokio::test(flavor = "current_thread")]
8275 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8276 let dir = ScratchDir::new("terminals-off-worker");
8277 let journal_path = dir.join("terminals.jsonl");
8278 let registry = Arc::new(Registry::default());
8279 let supervisor_handle = SupervisorHandle::new();
8280 let supervisor =
8281 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8282 .with_handle(supervisor_handle.clone())
8283 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8284 let module = supervisor
8285 .spawn(ModuleSpec {
8286 module_id: "terminal-off-worker".to_string(),
8287 program: fake_aft_stub_path(),
8288 args: Vec::new(),
8289 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8290 reserved: false,
8291 reserved_prefixes: Vec::new(),
8292 protocol: ModuleProtocol::Subc,
8293 overlap: Default::default(),
8294 })
8295 .unwrap();
8296 let deadline = Instant::now() + Duration::from_secs(5);
8297 while module.terminal_history().entries.len() != 2 {
8298 assert!(Instant::now() < deadline, "module did not record two exits");
8299 sleep(Duration::from_millis(10)).await;
8300 }
8301
8302 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8303 let handler =
8304 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
8305 let frame = Frame::build(
8306 FrameType::Request,
8307 control_flags(),
8308 0,
8309 0,
8310 1,
8311 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8312 module_id: "terminal-off-worker".to_string(),
8313 })
8314 .unwrap(),
8315 )
8316 .unwrap();
8317 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8318 let spawned_at = std::time::Instant::now();
8319 let read = tokio::spawn({
8320 let handler = Arc::clone(&handler);
8321 async move { handler.handle_control_frame(&ctx, frame).await }
8322 });
8323 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8326 .await
8327 .unwrap()
8328 .expect("the history read reached its pause");
8329 let elapsed = spawned_at.elapsed();
8330 assert!(
8331 elapsed < Duration::from_secs(2) && !read.is_finished(),
8332 "this task could not run while the history read was paused \
8333 (resumed after {elapsed:?}, read finished: {})",
8334 read.is_finished()
8335 );
8336
8337 drop(release);
8338 let responses = read.await.unwrap().unwrap();
8339 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8340 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8341 panic!("expected supervisor.terminals response");
8342 };
8343 assert_eq!(terminals.entries.len(), 2);
8344 assert_eq!(terminals.journal_skipped_lines, 0);
8345 assert_eq!(terminals.journal_read_errors, 0);
8346 }
8347
8348 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8349 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8350 let registry = Arc::new(Registry::default());
8351 let supervisor_handle = SupervisorHandle::new();
8352 let supervisor =
8353 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8354 .with_handle(supervisor_handle.clone());
8355 let module = supervisor
8356 .spawn(ModuleSpec {
8357 module_id: "terminal-golden".to_string(),
8358 program: fake_aft_stub_path(),
8359 args: Vec::new(),
8360 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8361 reserved: false,
8362 reserved_prefixes: Vec::new(),
8363 protocol: ModuleProtocol::Subc,
8364 overlap: Default::default(),
8365 })
8366 .unwrap();
8367
8368 let deadline = Instant::now() + Duration::from_secs(5);
8369 while module.terminal_history().entries.len() != 2 {
8370 assert!(
8371 Instant::now() < deadline,
8372 "module did not retain two terminal exits: {:?}",
8373 module.terminal_history()
8374 );
8375 sleep(Duration::from_millis(10)).await;
8376 }
8377
8378 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8379 let request = ClientControlRequest::SupervisorTerminals {
8380 module_id: "terminal-golden".to_string(),
8381 };
8382 let frame = Frame::build(
8383 FrameType::Request,
8384 control_flags(),
8385 0,
8386 0,
8387 1,
8388 serde_json::to_vec(&request).unwrap(),
8389 )
8390 .unwrap();
8391 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8392 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8393 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8394 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8395 panic!("expected supervisor.terminals response");
8396 };
8397 assert_eq!(terminals.entries.len(), 2);
8398 assert_eq!(terminals.dropped, 0);
8399
8400 let mut rendered = serde_json::to_value(response).unwrap();
8401 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
8404 for (index, entry) in rendered["entries"]
8405 .as_array_mut()
8406 .expect("terminal response entries array")
8407 .iter_mut()
8408 .enumerate()
8409 {
8410 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
8411 }
8412
8413 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8414 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
8415 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
8416 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8417 std::fs::write(&golden_path, &serialized).unwrap();
8418 }
8419 let expected: Value =
8420 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
8421 assert_eq!(rendered, expected);
8422 }
8423
8424 #[test]
8425 fn hello_registers_manifest_and_returns_ack() {
8426 let registry = Arc::new(Registry::default());
8427 let handler = ControlHandler::new(Arc::clone(®istry));
8428 let conn = ConnectionId::new(1);
8429
8430 let responses = handler
8431 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
8432 .unwrap();
8433
8434 assert_eq!(responses.len(), 1);
8435 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8436 assert_eq!(responses[0].header.channel, 0);
8437 assert_eq!(responses[0].header.corr, 7);
8438 let ack = parse_ack(&responses[0]);
8439 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8440 assert!(ack
8441 .subc_capabilities
8442 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
8443 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
8444 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
8445 assert!(ack
8446 .subc_ops
8447 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
8448 assert!(ack
8449 .subc_ops
8450 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
8451
8452 let registration = registry.get_module("aft").unwrap().unwrap();
8453 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
8454 assert_eq!(registration.state, ChannelState::Active);
8455 assert_eq!(registration.connection_id, conn);
8456 assert_eq!(registration.control_ops, module_baseline_control_ops());
8457 }
8458
8459 #[test]
8460 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
8461 let invalid_identifiers = [
8462 ("case_change", "credentials-Provider/v1"),
8463 ("leading_zero", "credentials-provider/v01"),
8464 ("trailing_hyphen", "credentials-provider-/v1"),
8465 ("consecutive_hyphens", "credentials--provider/v1"),
8466 ("uppercase", "Credentials-provider/v1"),
8467 ("missing_v", "credentials-provider/1"),
8468 ("whitespace", "credentials provider/v1"),
8469 ("zero_version", "credentials-provider/v0"),
8470 ("out_of_range_version", "credentials-provider/v4294967296"),
8471 (
8472 "overlength_name",
8473 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
8474 ),
8475 ];
8476 let mut cases = invalid_identifiers
8477 .into_iter()
8478 .map(|(name, identifier)| {
8479 (
8480 format!("identifier_{name}"),
8481 "capabilities.provides[0]".to_string(),
8482 identifier.to_string(),
8483 json!({ "provides": [identifier] }),
8484 None,
8485 )
8486 })
8487 .collect::<Vec<_>>();
8488 cases.extend([
8489 (
8490 "unknown_need".to_string(),
8491 "capabilities.requires[0].need".to_string(),
8492 "deferred".to_string(),
8493 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
8494 None,
8495 ),
8496 (
8497 "duplicate_provides".to_string(),
8498 "capabilities.provides[1]".to_string(),
8499 "credentials-provider/v1".to_string(),
8500 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
8501 None,
8502 ),
8503 (
8504 "duplicate_must_never_reach".to_string(),
8505 "capabilities.must_never_reach[1]".to_string(),
8506 "credentials-provider/v1".to_string(),
8507 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
8508 None,
8509 ),
8510 (
8511 "duplicate_requires_same_need".to_string(),
8512 "capabilities.requires[1]".to_string(),
8513 "credentials-provider/v1".to_string(),
8514 json!({ "requires": [
8515 { "capability": "credentials-provider/v1", "need": "required" },
8516 { "capability": "credentials-provider/v1", "need": "required" }
8517 ] }),
8518 None,
8519 ),
8520 (
8521 "duplicate_requires_conflicting_need".to_string(),
8522 "capabilities.requires[1]".to_string(),
8523 "credentials-provider/v1".to_string(),
8524 json!({ "requires": [
8525 { "capability": "credentials-provider/v1", "need": "required" },
8526 { "capability": "credentials-provider/v1", "need": "optional" }
8527 ] }),
8528 None,
8529 ),
8530 (
8531 "capabilities_root_pointer".to_string(),
8532 "runtime_computed[0]".to_string(),
8533 "/capabilities".to_string(),
8534 json!({}),
8535 Some(json!(["/capabilities"])),
8536 ),
8537 (
8538 "capabilities_descendant_pointer".to_string(),
8539 "runtime_computed[0]".to_string(),
8540 "/capabilities/provides".to_string(),
8541 json!({}),
8542 Some(json!(["/capabilities/provides"])),
8543 ),
8544 (
8545 "malformed_pointer_without_leading_slash".to_string(),
8546 "runtime_computed[0]".to_string(),
8547 "capabilities".to_string(),
8548 json!({}),
8549 Some(json!(["capabilities"])),
8550 ),
8551 (
8552 "malformed_pointer_escape".to_string(),
8553 "runtime_computed[0]".to_string(),
8554 "/roles/~2/tools".to_string(),
8555 json!({}),
8556 Some(json!(["/roles/~2/tools"])),
8557 ),
8558 (
8559 "unknown_capabilities_field".to_string(),
8560 "capabilities.future".to_string(),
8561 "<array>".to_string(),
8562 json!({ "future": [] }),
8563 None,
8564 ),
8565 ]);
8566
8567 for (index, (name, field, value, capabilities, runtime_computed)) in
8568 cases.into_iter().enumerate()
8569 {
8570 let registry = Arc::new(Registry::default());
8571 let handler = ControlHandler::new(Arc::clone(®istry));
8572 let response = handler
8573 .handle_control(
8574 ConnectionId::new((index + 1) as u64),
8575 capability_grammar_hello_frame(
8576 capabilities,
8577 runtime_computed,
8578 index as u64 + 1,
8579 ),
8580 )
8581 .expect("invalid HELLO returns a refusal");
8582
8583 assert_eq!(response.len(), 1, "{name} must emit one refusal");
8584 let error = parse_error(&response[0]);
8585 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
8586 let message = error["message"]
8587 .as_str()
8588 .expect("error message is a string");
8589 assert!(
8590 message.contains(&field),
8591 "{name}: field missing from {message}"
8592 );
8593 assert!(
8594 message.contains(&value),
8595 "{name}: value missing from {message}"
8596 );
8597 assert_eq!(
8598 registry
8599 .active_registration_count()
8600 .expect("registry reads"),
8601 0,
8602 "{name}: refused HELLO must not create a catalog entry"
8603 );
8604 }
8605 }
8606
8607 #[test]
8608 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
8609 let registry = Arc::new(Registry::default());
8610 let handler = ControlHandler::new(Arc::clone(®istry));
8611 let capabilities = json!({
8612 "provides": ["credentials-provider/v1"],
8613 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
8614 "must_never_reach": ["federation-transport/v1"]
8615 });
8616 let response = handler
8617 .handle_control(
8618 ConnectionId::new(99),
8619 capability_grammar_hello_frame(
8620 capabilities.clone(),
8621 Some(json!(["/roles/0/tools"])),
8622 99,
8623 ),
8624 )
8625 .expect("valid HELLO registers");
8626 assert_eq!(response[0].header.ty, FrameType::HelloAck);
8627
8628 let request = Frame::build(
8629 FrameType::Request,
8630 control_flags(),
8631 0,
8632 0,
8633 100,
8634 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8635 .expect("catalog request serializes"),
8636 )
8637 .expect("catalog request frame builds");
8638 let response = handler
8639 .handle_catalog_list(request, None)
8640 .expect("catalog list succeeds");
8641 let ClientControlResponse::CatalogList { modules, .. } =
8642 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
8643 else {
8644 panic!("catalog request must return catalog.list");
8645 };
8646 assert_eq!(modules.len(), 1);
8647 assert_eq!(
8648 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
8649 capabilities
8650 );
8651 }
8652
8653 #[test]
8654 fn catalog_list_mirrors_management_operation_description() {
8655 let registry = Arc::new(Registry::default());
8656 let handler = ControlHandler::new(Arc::clone(®istry));
8657 let description = "List managed records and return their identifiers and metadata.";
8658 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
8659 manifest.provides = vec![ProviderRole::ManagementSurface {
8660 operations: vec![ManagementOperation {
8661 name: "records.list".to_string(),
8662 kind: ManagementOperationKind::Query,
8663 description: Some(description.to_string()),
8664 }],
8665 config_schema: json!({"type": "object"}),
8666 observability: vec![ObservabilitySurface {
8667 name: "records.stats".to_string(),
8668 kind: ObservabilityKind::Snapshot,
8669 }],
8670 identity_scope: vec![IdentityScope::Project],
8671 concurrency: Concurrency::ModuleManaged,
8672 }];
8673 registry
8674 .register_with_control_ops(
8675 manifest,
8676 PROTOCOL_VERSION,
8677 ConnectionId::new(99),
8678 Vec::new(),
8679 )
8680 .expect("described management manifest registers");
8681
8682 let request = Frame::build(
8683 FrameType::Request,
8684 control_flags(),
8685 0,
8686 0,
8687 100,
8688 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8689 .expect("catalog request serializes"),
8690 )
8691 .expect("catalog request frame builds");
8692 let response = handler
8693 .handle_catalog_list(request, None)
8694 .expect("catalog list succeeds");
8695 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8696 assert_eq!(
8697 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
8698 "catalog.list must preserve the declared operation description verbatim"
8699 );
8700 }
8701
8702 #[test]
8703 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
8704 let registry = Arc::new(Registry::default());
8705 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8706 [("vault".to_string(), true), ("squatter".to_string(), true)],
8707 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8708 );
8709 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
8710 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8711 provides: vec!["credentials-provider/v1".to_string()],
8712 requires: Vec::new(),
8713 must_never_reach: Vec::new(),
8714 });
8715 let frame = Frame::build(
8716 FrameType::Hello,
8717 control_flags(),
8718 0,
8719 0,
8720 77,
8721 serde_json::to_vec(&ModuleHelloBody {
8722 manifest: squatter,
8723 protocol_ver: PROTOCOL_VERSION,
8724 control_ops: None,
8725 launch_nonce: None,
8726 })
8727 .expect("HELLO serializes"),
8728 )
8729 .expect("HELLO frame builds");
8730 let response = handler
8731 .handle_control(ConnectionId::new(77), frame)
8732 .expect("reserved claim receives a typed refusal");
8733 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8734 assert_eq!(
8735 registry
8736 .active_registration_count()
8737 .expect("registry reads"),
8738 0,
8739 "a reserved capability refusal must not leave a catalog entry"
8740 );
8741 }
8742
8743 #[test]
8744 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
8745 for settlement in ["timeout", "inconclusive", "drop"] {
8746 let breakers = RouteBindBreakers::default();
8747 let RouteBindAdmission::Admitted {
8748 guard: mut old,
8749 probe: false,
8750 } = breakers.admit("prov")
8751 else {
8752 panic!("ordinary relay admitted")
8753 };
8754 let RouteBindAdmission::Admitted {
8755 guard: mut opener, ..
8756 } = breakers.admit("prov")
8757 else {
8758 panic!("second relay admitted")
8759 };
8760 assert!(
8761 !opener
8762 .record_timeout(1, Duration::ZERO)
8763 .unwrap()
8764 .reopened_after_probe
8765 );
8766 let RouteBindAdmission::Admitted {
8767 guard: mut probe,
8768 probe: true,
8769 } = breakers.admit("prov")
8770 else {
8771 panic!("one cooldown probe admitted")
8772 };
8773 match settlement {
8774 "timeout" => assert!(
8775 !old.record_timeout(1, Duration::ZERO)
8776 .unwrap()
8777 .reopened_after_probe
8778 ),
8779 "inconclusive" => old.record_inconclusive(),
8780 "drop" => drop(old),
8781 _ => unreachable!(),
8782 }
8783 assert!(
8784 matches!(
8785 breakers.admit("prov"),
8786 RouteBindAdmission::Refused {
8787 probe_in_flight: true,
8788 ..
8789 }
8790 ),
8791 "{settlement} of a pre-open relay cannot release the real probe"
8792 );
8793 assert!(
8794 probe
8795 .record_timeout(1, Duration::ZERO)
8796 .unwrap()
8797 .reopened_after_probe
8798 );
8799 assert!(matches!(
8800 breakers.admit("prov"),
8801 RouteBindAdmission::Admitted { probe: true, .. }
8802 ));
8803 }
8804 let breakers = RouteBindBreakers::default();
8805 let admit = || match breakers.admit("prov") {
8806 RouteBindAdmission::Admitted { guard, .. } => guard,
8807 _ => panic!("relay admitted"),
8808 };
8809 admit().record_timeout(1, Duration::ZERO);
8810 let mut old_probe = admit();
8811 breakers.reset_for_new_module_connection("prov");
8812 admit().record_timeout(1, Duration::ZERO);
8813 let _new_probe = admit();
8814 old_probe.record_inconclusive();
8815 assert!(matches!(
8816 breakers.admit("prov"),
8817 RouteBindAdmission::Refused {
8818 probe_in_flight: true,
8819 ..
8820 }
8821 ));
8822 }
8823
8824 #[tokio::test]
8825 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
8826 for candidate in [false, true] {
8827 let registry = Arc::new(Registry::default());
8828 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8829 [("vault".to_string(), true), ("squatter".to_string(), true)],
8830 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8831 );
8832 let conn = ConnectionId::new(77);
8833 let (ctx, mut rx) = route_ctx(conn);
8834 let initial = capability_manifest("squatter", &[], &[]);
8835 if candidate {
8836 registry
8837 .register_candidate_with_control_ops(
8838 initial.clone(),
8839 PROTOCOL_VERSION,
8840 conn,
8841 module_baseline_control_ops(),
8842 )
8843 .unwrap();
8844 handler
8845 .forwarding
8846 .register_candidate_module_connection(
8847 conn,
8848 "squatter".to_string(),
8849 PROTOCOL_VERSION,
8850 manifest_concurrency(&initial),
8851 ctx.egress.clone(),
8852 )
8853 .unwrap();
8854 } else {
8855 hello_via_sink(
8856 &handler,
8857 &ctx,
8858 &mut rx,
8859 hello_frame_with_manifest(initial.clone(), 1),
8860 )
8861 .await;
8862 }
8863 let response = handler
8864 .handle_control_frame(
8865 &ctx,
8866 catalog_update_with_capabilities_frame(
8867 2,
8868 capability_manifest("squatter", &["credentials-provider/v1"], &[])
8869 .capabilities
8870 .unwrap(),
8871 ),
8872 )
8873 .await
8874 .unwrap();
8875 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8876 assert_eq!(
8877 registry
8878 .get_module_by_connection(conn)
8879 .unwrap()
8880 .unwrap()
8881 .manifest,
8882 initial
8883 );
8884 }
8885 }
8886
8887 #[test]
8888 fn server_describe_surfaces_required_capability_verdict_fields() {
8889 let registry = Arc::new(Registry::default());
8890 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8891 [
8892 ("consumer".to_string(), true),
8893 ("provider".to_string(), false),
8894 ],
8895 BTreeMap::new(),
8896 );
8897 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
8898 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8899 provides: Vec::new(),
8900 requires: vec![subc_protocol::manifest::CapabilityRequirement {
8901 capability: "credentials-provider/v1".to_string(),
8902 need: subc_protocol::manifest::CapabilityNeed::Required,
8903 }],
8904 must_never_reach: Vec::new(),
8905 });
8906 let hello = Frame::build(
8907 FrameType::Hello,
8908 control_flags(),
8909 0,
8910 0,
8911 78,
8912 serde_json::to_vec(&ModuleHelloBody {
8913 manifest: consumer,
8914 protocol_ver: PROTOCOL_VERSION,
8915 control_ops: None,
8916 launch_nonce: None,
8917 })
8918 .expect("HELLO serializes"),
8919 )
8920 .expect("HELLO frame builds");
8921 handler
8922 .handle_control(ConnectionId::new(78), hello)
8923 .expect("consumer registers");
8924 let describe = Frame::build(
8925 FrameType::Request,
8926 control_flags(),
8927 0,
8928 0,
8929 79,
8930 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
8931 .expect("request serializes"),
8932 )
8933 .expect("describe frame builds");
8934 let response = handler
8935 .handle_server_describe(describe)
8936 .expect("server.describe succeeds");
8937 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
8938 let requirement = &rendered["capability_requirements"][0];
8939 assert_eq!(requirement["consumer"], "consumer");
8940 assert_eq!(requirement["verdict"], "never_provided");
8941 assert_eq!(requirement["episode_seq"], 1);
8942 assert_eq!(requirement["config_satisfiable"], false);
8943 assert_eq!(requirement["runtime_available"], false);
8944 assert!(requirement["detail"]
8945 .as_str()
8946 .expect("detail string")
8947 .contains("credentials-provider/v1"));
8948 }
8949
8950 #[test]
8951 fn catalog_list_omits_capabilities_for_legacy_manifest() {
8952 let registry = Arc::new(Registry::default());
8953 let handler = ControlHandler::new(Arc::clone(®istry));
8954 let hello = handler
8955 .handle_control(
8956 ConnectionId::new(101),
8957 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
8958 )
8959 .expect("legacy HELLO registers");
8960 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
8961
8962 let request = Frame::build(
8963 FrameType::Request,
8964 control_flags(),
8965 0,
8966 0,
8967 102,
8968 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8969 .expect("catalog request serializes"),
8970 )
8971 .expect("catalog request frame builds");
8972 let response = handler
8973 .handle_catalog_list(request, None)
8974 .expect("catalog list succeeds");
8975 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8976 assert!(
8977 body["modules"][0].get("capabilities").is_none(),
8978 "legacy manifest must retain an absent capabilities field on catalog.list"
8979 );
8980 }
8981
8982 #[test]
8983 fn hello_ack_omits_storage_when_no_storage_config() {
8984 let registry = Arc::new(Registry::default());
8985 let handler = ControlHandler::new(Arc::clone(®istry));
8986 let responses = handler
8987 .handle_control(
8988 ConnectionId::new(1),
8989 hello_frame("aft", PROTOCOL_VERSION, 7),
8990 )
8991 .unwrap();
8992 let ack = parse_ack(&responses[0]);
8993 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
8994 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
8995 }
8996
8997 #[tokio::test]
8998 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
8999 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
9000 let registry = Arc::new(Registry::default());
9001 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
9002 let responses = handler
9003 .handle_control(
9004 ConnectionId::new(1),
9005 hello_frame("aft", PROTOCOL_VERSION, 7),
9006 )
9007 .unwrap();
9008 let ack = parse_ack(&responses[0]);
9009 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
9010
9011 let described = handler
9012 .handle_control_frame(
9013 &route_ctx(ConnectionId::new(2)).0,
9014 Frame::build(
9015 FrameType::Request,
9016 control_flags(),
9017 0,
9018 0,
9019 9,
9020 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9021 )
9022 .unwrap(),
9023 )
9024 .await
9025 .unwrap();
9026 let ClientControlResponse::ServerDescribe { machine_id, .. } =
9027 serde_json::from_slice(&described[0].body).unwrap()
9028 else {
9029 panic!("server.describe answered with another shape");
9030 };
9031 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
9032 }
9033
9034 #[test]
9035 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
9036 let registry = Arc::new(Registry::default());
9039 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
9040 crate::daemon_config::StorageConfig::Sqlite {
9041 data_home: std::path::PathBuf::from("/data"),
9042 },
9043 ));
9044
9045 let responses = handler
9046 .handle_control(
9047 ConnectionId::new(1),
9048 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
9049 )
9050 .unwrap();
9051 let ack = parse_ack(&responses[0]);
9052 assert_eq!(
9053 ack.storage,
9054 Some(serde_json::json!({
9055 "module_id": "alfonso-routing",
9056 "storage_namespace": "default",
9057 "isolation": { "kind": "module" },
9058 "backend": {
9059 "backend": "sqlite",
9060 "path": "/data/cortexkit/alfonso-routing/store.db"
9061 }
9062 })),
9063 "the delivered descriptor is the module's own sqlite store path"
9064 );
9065 }
9066
9067 #[test]
9068 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
9069 let registry = Arc::new(Registry::default());
9070 let handler = ControlHandler::new(Arc::clone(®istry));
9071 let conn = ConnectionId::new(1);
9072 let responses = handler
9073 .handle_control(
9074 conn,
9075 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9076 )
9077 .unwrap();
9078 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9079 let registration = registry.get_module("aft").unwrap().unwrap();
9080 assert_eq!(registration.control_ops, module_baseline_control_ops());
9081
9082 let frame =
9083 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
9084 assert!(handler
9085 .guard_module_control_op(&frame, "aft", "route.bind")
9086 .unwrap()
9087 .is_none());
9088 let error = handler
9089 .guard_module_control_op(&frame, "aft", "test.synthetic")
9090 .unwrap()
9091 .expect("synthetic ungranted op should be rejected");
9092 assert_eq!(error.header.ty, FrameType::Error);
9093 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
9094 }
9095
9096 #[test]
9097 fn hello_control_ops_some_adds_optional_grants() {
9098 let registry = Arc::new(Registry::default());
9099 let handler = ControlHandler::new(Arc::clone(®istry));
9100 handler
9101 .handle_control(
9102 ConnectionId::new(1),
9103 hello_frame_with_control_ops(
9104 "aft",
9105 PROTOCOL_VERSION,
9106 7,
9107 Some(vec![
9108 "future.synthetic".to_string(),
9109 "route.bind".to_string(),
9110 ]),
9111 ),
9112 )
9113 .unwrap();
9114 let registration = registry.get_module("aft").unwrap().unwrap();
9115 assert_eq!(
9116 registration.control_ops,
9117 vec![
9118 "route.bind".to_string(),
9119 "route.status".to_string(),
9120 "future.synthetic".to_string(),
9121 ]
9122 );
9123 let frame =
9124 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
9125 assert!(handler
9126 .guard_module_control_op(&frame, "aft", "future.synthetic")
9127 .unwrap()
9128 .is_none());
9129 }
9130
9131 #[tokio::test]
9132 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
9133 let registry = Arc::new(Registry::default());
9134 let forwarding = Arc::new(ForwardingTable::default());
9135 let handler =
9136 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9137 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
9138 hello_via_sink(
9139 &handler,
9140 &module_ctx,
9141 &mut module_rx,
9142 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9143 )
9144 .await;
9145
9146 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
9147 let responses = handler
9148 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
9149 .await
9150 .unwrap();
9151 assert_eq!(responses.len(), 1);
9152 assert_eq!(responses[0].header.ty, FrameType::Error);
9153 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
9154 assert!(module_rx.try_recv().is_err());
9155 }
9156
9157 #[tokio::test]
9158 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
9159 let registry = Arc::new(Registry::default());
9160 let forwarding = Arc::new(ForwardingTable::default());
9161 let handler =
9162 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9163 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
9164 hello_via_sink(
9165 &handler,
9166 &module_ctx,
9167 &mut module_rx,
9168 hello_frame_with_control_ops(
9169 "aft",
9170 PROTOCOL_VERSION,
9171 7,
9172 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9173 ),
9174 )
9175 .await;
9176
9177 let project_root = unique_project_root("demux");
9178 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
9179 let route_handler = handler.clone();
9180 let route_task = tokio::spawn(async move {
9181 route_handler
9182 .handle_control_frame(
9183 &route_client_ctx,
9184 route_open_frame(100, "aft", project_root),
9185 )
9186 .await
9187 .unwrap()
9188 });
9189 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9190 .await
9191 .unwrap()
9192 .unwrap();
9193 assert!(matches!(
9194 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
9195 ModuleControlRequest::RouteBind { .. }
9196 ));
9197
9198 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
9199 let health_handler = handler.clone();
9200 let health_task = tokio::spawn(async move {
9201 health_handler
9202 .handle_control_frame(
9203 &health_client_ctx,
9204 supervisor_health_probe_frame(101, "aft"),
9205 )
9206 .await
9207 .unwrap()
9208 });
9209 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9210 .await
9211 .unwrap()
9212 .unwrap();
9213 assert_eq!(
9214 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9215 ModuleControlRequest::HealthCheck {}
9216 );
9217
9218 handler
9219 .handle_control_frame(
9220 &module_ctx,
9221 health_response(health_frame.header.corr, HealthStatus::Degraded),
9222 )
9223 .await
9224 .unwrap();
9225 let health_response = health_task.await.unwrap();
9226 assert_eq!(health_response.len(), 1);
9227 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9228 ClientControlResponse::SupervisorHealthProbe {
9229 module_id,
9230 status,
9231 detail,
9232 metrics,
9233 } => {
9234 assert_eq!(module_id, "aft");
9235 assert_eq!(status, HealthStatus::Degraded);
9236 assert_eq!(detail.as_deref(), Some("warming"));
9237 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
9238 }
9239 other => panic!("unexpected health response: {other:?}"),
9240 }
9241
9242 handler
9243 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9244 .await
9245 .unwrap();
9246 let route_response = route_task.await.unwrap();
9247 assert!(route_response.is_empty());
9248 let published = route_client_rx.recv().await.unwrap();
9249 assert!(matches!(
9250 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9251 ClientControlResponse::RouteOpen { .. }
9252 ));
9253 }
9254
9255 async fn relay_route_open(
9260 handler: &ControlHandler,
9261 client_connection: ConnectionId,
9262 client_egress: &FrameSink,
9263 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9264 corr: u64,
9265 module_id: &str,
9266 project_root_label: &str,
9267 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
9268 let ctx = RouteCtx {
9269 connection_id: client_connection,
9270 egress: client_egress.clone(),
9271 };
9272 let handler = handler.clone();
9273 let project_root = unique_project_root(project_root_label);
9274 let module_id = module_id.to_string();
9275 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
9276 let task = tokio::spawn(async move {
9277 let _guard = tracing::dispatcher::set_default(&dispatch);
9278 handler
9279 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
9280 .await
9281 .unwrap()
9282 });
9283 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
9284 .await
9285 .expect("module receives the relayed route.bind")
9286 .expect("module egress is open");
9287 (task, bind.frame)
9288 }
9289
9290 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
9291 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
9292 ModuleControlRequest::RouteBind {
9293 route_channel,
9294 epoch,
9295 ..
9296 } => (route_channel, epoch),
9297 other => panic!("expected a route.bind request, got {other:?}"),
9298 }
9299 }
9300
9301 fn published_route(frame: &Frame) -> (u16, u32) {
9302 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
9303 ClientControlResponse::RouteOpen {
9304 route_channel,
9305 route_epoch,
9306 } => (route_channel, route_epoch),
9307 other => panic!("expected a route.open response, got {other:?}"),
9308 }
9309 }
9310
9311 #[tokio::test]
9331 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
9332 let registry = Arc::new(Registry::default());
9333 let forwarding = Arc::new(ForwardingTable::default());
9334 let handler =
9335 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9336
9337 let module_connection = ConnectionId::new(30);
9338 let (module_ctx, mut module_rx) = route_ctx(module_connection);
9339 hello_via_sink(
9340 &handler,
9341 &module_ctx,
9342 &mut module_rx,
9343 hello_frame("aft", PROTOCOL_VERSION, 7),
9344 )
9345 .await;
9346
9347 let dying_client = ConnectionId::new(31);
9348 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
9349
9350 let (first_task, first_bind) = relay_route_open(
9353 &handler,
9354 dying_client,
9355 &dying_ctx.egress,
9356 &mut module_rx,
9357 100,
9358 "aft",
9359 "closing-first",
9360 )
9361 .await;
9362 handler
9363 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
9364 .await
9365 .unwrap();
9366 assert!(first_task.await.unwrap().is_empty());
9367 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
9368
9369 let (second_task, second_bind) = relay_route_open(
9371 &handler,
9372 dying_client,
9373 &dying_ctx.egress,
9374 &mut module_rx,
9375 101,
9376 "aft",
9377 "closing-second",
9378 )
9379 .await;
9380 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
9381
9382 assert!(forwarding
9385 .escalate_client_delivery_failure(
9386 dying_client,
9387 first_channel,
9388 first_epoch,
9389 CloseReason::new(
9390 "module_to_client_delivery_failed",
9391 "client egress refused a module frame",
9392 ),
9393 crate::forwarding::UndeliveredFrame {
9394 module_id: None,
9395 sink: &dying_ctx.egress,
9396 },
9397 )
9398 .unwrap());
9399 assert!(!dying_ctx.egress.is_closed());
9400
9401 let ack = handler
9403 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
9404 .await;
9405 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
9406 if module_loop_error.is_some() {
9407 handler.cleanup_connection(module_connection).unwrap();
9411 }
9412 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9416 .await
9417 .ok()
9418 .flatten();
9419
9420 assert!(
9422 registry
9423 .get_module_by_connection(module_connection)
9424 .unwrap()
9425 .is_some(),
9426 "one client's closing connection ended the shared module connection: \
9427 {module_loop_error:?}"
9428 );
9429 let cotenant = ConnectionId::new(32);
9431 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
9432 let (cotenant_task, cotenant_bind) = relay_route_open(
9433 &handler,
9434 cotenant,
9435 &cotenant_ctx.egress,
9436 &mut module_rx,
9437 102,
9438 "aft",
9439 "closing-cotenant",
9440 )
9441 .await;
9442 handler
9443 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
9444 .await
9445 .unwrap();
9446 assert!(cotenant_task.await.unwrap().is_empty());
9447 let (cotenant_channel, cotenant_epoch) =
9448 published_route(&cotenant_rx.recv().await.unwrap());
9449 assert!(matches!(
9450 forwarding
9451 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
9452 .unwrap(),
9453 DataRoute::Client(DataRouteState::Bound(_))
9454 ));
9455
9456 let goodbye = post_ack_module_frame
9459 .expect("module receives a GOODBYE for the abandoned route channel");
9460 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9461 assert_eq!(goodbye.header.channel, abandoned_channel);
9462 assert_eq!(goodbye.header.epoch, abandoned_epoch);
9463
9464 assert!(dying_rx.try_recv().is_err());
9468 let second_response = second_task.await.unwrap();
9469 assert_eq!(second_response.len(), 1);
9470 assert_eq!(
9471 parse_error(&second_response[0])["code"],
9472 "target_unavailable"
9473 );
9474 }
9475
9476 async fn catalog_snapshot(
9479 handler: &ControlHandler,
9480 client_ctx: &RouteCtx,
9481 corr: u64,
9482 ) -> (u64, Vec<(String, bool)>) {
9483 let body =
9484 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
9485 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
9486 let reply = handler
9487 .handle_control_frame(client_ctx, request)
9488 .await
9489 .unwrap()
9490 .pop()
9491 .unwrap();
9492 let ClientControlResponse::CatalogList {
9493 generation,
9494 modules,
9495 ..
9496 } = serde_json::from_slice(&reply.body).unwrap()
9497 else {
9498 panic!("not a catalog.list reply");
9499 };
9500 (
9501 generation,
9502 modules
9503 .into_iter()
9504 .map(|entry| (entry.module_id, entry.ready))
9505 .collect(),
9506 )
9507 }
9508
9509 fn hello_held_before_forwarding() -> (
9515 ControlHandler,
9516 Arc<std::sync::Barrier>,
9517 std::thread::JoinHandle<Vec<Frame>>,
9518 RouteCtx,
9519 mpsc::Receiver<crate::router::OutboundFrame>,
9520 ) {
9521 let hold = Arc::new(std::sync::Barrier::new(2));
9522 let handler = ControlHandler::with_forwarding(
9523 Arc::new(Registry::default()),
9524 Arc::new(ForwardingTable::default()),
9525 )
9526 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
9527 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9528 let hello = {
9529 let handler = handler.clone();
9530 let module_ctx = module_ctx.clone();
9531 std::thread::spawn(move || {
9532 handler
9533 .handle_hello(
9534 module_ctx.connection_id,
9535 Some(module_ctx.egress.clone()),
9536 hello_frame("aft", PROTOCOL_VERSION, 7),
9537 )
9538 .unwrap()
9539 })
9540 };
9541 (handler, hold, hello, module_ctx, module_rx)
9542 }
9543
9544 #[tokio::test]
9553 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
9554 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
9555 let client = ConnectionId::new(41);
9556 let (client_ctx, mut client_rx) = route_ctx(client);
9557
9558 hold.wait();
9559 assert!(handler.registry.get_module("aft").unwrap().is_some());
9561 assert!(!handler
9562 .forwarding
9563 .has_live_module_connection("aft")
9564 .unwrap());
9565 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
9566 assert!(
9567 listed.is_empty(),
9568 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
9569 );
9570 let refused = handler
9571 .handle_control_frame(
9572 &client_ctx,
9573 route_open_frame(301, "aft", unique_project_root("listed-window")),
9574 )
9575 .await
9576 .unwrap();
9577 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
9578 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
9579 let duplicate = handler
9580 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
9581 .await
9582 .unwrap();
9583 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9584
9585 hold.wait();
9586 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9587 let ack = module_rx.try_recv().unwrap().frame;
9588 assert_eq!(ack.header.ty, FrameType::HelloAck);
9589 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9590 assert_eq!(listed, vec![("aft".to_string(), true)]);
9591
9592 let (open_task, bind) = relay_route_open(
9593 &handler,
9594 client,
9595 &client_ctx.egress,
9596 &mut module_rx,
9597 303,
9598 "aft",
9599 "listed-after",
9600 )
9601 .await;
9602 assert!(handler
9603 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9604 .await
9605 .unwrap()
9606 .is_empty());
9607 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
9608 published_route(&client_rx.try_recv().unwrap().frame);
9609 }
9610
9611 #[tokio::test]
9618 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
9619 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
9620 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9621 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9622
9623 hold.wait();
9624 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
9625 assert!(listed.is_empty());
9626 assert_eq!(
9627 during, before,
9628 "the generation moved while the module was not listed"
9629 );
9630
9631 hold.wait();
9632 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9633 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9634 assert_eq!(listed, vec![("aft".to_string(), true)]);
9635 assert_eq!(
9636 after,
9637 before + 1,
9638 "the module appeared without exactly one generation bump"
9639 );
9640 }
9641
9642 #[tokio::test]
9647 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
9648 let registry = Arc::new(Registry::default());
9649 let handler = ControlHandler::with_forwarding(
9650 Arc::clone(®istry),
9651 Arc::new(ForwardingTable::default()),
9652 );
9653 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9654 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9655 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9656 drop(module_rx);
9657
9658 let replies = handler
9659 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
9660 .await
9661 .unwrap();
9662 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
9663 assert!(registry.get_module("aft").unwrap().is_none());
9664 assert!(registry
9665 .get_module_by_connection(module_ctx.connection_id)
9666 .unwrap()
9667 .is_none());
9668 assert_eq!(
9669 catalog_snapshot(&handler, &client_ctx, 301).await,
9670 (before, Vec::new())
9671 );
9672 }
9673
9674 async fn route_open_held_after_bind_send() -> (
9678 ControlHandler,
9679 Arc<tokio::sync::Barrier>,
9680 RouteCtx,
9681 mpsc::Receiver<crate::router::OutboundFrame>,
9682 ) {
9683 let hold = Arc::new(tokio::sync::Barrier::new(2));
9684 let handler = ControlHandler::with_forwarding(
9685 Arc::new(Registry::default()),
9686 Arc::new(ForwardingTable::default()),
9687 )
9688 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
9689 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
9690 hello_via_sink(
9691 &handler,
9692 &module_ctx,
9693 &mut module_rx,
9694 hello_frame("aft", PROTOCOL_VERSION, 7),
9695 )
9696 .await;
9697 (handler, hold, module_ctx, module_rx)
9698 }
9699
9700 #[tokio::test]
9707 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
9708 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9709 let client = ConnectionId::new(41);
9710 let (client_ctx, mut client_rx) = route_ctx(client);
9711
9712 let (open_task, bind) = relay_route_open(
9713 &handler,
9714 client,
9715 &client_ctx.egress,
9716 &mut module_rx,
9717 200,
9718 "aft",
9719 "acked-before-mark",
9720 )
9721 .await;
9722 let (module_channel, module_epoch) = route_bind_channel(&bind);
9723 hold.wait().await;
9725
9726 assert!(handler
9728 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9729 .await
9730 .unwrap()
9731 .is_empty());
9732 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
9733
9734 hold.wait().await;
9735 assert!(
9736 open_task.await.unwrap().is_empty(),
9737 "route.open answers accepted"
9738 );
9739
9740 match module_rx.try_recv() {
9743 Err(mpsc::error::TryRecvError::Empty) => {}
9744 Ok(outbound) => panic!(
9745 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
9746 outbound.frame.header.ty,
9747 outbound.frame.header.channel,
9748 outbound.frame.header.epoch
9749 ),
9750 Err(err) => panic!("module egress closed: {err}"),
9751 }
9752
9753 let router = Router::with_control_handler(Arc::new(handler.clone()));
9756 let request = Frame::build(
9757 FrameType::Request,
9758 Flags::new(true, Priority::Interactive, false),
9759 client_channel,
9760 client_epoch,
9761 201,
9762 b"first request".to_vec(),
9763 )
9764 .unwrap();
9765 router
9766 .route_for_connection(&client_ctx, request)
9767 .await
9768 .unwrap();
9769 let delivered = module_rx.try_recv().unwrap().frame;
9770 assert_eq!(delivered.header.ty, FrameType::Request);
9771 assert_eq!(
9772 (delivered.header.channel, delivered.header.epoch),
9773 (module_channel, module_epoch)
9774 );
9775 assert_eq!(delivered.body, b"first request");
9776 }
9777
9778 #[tokio::test]
9784 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
9785 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9786 let client = ConnectionId::new(42);
9787 let (client_ctx, _client_rx) = route_ctx(client);
9788
9789 let (open_task, bind) = relay_route_open(
9790 &handler,
9791 client,
9792 &client_ctx.egress,
9793 &mut module_rx,
9794 300,
9795 "aft",
9796 "released-before-mark",
9797 )
9798 .await;
9799 let (module_channel, module_epoch) = route_bind_channel(&bind);
9800 hold.wait().await;
9801
9802 handler.cleanup_connection(client).unwrap();
9803 assert!(
9804 module_rx.try_recv().is_err(),
9805 "client teardown must not GOODBYE a bind route.open is still queueing"
9806 );
9807
9808 hold.wait().await;
9809 let response = open_task.await.unwrap();
9810 assert_eq!(response.len(), 1);
9811 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9812
9813 let goodbye = module_rx
9814 .try_recv()
9815 .expect("module is told to drop the binding the daemon abandoned")
9816 .frame;
9817 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9818 assert_eq!(
9819 (goodbye.header.channel, goodbye.header.epoch),
9820 (module_channel, module_epoch)
9821 );
9822 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9823 }
9824
9825 #[tokio::test]
9830 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
9831 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9832 let client = ConnectionId::new(43);
9833 let (client_ctx, client_rx) = route_ctx(client);
9834
9835 let (open_task, bind) = relay_route_open(
9836 &handler,
9837 client,
9838 &client_ctx.egress,
9839 &mut module_rx,
9840 400,
9841 "aft",
9842 "gone-client-before-mark",
9843 )
9844 .await;
9845 let (module_channel, module_epoch) = route_bind_channel(&bind);
9846 hold.wait().await;
9847
9848 drop(client_rx);
9849 assert!(client_ctx.egress.is_closed());
9850 handler
9851 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9852 .await
9853 .unwrap();
9854
9855 hold.wait().await;
9856 let response = open_task.await.unwrap();
9857 assert_eq!(response.len(), 1);
9858 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9859
9860 let goodbye = module_rx
9861 .try_recv()
9862 .expect("module is told to drop the binding it acked")
9863 .frame;
9864 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9865 assert_eq!(
9866 (goodbye.header.channel, goodbye.header.epoch),
9867 (module_channel, module_epoch)
9868 );
9869 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9870 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
9871 }
9872
9873 #[test]
9880 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
9881 let handler = ControlHandler::default();
9882 let module_connection = ConnectionId::new(30);
9883 let client_connection = ConnectionId::new(31);
9884
9885 handler
9886 .refuse_to_end_module_connection_for_a_client(
9887 module_connection,
9888 77,
9889 ForwardingError::ConnectionClosing {
9890 connection_id: client_connection,
9891 },
9892 )
9893 .expect("a closing client must never end the module connection");
9894
9895 assert!(matches!(
9896 handler.refuse_to_end_module_connection_for_a_client(
9897 module_connection,
9898 78,
9899 ForwardingError::ConnectionClosing {
9900 connection_id: module_connection,
9901 },
9902 ),
9903 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
9904 connection_id
9905 })) if connection_id == module_connection
9906 ));
9907 assert!(matches!(
9908 handler.refuse_to_end_module_connection_for_a_client(
9909 module_connection,
9910 79,
9911 ForwardingError::Poisoned,
9912 ),
9913 Err(RouterError::Forwarding(ForwardingError::Poisoned))
9914 ));
9915 assert!(matches!(
9916 handler.refuse_to_end_module_connection_for_a_client(
9917 module_connection,
9918 80,
9919 ForwardingError::StaleModuleEndpoint,
9920 ),
9921 Err(RouterError::Forwarding(
9922 ForwardingError::StaleModuleEndpoint
9923 ))
9924 ));
9925 }
9926
9927 #[tokio::test]
9940 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
9941 let registry = Arc::new(Registry::default());
9942 let forwarding = Arc::new(ForwardingTable::default());
9943 let supervisor = SupervisorHandle::new();
9944 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9945 let handler =
9946 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9947 .with_supervisor(supervisor);
9948
9949 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9950 hello_via_sink(
9951 &handler,
9952 &target_ctx,
9953 &mut target_rx,
9954 hello_frame("target", PROTOCOL_VERSION, 1),
9955 )
9956 .await;
9957
9958 let wrong_nonce = handler
9962 .handle_control_frame(
9963 &route_ctx(ConnectionId::new(91)).0,
9964 route_open_frame_with_admission_facts(
9965 20,
9966 "target",
9967 unique_project_root("admission-facts"),
9968 Some(subc_control::ConsumerIdentity {
9969 module_id: "fed".to_string(),
9970 launch_nonce: "not-the-real-nonce".to_string(),
9971 }),
9972 None,
9973 ),
9974 )
9975 .await
9976 .unwrap();
9977 assert_eq!(
9978 parse_error(&wrong_nonce[0])["code"],
9979 "bad_consumer_identity",
9980 "a mismatched launch nonce must be refused, not stamped Reserved"
9981 );
9982
9983 let never_spawned = handler
9987 .handle_control_frame(
9988 &route_ctx(ConnectionId::new(92)).0,
9989 route_open_frame_with_admission_facts(
9990 21,
9991 "target",
9992 unique_project_root("admission-facts"),
9993 Some(subc_control::ConsumerIdentity {
9994 module_id: "never-spawned".to_string(),
9995 launch_nonce: "any-nonce".to_string(),
9996 }),
9997 None,
9998 ),
9999 )
10000 .await
10001 .unwrap();
10002 assert_eq!(
10003 parse_error(&never_spawned[0])["code"],
10004 "bad_consumer_identity",
10005 "an unspawned module_id must be refused rather than accepted for lack of a record"
10006 );
10007 }
10008
10009 #[tokio::test]
10032 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
10033 let registry = Arc::new(Registry::default());
10034 let forwarding = Arc::new(ForwardingTable::default());
10035 let supervisor = SupervisorHandle::new();
10036 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10037 let handler =
10038 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10039 .with_supervisor(supervisor);
10040
10041 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
10042 hello_via_sink(
10043 &handler,
10044 &target_ctx,
10045 &mut target_rx,
10046 hello_frame("target", PROTOCOL_VERSION, 1),
10047 )
10048 .await;
10049
10050 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
10051 let route_handler = handler.clone();
10052 let route_task = tokio::spawn(async move {
10053 route_handler
10054 .handle_control_frame(
10055 &client_ctx,
10056 route_open_frame_with_admission_facts(
10057 30,
10058 "target",
10059 unique_project_root("admission-facts"),
10060 Some(subc_control::ConsumerIdentity {
10061 module_id: "fed".to_string(),
10062 launch_nonce: "fed-nonce".to_string(),
10063 }),
10064 None,
10065 ),
10066 )
10067 .await
10068 .unwrap()
10069 });
10070
10071 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10078 .await
10079 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
10080 .expect("module control channel closed before route.bind");
10081 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10082 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10083 panic!("expected route.bind")
10084 };
10085 assert_eq!(
10086 principal,
10087 Some(Principal::Reserved {
10088 module_id: "fed".to_string()
10089 }),
10090 "a correctly attested consumer must be stamped Reserved for its own id"
10091 );
10092
10093 handler
10094 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10095 .await
10096 .unwrap();
10097 assert!(route_task.await.unwrap().is_empty());
10098 assert!(
10099 matches!(
10100 serde_json::from_slice::<ClientControlResponse>(
10101 &client_rx.recv().await.unwrap().body
10102 )
10103 .unwrap(),
10104 ClientControlResponse::RouteOpen { .. }
10105 ),
10106 "the route must actually open, not merely avoid an error"
10107 );
10108 }
10109
10110 #[tokio::test(start_paused = true)]
10111 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
10112 let registry = Arc::new(Registry::default());
10113 let forwarding = Arc::new(ForwardingTable::default());
10114 let supervisor = SupervisorHandle::new();
10115 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10116 let handler =
10117 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10118 .with_supervisor(supervisor);
10119
10120 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
10121 hello_via_sink(
10122 &handler,
10123 &target_ctx,
10124 &mut target_rx,
10125 hello_frame("target", PROTOCOL_VERSION, 1),
10126 )
10127 .await;
10128
10129 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
10130 let direct_handler = handler.clone();
10131 let direct_open = tokio::spawn(async move {
10132 direct_handler
10133 .handle_control_frame(
10134 &direct_ctx,
10135 route_open_frame(2, "target", unique_project_root("route-census-direct")),
10136 )
10137 .await
10138 .unwrap()
10139 });
10140 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10141 .await
10142 .expect("no direct route.bind within 5s")
10143 .expect("target control channel closed before direct route.bind");
10144 handler
10145 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
10146 .await
10147 .unwrap();
10148 assert!(direct_open.await.unwrap().is_empty());
10149 let _ = direct_rx.recv().await.unwrap();
10150
10151 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
10152 let reserved_handler = handler.clone();
10153 let reserved_open = tokio::spawn(async move {
10154 reserved_handler
10155 .handle_control_frame(
10156 &reserved_ctx,
10157 route_open_frame_with_admission_facts(
10158 3,
10159 "target",
10160 unique_project_root("admission-facts"),
10161 Some(ConsumerIdentity {
10162 module_id: "fed".to_string(),
10163 launch_nonce: "fed-nonce".to_string(),
10164 }),
10165 None,
10166 ),
10167 )
10168 .await
10169 .unwrap()
10170 });
10171 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10172 .await
10173 .expect("no reserved route.bind within 5s")
10174 .expect("target control channel closed before reserved route.bind");
10175 handler
10176 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
10177 .await
10178 .unwrap();
10179 assert!(reserved_open.await.unwrap().is_empty());
10180 let _ = reserved_rx.recv().await.unwrap();
10181
10182 forwarding
10183 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
10184 .unwrap();
10185 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
10186 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
10187 module_id: Some("target".to_string()),
10188 })
10189 .unwrap();
10190 let census_frame =
10191 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
10192 let response = handler
10193 .handle_control_frame(&census_ctx, census_frame)
10194 .await
10195 .unwrap()
10196 .pop()
10197 .unwrap();
10198 let actual: Value = serde_json::from_slice(&response.body).unwrap();
10199 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
10200 assert!(matches!(
10201 decoded,
10202 ClientControlResponse::SupervisorRoutes { .. }
10203 ));
10204 let routes = actual["modules"][0]["routes"].as_array().unwrap();
10205 assert_eq!(routes.len(), 2);
10206 assert!(routes.iter().all(|route| route["draining"] == true));
10207 assert!(
10210 routes.iter().all(|route| route["drain_reason"] == "reload"),
10211 "draining routes must name the drain's reason: {routes:?}"
10212 );
10213 assert!(routes.iter().any(|route| {
10214 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
10215 }));
10216 assert!(routes.iter().any(|route| {
10217 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
10218 }));
10219
10220 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10221 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
10222 if std::env::var_os("UPDATE_GOLDEN").is_some() {
10223 std::fs::write(
10224 &golden_path,
10225 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
10226 )
10227 .unwrap();
10228 }
10229 let expected: Value =
10230 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
10231 assert_eq!(actual, expected);
10232 }
10233
10234 async fn query_live_roots(
10235 handler: &ControlHandler,
10236 module_ctx: &RouteCtx,
10237 ) -> ModuleControlResponseToModule {
10238 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
10239 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
10240 let response = handler
10241 .handle_control_frame(module_ctx, frame)
10242 .await
10243 .unwrap()
10244 .pop()
10245 .unwrap();
10246 serde_json::from_slice(&response.body).unwrap()
10247 }
10248
10249 #[tokio::test(start_paused = true)]
10250 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
10251 let registry = Arc::new(Registry::default());
10252 let forwarding = Arc::new(ForwardingTable::default());
10253 let handler = ControlHandler::with_forwarding(registry, forwarding);
10254 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
10255 hello_via_sink(
10256 &handler,
10257 &target_ctx,
10258 &mut target_rx,
10259 hello_frame("target", PROTOCOL_VERSION, 1),
10260 )
10261 .await;
10262 let root = unique_project_root("live-roots-known");
10263 let path = ProjectRootId::from_path_allowing_missing(root.path())
10264 .unwrap()
10265 .as_path()
10266 .to_path_buf();
10267 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
10268 let open_handler = handler.clone();
10269 let opened = tokio::spawn(async move {
10270 open_handler
10271 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
10272 .await
10273 .unwrap()
10274 });
10275 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10276 .await
10277 .unwrap()
10278 .unwrap();
10279 handler
10280 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
10281 .await
10282 .unwrap();
10283 assert!(opened.await.unwrap().is_empty());
10284 let _ = client_rx.recv().await.unwrap();
10285
10286 let root = unique_project_root("live-roots-pending");
10287 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
10288 .unwrap()
10289 .as_path()
10290 .to_path_buf();
10291 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
10292 let open_handler = handler.clone();
10293 let pending = tokio::spawn(async move {
10294 open_handler
10295 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
10296 .await
10297 .unwrap()
10298 });
10299 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10300 .await
10301 .unwrap()
10302 .unwrap();
10303 let actual = query_live_roots(&handler, &target_ctx).await;
10304 let ModuleControlResponseToModule::LiveRoots {
10305 roots,
10306 unknown_root_bindings,
10307 total_bindings,
10308 } = actual
10309 else {
10310 panic!("expected live roots")
10311 };
10312 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
10313 assert_eq!(unknown_root_bindings, 0);
10314 assert_eq!(
10315 roots.len(),
10316 2,
10317 "root-known arm must retain each canonical root"
10318 );
10319 assert_eq!(
10320 total_bindings,
10321 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10322 );
10323 let counts = roots
10324 .iter()
10325 .map(|root| (root.project_root.clone(), root.bound, root.pending))
10326 .collect::<Vec<_>>();
10327 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
10328 expected.sort_by(|a, b| a.0.cmp(&b.0));
10329 assert_eq!(
10330 counts, expected,
10331 "roots must sort by path and count pending separately"
10332 );
10333 handler
10334 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
10335 .await
10336 .unwrap();
10337 assert!(pending.await.unwrap().is_empty());
10338 }
10339
10340 #[tokio::test(start_paused = true)]
10341 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
10342 let forwarding = Arc::new(ForwardingTable::default());
10343 let handler =
10344 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10345 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
10346 hello_via_sink(
10347 &handler,
10348 &target_ctx,
10349 &mut target_rx,
10350 hello_frame("target", PROTOCOL_VERSION, 1),
10351 )
10352 .await;
10353 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
10354 let pending = forwarding
10355 .begin_route_bind_relay_for_test(
10356 client_ctx.connection_id,
10357 client_ctx.egress.clone(),
10358 2,
10359 "target",
10360 )
10361 .unwrap();
10362 forwarding
10363 .complete_pending_relay(
10364 target_ctx.connection_id,
10365 pending.corr,
10366 RouteBindRelayOutcome::Accepted,
10367 )
10368 .unwrap();
10369 let actual = query_live_roots(&handler, &target_ctx).await;
10370 let ModuleControlResponseToModule::LiveRoots {
10371 roots,
10372 unknown_root_bindings,
10373 total_bindings,
10374 } = actual
10375 else {
10376 panic!("expected live roots")
10377 };
10378 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
10379 assert_eq!(
10380 unknown_root_bindings, 1,
10381 "unknown-root arm must not read as no bindings"
10382 );
10383 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
10384 assert_eq!(
10385 total_bindings,
10386 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10387 );
10388 }
10389
10390 #[tokio::test(start_paused = true)]
10398 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
10399 let forwarding = Arc::new(ForwardingTable::default());
10400 let handler =
10401 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10402 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
10403 let replies = handler
10404 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
10405 .await
10406 .unwrap();
10407 let queued_by_hello = module_rx.len();
10408
10409 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
10410 let open_handler = handler.clone();
10411 let open = tokio::spawn(async move {
10412 open_handler
10413 .handle_control_frame(
10414 &client_ctx,
10415 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
10416 )
10417 .await
10418 .unwrap()
10419 });
10420 let mut spins = 0;
10422 while module_rx.len() == queued_by_hello {
10423 spins += 1;
10424 assert!(spins < 10_000, "route.open never queued a route.bind");
10425 tokio::task::yield_now().await;
10426 }
10427
10428 for reply in replies {
10430 module_ctx.egress.send(reply).await.unwrap();
10431 }
10432
10433 let first = module_rx.recv().await.unwrap().frame;
10434 assert_eq!(
10435 first.header.ty,
10436 FrameType::HelloAck,
10437 "the first frame a registering module reads must be its HELLO_ACK"
10438 );
10439 assert_eq!(first.header.corr, 7);
10440 let second = module_rx.recv().await.unwrap().frame;
10441 assert_eq!(second.header.ty, FrameType::Request);
10442 assert!(
10443 matches!(
10444 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
10445 ModuleControlRequest::RouteBind { .. }
10446 ),
10447 "the route.bind follows the ack"
10448 );
10449 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
10450
10451 handler
10452 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
10453 .await
10454 .unwrap();
10455 assert!(open.await.unwrap().is_empty());
10456 let _ = client_rx.recv().await.unwrap();
10457 }
10458
10459 #[tokio::test(start_paused = true)]
10460 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
10461 let handler = ControlHandler::with_forwarding(
10462 Arc::new(Registry::default()),
10463 Arc::new(ForwardingTable::default()),
10464 );
10465 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
10466 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
10467 hello_via_sink(
10468 &handler,
10469 &first_ctx,
10470 &mut first_rx,
10471 hello_frame("first", PROTOCOL_VERSION, 1),
10472 )
10473 .await;
10474 hello_via_sink(
10475 &handler,
10476 &second_ctx,
10477 &mut second_rx,
10478 hello_frame("second", PROTOCOL_VERSION, 2),
10479 )
10480 .await;
10481 let root = unique_project_root("second-only");
10482 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
10483 let cloned = handler.clone();
10484 let open = tokio::spawn(async move {
10485 cloned
10486 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
10487 .await
10488 .unwrap()
10489 });
10490 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
10491 .await
10492 .unwrap()
10493 .unwrap();
10494 let first = query_live_roots(&handler, &first_ctx).await;
10495 let second = query_live_roots(&handler, &second_ctx).await;
10496 assert!(
10497 matches!(
10498 first,
10499 ModuleControlResponseToModule::LiveRoots {
10500 total_bindings: 0,
10501 ..
10502 }
10503 ),
10504 "cross-module scope must not expose another module's roots"
10505 );
10506 assert!(
10507 matches!(
10508 second,
10509 ModuleControlResponseToModule::LiveRoots {
10510 total_bindings: 1,
10511 ..
10512 }
10513 ),
10514 "second module must see its pending route"
10515 );
10516 handler
10517 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
10518 .await
10519 .unwrap();
10520 assert!(open.await.unwrap().is_empty());
10521 }
10522
10523 #[tokio::test(start_paused = true)]
10524 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
10525 let handler = ControlHandler::with_forwarding(
10526 Arc::new(Registry::default()),
10527 Arc::new(ForwardingTable::default()),
10528 );
10529 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
10530 hello_via_sink(
10531 &handler,
10532 &target_ctx,
10533 &mut target_rx,
10534 hello_frame("target", PROTOCOL_VERSION, 1),
10535 )
10536 .await;
10537 let actual = query_live_roots(&handler, &target_ctx).await;
10538 let ModuleControlResponseToModule::LiveRoots {
10539 roots,
10540 unknown_root_bindings,
10541 total_bindings,
10542 } = actual
10543 else {
10544 panic!("expected live roots")
10545 };
10546 assert!(roots.is_empty());
10547 assert_eq!(unknown_root_bindings, 0);
10548 assert_eq!(total_bindings, 0);
10549 assert_eq!(
10550 total_bindings,
10551 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10552 );
10553 }
10554
10555 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
10562 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10563 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
10564 let text = std::fs::read_to_string(&path)
10565 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
10566 let vectors: Vec<(String, Value)> = text
10567 .lines()
10568 .filter(|line| !line.trim().is_empty())
10569 .map(|line| {
10570 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
10571 let id = entry["corpus_id"]
10572 .as_str()
10573 .expect("every vector carries a corpus_id")
10574 .to_string();
10575 (id, entry["package"].clone())
10576 })
10577 .collect();
10578 assert_eq!(
10582 vectors.len(),
10583 3,
10584 "vendored fed corpus changed size; re-sync from subc-federation"
10585 );
10586
10587 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
10602 let richest = vectors
10603 .iter()
10604 .filter_map(|(_, package)| package.as_object())
10605 .map(|object| {
10606 object
10607 .keys()
10608 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
10609 .count()
10610 })
10611 .max()
10612 .unwrap_or(0);
10613 assert!(
10614 richest >= 2,
10615 "vendored corpus no longer carries a package with unmodelled fields, \
10616 so the relay test can no longer distinguish a verbatim relay from a lossy one"
10617 );
10618
10619 vectors
10620 }
10621
10622 #[tokio::test]
10630 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
10631 for (corpus_id, package) in fed_admission_facts_vectors() {
10632 let registry = Arc::new(Registry::default());
10633 let forwarding = Arc::new(ForwardingTable::default());
10634 let supervisor = SupervisorHandle::new();
10635 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10636 let handler =
10637 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10638 .with_supervisor(supervisor)
10639 .with_admission_facts_config(
10640 Some("fed".to_string()),
10641 Some(vec!["target".to_string()]),
10642 );
10643
10644 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10645 hello_via_sink(
10646 &handler,
10647 &target_ctx,
10648 &mut target_rx,
10649 hello_frame("target", PROTOCOL_VERSION, 1),
10650 )
10651 .await;
10652
10653 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
10654 let route_handler = handler.clone();
10655 let expected = package.clone();
10656 let route_task = tokio::spawn(async move {
10657 route_handler
10658 .handle_control_frame(
10659 &client_ctx,
10660 route_open_frame_with_admission_facts(
10661 20,
10662 "target",
10663 unique_project_root("admission-facts"),
10664 Some(subc_control::ConsumerIdentity {
10665 module_id: "fed".to_string(),
10666 launch_nonce: "fed-nonce".to_string(),
10667 }),
10668 Some(package),
10669 ),
10670 )
10671 .await
10672 .unwrap()
10673 });
10674
10675 let bind_frame = target_rx.recv().await.unwrap();
10676 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10677 let ModuleControlRequest::RouteBind {
10678 admission_facts, ..
10679 } = bind
10680 else {
10681 panic!("{corpus_id}: expected route.bind")
10682 };
10683 assert_eq!(
10684 admission_facts,
10685 Some(expected),
10686 "{corpus_id}: relay must not add, drop or reshape any field"
10687 );
10688
10689 handler
10690 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10691 .await
10692 .unwrap();
10693 route_task.await.unwrap();
10694 }
10695 }
10696
10697 #[tokio::test]
10698 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
10699 let registry = Arc::new(Registry::default());
10700 let forwarding = Arc::new(ForwardingTable::default());
10701 let supervisor = SupervisorHandle::new();
10702 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10703 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
10704 let handler =
10705 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10706 .with_supervisor(supervisor)
10707 .with_admission_facts_config(
10708 Some("fed".to_string()),
10709 Some(vec!["target".to_string()]),
10710 );
10711
10712 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
10713 hello_via_sink(
10714 &handler,
10715 &target_ctx,
10716 &mut target_rx,
10717 hello_frame("target", PROTOCOL_VERSION, 1),
10718 )
10719 .await;
10720 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
10721 hello_via_sink(
10722 &handler,
10723 &other_ctx,
10724 &mut other_rx,
10725 hello_frame("other", PROTOCOL_VERSION, 2),
10726 )
10727 .await;
10728
10729 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
10730 let expected_facts = facts.clone();
10731 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
10732 let route_handler = handler.clone();
10733 let route_task = tokio::spawn(async move {
10734 route_handler
10735 .handle_control_frame(
10736 &client_ctx,
10737 route_open_frame_with_admission_facts(
10738 10,
10739 "target",
10740 unique_project_root("admission-facts"),
10741 Some(subc_control::ConsumerIdentity {
10742 module_id: "fed".to_string(),
10743 launch_nonce: "fed-nonce".to_string(),
10744 }),
10745 Some(facts.clone()),
10746 ),
10747 )
10748 .await
10749 .unwrap()
10750 });
10751 let bind_frame = target_rx.recv().await.unwrap();
10752 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10753 let ModuleControlRequest::RouteBind {
10754 admission_facts, ..
10755 } = bind
10756 else {
10757 panic!("expected route.bind")
10758 };
10759 assert_eq!(admission_facts, Some(expected_facts));
10760 handler
10761 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10762 .await
10763 .unwrap();
10764 assert!(route_task.await.unwrap().is_empty());
10765 assert!(matches!(
10766 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
10767 .unwrap(),
10768 ClientControlResponse::RouteOpen { .. }
10769 ));
10770
10771 let direct = handler
10772 .handle_control_frame(
10773 &route_ctx(ConnectionId::new(73)).0,
10774 route_open_frame_with_admission_facts(
10775 11,
10776 "target",
10777 unique_project_root("admission-facts"),
10778 None,
10779 Some(json!({"x": 1})),
10780 ),
10781 )
10782 .await
10783 .unwrap();
10784 assert_eq!(
10785 parse_error(&direct[0])["code"],
10786 "admission_facts_not_permitted"
10787 );
10788
10789 let different_reserved = handler
10790 .handle_control_frame(
10791 &route_ctx(ConnectionId::new(77)).0,
10792 route_open_frame_with_admission_facts(
10793 15,
10794 "target",
10795 unique_project_root("admission-facts"),
10796 Some(subc_control::ConsumerIdentity {
10797 module_id: "other".to_string(),
10798 launch_nonce: "other-nonce".to_string(),
10799 }),
10800 Some(json!({"x": 1})),
10801 ),
10802 )
10803 .await
10804 .unwrap();
10805 assert_eq!(
10806 parse_error(&different_reserved[0])["code"],
10807 "admission_facts_not_permitted"
10808 );
10809
10810 let other_target = handler
10811 .handle_control_frame(
10812 &route_ctx(ConnectionId::new(74)).0,
10813 route_open_frame_with_admission_facts(
10814 12,
10815 "other",
10816 unique_project_root("admission-facts"),
10817 Some(subc_control::ConsumerIdentity {
10818 module_id: "fed".to_string(),
10819 launch_nonce: "fed-nonce".to_string(),
10820 }),
10821 Some(json!({"x": 1})),
10822 ),
10823 )
10824 .await
10825 .unwrap();
10826 assert_eq!(
10827 parse_error(&other_target[0])["code"],
10828 "admission_facts_target_not_allowed"
10829 );
10830
10831 let nonexistent = handler
10832 .handle_control_frame(
10833 &route_ctx(ConnectionId::new(75)).0,
10834 route_open_frame_with_admission_facts(
10835 13,
10836 "missing",
10837 unique_project_root("admission-facts"),
10838 None,
10839 Some(json!({"x": 1})),
10840 ),
10841 )
10842 .await
10843 .unwrap();
10844 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
10845
10846 let described = handler
10847 .handle_control_frame(
10848 &route_ctx(ConnectionId::new(76)).0,
10849 Frame::build(
10850 FrameType::Request,
10851 control_flags(),
10852 0,
10853 0,
10854 14,
10855 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10856 )
10857 .unwrap(),
10858 )
10859 .await
10860 .unwrap();
10861 let ClientControlResponse::ServerDescribe { capabilities, .. } =
10862 serde_json::from_slice(&described[0].body).unwrap()
10863 else {
10864 panic!("expected server.describe response")
10865 };
10866 assert!(capabilities
10867 .iter()
10868 .any(|cap| cap == "admission_facts_relay_v1"));
10869 }
10870
10871 #[tokio::test]
10872 async fn admission_facts_without_configured_carrier_are_rejected() {
10873 let registry = Arc::new(Registry::default());
10874 let forwarding = Arc::new(ForwardingTable::default());
10875 let handler = ControlHandler::with_forwarding(registry, forwarding);
10876 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
10877 hello_via_sink(
10878 &handler,
10879 &target_ctx,
10880 &mut target_rx,
10881 hello_frame("target", PROTOCOL_VERSION, 1),
10882 )
10883 .await;
10884
10885 let responses = handler
10886 .handle_control_frame(
10887 &route_ctx(ConnectionId::new(79)).0,
10888 route_open_frame_with_admission_facts(
10889 16,
10890 "target",
10891 unique_project_root("admission-facts"),
10892 None,
10893 Some(json!({"x": 1})),
10894 ),
10895 )
10896 .await
10897 .unwrap();
10898 assert_eq!(
10899 parse_error(&responses[0])["code"],
10900 "admission_facts_not_permitted"
10901 );
10902 }
10903
10904 #[tokio::test]
10905 async fn route_open_relays_consumer_capabilities_verbatim() {
10906 let registry = Arc::new(Registry::default());
10907 let forwarding = Arc::new(ForwardingTable::default());
10908 let handler =
10909 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10910 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
10911 hello_via_sink(
10912 &handler,
10913 &module_ctx,
10914 &mut module_rx,
10915 hello_frame("aft", PROTOCOL_VERSION, 7),
10916 )
10917 .await;
10918
10919 let expected = vec!["elicitation".to_string(), "roots".to_string()];
10920 let expected_for_request = expected.clone();
10921 let project_root = unique_project_root("consumer-capabilities-present");
10922 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
10923 let route_handler = handler.clone();
10924 let route_task = tokio::spawn(async move {
10925 route_handler
10926 .handle_control_frame(
10927 &client_ctx,
10928 route_open_frame_with_consumer_capabilities(
10929 401,
10930 "aft",
10931 project_root,
10932 Some(expected_for_request),
10933 ),
10934 )
10935 .await
10936 .unwrap()
10937 });
10938 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10939 .await
10940 .unwrap()
10941 .unwrap();
10942 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10943 let ModuleControlRequest::RouteBind {
10944 consumer_capabilities,
10945 ..
10946 } = bind
10947 else {
10948 panic!("expected route.bind request, got {bind:?}");
10949 };
10950 assert_eq!(consumer_capabilities, Some(expected.clone()));
10951
10952 handler
10953 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10954 .await
10955 .unwrap();
10956 let route_response = route_task.await.unwrap();
10957 assert!(route_response.is_empty());
10958 let published = client_rx.recv().await.unwrap();
10959 assert!(matches!(
10960 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10961 ClientControlResponse::RouteOpen { .. }
10962 ));
10963 }
10964
10965 #[tokio::test]
10966 async fn route_open_without_consumer_capabilities_relays_none() {
10967 let registry = Arc::new(Registry::default());
10968 let forwarding = Arc::new(ForwardingTable::default());
10969 let handler =
10970 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10971 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
10972 hello_via_sink(
10973 &handler,
10974 &module_ctx,
10975 &mut module_rx,
10976 hello_frame("aft", PROTOCOL_VERSION, 7),
10977 )
10978 .await;
10979
10980 let project_root = unique_project_root("consumer-capabilities-absent");
10981 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
10982 let route_handler = handler.clone();
10983 let route_task = tokio::spawn(async move {
10984 route_handler
10985 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
10986 .await
10987 .unwrap()
10988 });
10989 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10990 .await
10991 .unwrap()
10992 .unwrap();
10993 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10994 let ModuleControlRequest::RouteBind {
10995 consumer_capabilities,
10996 ..
10997 } = bind
10998 else {
10999 panic!("expected route.bind request, got {bind:?}");
11000 };
11001 assert_eq!(consumer_capabilities, None);
11002
11003 handler
11004 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11005 .await
11006 .unwrap();
11007 let route_response = route_task.await.unwrap();
11008 assert!(route_response.is_empty());
11009 let published = client_rx.recv().await.unwrap();
11010 assert!(matches!(
11011 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11012 ClientControlResponse::RouteOpen { .. }
11013 ));
11014 }
11015
11016 async fn bind_role_versions_for(
11020 sent: Option<BTreeMap<String, String>>,
11021 connection: u64,
11022 ) -> Option<BTreeMap<String, String>> {
11023 let registry = Arc::new(Registry::default());
11024 let forwarding = Arc::new(ForwardingTable::default());
11025 let handler =
11026 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11027 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
11028 hello_via_sink(
11029 &handler,
11030 &module_ctx,
11031 &mut module_rx,
11032 hello_frame("aft", PROTOCOL_VERSION, 7),
11033 )
11034 .await;
11035 let project_root = unique_project_root("role-versions");
11036 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
11037 let route_handler = handler.clone();
11038 let route_task = tokio::spawn(async move {
11039 route_handler
11040 .handle_control_frame(
11041 &client_ctx,
11042 route_open_frame_with_role_versions(403, "aft", project_root, sent),
11043 )
11044 .await
11045 .unwrap()
11046 });
11047 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11048 .await
11049 .expect("a well-formed route.open reaches the module as a bind")
11050 .unwrap();
11051 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11052 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
11053 panic!("expected route.bind request, got {bind:?}");
11054 };
11055 handler
11056 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11057 .await
11058 .unwrap();
11059 assert!(route_task.await.unwrap().is_empty());
11060 let published = client_rx.recv().await.unwrap();
11061 assert!(matches!(
11062 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11063 ClientControlResponse::RouteOpen { .. }
11064 ));
11065 role_versions
11066 }
11067
11068 #[tokio::test]
11069 async fn route_open_relays_role_versions_verbatim() {
11070 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
11071 assert_eq!(
11072 bind_role_versions_for(Some(sent.clone()), 141).await,
11073 Some(sent)
11074 );
11075 }
11076
11077 #[tokio::test]
11080 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
11081 assert_eq!(bind_role_versions_for(None, 143).await, None);
11082 assert_eq!(
11083 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
11084 None
11085 );
11086 }
11087
11088 #[tokio::test]
11089 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
11090 let registry = Arc::new(Registry::default());
11091 let forwarding = Arc::new(ForwardingTable::default());
11092 let handler =
11093 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11094 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
11095 hello_via_sink(
11096 &handler,
11097 &module_ctx,
11098 &mut module_rx,
11099 hello_frame("aft", PROTOCOL_VERSION, 7),
11100 )
11101 .await;
11102
11103 let nine: BTreeMap<String, String> = (0..9)
11104 .map(|index| (format!("role-{index}"), "v1".to_string()))
11105 .collect();
11106 for (label, malformed) in [
11107 (
11108 "invalid role name",
11109 role_versions(&[("Tool_Provider", "v1")]),
11110 ),
11111 ("invalid version", role_versions(&[("tool-provider", "v0")])),
11112 ("nine entries", nine),
11113 ] {
11114 let responses = tokio::time::timeout(
11117 Duration::from_secs(1),
11118 handler.handle_control_frame(
11119 &route_ctx(ConnectionId::new(148)).0,
11120 route_open_frame_with_role_versions(
11121 404,
11122 "aft",
11123 unique_project_root("role-versions-malformed"),
11124 Some(malformed),
11125 ),
11126 ),
11127 )
11128 .await
11129 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
11130 .unwrap();
11131 assert_eq!(responses.len(), 1, "{label}");
11132 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
11133 let error = parse_error(&responses[0]);
11134 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
11135 assert_eq!(
11136 error["detail"]["field"], "role_versions",
11137 "{label}: {error}"
11138 );
11139 assert!(
11140 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
11141 "{label}: a malformed declaration is terminal"
11142 );
11143 assert!(
11144 module_rx.try_recv().is_err(),
11145 "{label}: the module must never see a bind"
11146 );
11147 }
11148 }
11149
11150 #[tokio::test]
11154 async fn route_role_versions_capability_is_advertised() {
11155 let handler = ControlHandler::new(Arc::new(Registry::default()));
11156 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
11157 let ack = hello_via_sink(
11158 &handler,
11159 &ctx,
11160 &mut rx,
11161 hello_frame("m", PROTOCOL_VERSION, 1),
11162 )
11163 .await;
11164 let ack = parse_ack(&ack);
11165 assert!(
11166 ack.subc_capabilities
11167 .iter()
11168 .any(|c| c == "route-role-versions/v1"),
11169 "{:?}",
11170 ack.subc_capabilities
11171 );
11172
11173 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11174 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11175 let reply = handler
11176 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
11177 .await
11178 .unwrap()
11179 .pop()
11180 .unwrap();
11181 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11182 serde_json::from_slice(&reply.body).unwrap()
11183 else {
11184 panic!("not a server.describe reply");
11185 };
11186 assert!(
11187 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
11188 "{capabilities:?}"
11189 );
11190 }
11191
11192 #[tokio::test]
11196 async fn catalog_events_capability_is_advertised() {
11197 let handler = ControlHandler::new(Arc::new(Registry::default()));
11198 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11199 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11200 let reply = handler
11201 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
11202 .await
11203 .unwrap()
11204 .pop()
11205 .unwrap();
11206 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11207 serde_json::from_slice(&reply.body).unwrap()
11208 else {
11209 panic!("not a server.describe reply");
11210 };
11211 assert!(
11212 capabilities
11213 .iter()
11214 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
11215 "{capabilities:?}"
11216 );
11217 }
11218
11219 #[tokio::test]
11220 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
11221 let handler = ControlHandler::new(Arc::new(Registry::default()));
11222 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
11223 let ack = hello_via_sink(
11224 &handler,
11225 &module,
11226 &mut module_rx,
11227 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
11228 )
11229 .await;
11230 let ack = parse_ack(&ack);
11231 assert!(
11232 ack.subc_capabilities
11233 .iter()
11234 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11235 "{:?}",
11236 ack.subc_capabilities
11237 );
11238 assert!(ack
11239 .subc_capabilities
11240 .iter()
11241 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11242
11243 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
11244 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11245 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11246 let reply = handler
11247 .handle_control_frame(&client, frame)
11248 .await
11249 .unwrap()
11250 .pop()
11251 .unwrap();
11252 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11253 serde_json::from_slice(&reply.body).unwrap()
11254 else {
11255 panic!("not a server.describe reply");
11256 };
11257 assert!(
11258 capabilities
11259 .iter()
11260 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11261 "{capabilities:?}"
11262 );
11263 assert!(capabilities
11264 .iter()
11265 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11266 }
11267
11268 #[tokio::test]
11269 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
11270 let handler = ControlHandler::new(Arc::new(Registry::default()));
11271 let (module, mut rx) = route_ctx(ConnectionId::new(154));
11272 let ack = hello_via_sink(
11273 &handler,
11274 &module,
11275 &mut rx,
11276 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
11277 )
11278 .await;
11279 assert!(parse_ack(&ack)
11280 .subc_capabilities
11281 .iter()
11282 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11283 let (client, _rx) = route_ctx(ConnectionId::new(155));
11284 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11285 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11286 let reply = handler
11287 .handle_control_frame(&client, frame)
11288 .await
11289 .unwrap()
11290 .pop()
11291 .unwrap();
11292 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11293 serde_json::from_slice(&reply.body).unwrap()
11294 else {
11295 panic!("not a server.describe reply")
11296 };
11297 assert!(capabilities
11298 .iter()
11299 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11300 }
11301
11302 #[tokio::test]
11303 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
11304 let registry = Arc::new(Registry::default());
11305 let forwarding = Arc::new(ForwardingTable::default());
11306 let handler =
11307 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11308 .with_health_probe_timeout(Duration::from_secs(5));
11309 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
11310 hello_via_sink(
11311 &handler,
11312 &module_ctx,
11313 &mut module_rx,
11314 non_routable_hello_frame_with_control_ops(
11315 "mcp",
11316 300,
11317 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11318 ),
11319 )
11320 .await;
11321 assert!(registry
11322 .get_module("mcp")
11323 .unwrap()
11324 .unwrap()
11325 .manifest
11326 .provides
11327 .is_empty());
11328
11329 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
11330 let route_response = handler
11331 .handle_control_frame(
11332 &route_client_ctx,
11333 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
11334 )
11335 .await
11336 .unwrap();
11337 assert_eq!(route_response[0].header.ty, FrameType::Error);
11338 assert_eq!(
11339 parse_error(&route_response[0])["code"],
11340 "target_unavailable"
11341 );
11342 assert!(parse_error(&route_response[0])["message"]
11343 .as_str()
11344 .unwrap()
11345 .contains("does not provide the requested target"));
11346 assert!(module_rx.try_recv().is_err());
11347
11348 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
11349 let health_handler = handler.clone();
11350 let health_task = tokio::spawn(async move {
11351 health_handler
11352 .handle_control_frame(
11353 &health_client_ctx,
11354 supervisor_health_probe_frame(302, "mcp"),
11355 )
11356 .await
11357 .unwrap()
11358 });
11359 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11360 .await
11361 .unwrap()
11362 .unwrap();
11363 assert_eq!(
11364 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11365 ModuleControlRequest::HealthCheck {}
11366 );
11367 handler
11368 .handle_control_frame(
11369 &module_ctx,
11370 health_response(health_frame.header.corr, HealthStatus::Ok),
11371 )
11372 .await
11373 .unwrap();
11374 let health_response = health_task.await.unwrap();
11375 assert_eq!(health_response[0].header.ty, FrameType::Response);
11376 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11377 ClientControlResponse::SupervisorHealthProbe {
11378 module_id, status, ..
11379 } => {
11380 assert_eq!(module_id, "mcp");
11381 assert_eq!(status, HealthStatus::Ok);
11382 }
11383 other => panic!("unexpected health response: {other:?}"),
11384 }
11385
11386 forwarding
11391 .cleanup_connection(module_ctx.connection_id)
11392 .unwrap();
11393 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
11394 let cleanup_response = tokio::time::timeout(
11395 Duration::from_millis(200),
11396 handler.handle_control_frame(
11397 &cleanup_probe_ctx,
11398 supervisor_health_probe_frame(303, "mcp"),
11399 ),
11400 )
11401 .await
11402 .expect("probe should fail immediately when the control lane is gone")
11403 .unwrap();
11404 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
11405 assert_eq!(
11406 parse_error(&cleanup_response[0])["code"],
11407 "target_unavailable"
11408 );
11409 assert!(parse_error(&cleanup_response[0])["message"]
11410 .as_str()
11411 .unwrap()
11412 .contains("no module connection"));
11413
11414 handler
11415 .cleanup_connection(module_ctx.connection_id)
11416 .unwrap();
11417 }
11418
11419 #[tokio::test]
11420 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
11421 let registry = Arc::new(Registry::default());
11422 let supervisor_handle = SupervisorHandle::new();
11423 let supervisor =
11424 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11425 .with_handle(supervisor_handle.clone())
11426 .with_connection_file_path(
11427 std::env::temp_dir()
11428 .join(format!("subc-route-open-warming-{}", std::process::id())),
11429 );
11430 let module = supervisor
11431 .supervise_configured(
11432 ModuleSpec {
11433 module_id: "warming".to_string(),
11434 program: fake_aft_stub_path(),
11435 args: Vec::new(),
11436 env: Vec::new(),
11437 reserved: false,
11438 reserved_prefixes: Vec::new(),
11439 protocol: ModuleProtocol::Subc,
11440 overlap: Default::default(),
11441 },
11442 true,
11443 )
11444 .unwrap();
11445 assert_eq!(module.state().unwrap(), ModuleState::Running);
11446
11447 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11448 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
11449 let response = handler
11450 .handle_control_frame(
11451 &ctx,
11452 route_open_frame(304, "warming", unique_project_root("warming")),
11453 )
11454 .await
11455 .unwrap();
11456 module.stop().await.unwrap();
11457
11458 assert_eq!(response[0].header.ty, FrameType::Error);
11459 let error = parse_error(&response[0]);
11460 assert_eq!(error["code"], "module_warming");
11461 assert!(error["message"]
11462 .as_str()
11463 .unwrap()
11464 .contains("state=running, enabled=true, live=false"));
11465 }
11466
11467 #[test]
11468 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
11469 let handler = ControlHandler::new(Arc::new(Registry::default()));
11470 let capture = EventCapture::default();
11471 let _subscriber =
11472 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11473 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
11474 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
11475 let pending = (0..limit).collect::<Vec<_>>();
11476 let response = handler
11477 .route_open_capacity_refusal(
11478 &ctx,
11479 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
11480 "busy",
11481 pending.len(),
11482 limit,
11483 )
11484 .unwrap();
11485 assert_eq!(parse_error(&response)["code"], "target_unavailable");
11486 let event = capture
11487 .events()
11488 .into_iter()
11489 .find(|event| {
11490 event.target == "control"
11491 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
11492 })
11493 .expect("connection admission refusal event");
11494 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11495 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11496 }
11497
11498 #[test]
11499 fn route_open_target_cap_logs_admission_reason_and_capacity() {
11500 let handler = ControlHandler::new(Arc::new(Registry::default()));
11501 let capture = EventCapture::default();
11502 let _subscriber =
11503 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11504 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
11505 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
11506 let guards = (0..limit)
11507 .map(|_| {
11508 handler
11509 .route_bind_concurrency
11510 .try_admit("busy", limit)
11511 .unwrap()
11512 })
11513 .collect::<Vec<_>>();
11514 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
11515 Err(in_flight) => in_flight,
11516 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
11517 };
11518 let response = handler
11519 .route_open_target_capacity_refusal(
11520 &ctx,
11521 &route_open_frame(397, "busy", unique_project_root("target-cap")),
11522 "busy",
11523 in_flight,
11524 )
11525 .unwrap();
11526 assert_eq!(parse_error(&response)["code"], "target_unavailable");
11527 let event = capture
11528 .events()
11529 .into_iter()
11530 .find(|event| {
11531 event.target == "control"
11532 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
11533 })
11534 .expect("target admission refusal event");
11535 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11536 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11537 drop(guards);
11538 }
11539
11540 #[tokio::test(flavor = "current_thread")]
11541 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
11542 let handler = ControlHandler::new(Arc::new(Registry::default()));
11543 let capture = EventCapture::default();
11544 let _subscriber =
11545 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11546 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
11547 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11548
11549 let _ = handler
11550 .handle_client_control_request(
11551 &ctx,
11552 frame,
11553 ClientControlRequest::SupervisorSetEnabled {
11554 module_id: "broca".to_string(),
11555 enabled: false,
11556 },
11557 &mut None,
11558 )
11559 .await
11560 .unwrap();
11561
11562 let events = capture
11563 .events()
11564 .into_iter()
11565 .filter(|event| {
11566 event.target == "control"
11567 && event.fields.get("message").map(String::as_str)
11568 == Some("supervisor request received")
11569 })
11570 .collect::<Vec<_>>();
11571 assert_eq!(events.len(), 1, "one supervisor request log line");
11572 let event = &events[0];
11573 assert_eq!(event.level, tracing::Level::INFO);
11574 assert_eq!(
11575 event.fields.get("op"),
11576 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
11577 );
11578 assert_eq!(
11579 event.fields.get("module_id"),
11580 Some(&"\"broca\"".to_string())
11581 );
11582 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
11583 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
11584 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
11585 }
11586
11587 #[tokio::test(flavor = "current_thread")]
11588 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
11589 let registry = Arc::new(Registry::default());
11590 let module_connection = ConnectionId::new(303);
11591 registry
11592 .register_with_control_ops(
11593 manifest("aft", PROTOCOL_VERSION),
11594 PROTOCOL_VERSION,
11595 module_connection,
11596 module_baseline_control_ops(),
11597 )
11598 .unwrap();
11599 let handler = ControlHandler::new(registry);
11600 let capture = EventCapture::default();
11601 let _subscriber =
11602 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11603 let (ctx, _rx) = route_ctx(module_connection);
11604 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11605
11606 let _ = handler
11607 .handle_client_control_request(
11608 &ctx,
11609 frame,
11610 ClientControlRequest::SupervisorSetEnabled {
11611 module_id: "broca".to_string(),
11612 enabled: false,
11613 },
11614 &mut None,
11615 )
11616 .await
11617 .unwrap();
11618
11619 let callers = capture
11620 .events()
11621 .into_iter()
11622 .filter(|event| {
11623 event.fields.get("message").map(String::as_str)
11624 == Some("supervisor request received")
11625 })
11626 .map(|event| event.fields.get("caller").cloned())
11627 .collect::<Vec<_>>();
11628 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
11629 }
11630
11631 #[tokio::test(flavor = "current_thread")]
11632 async fn supervisor_list_does_not_log_request_received() {
11633 let handler = ControlHandler::new(Arc::new(Registry::default()));
11634 let capture = EventCapture::default();
11635 let _subscriber =
11636 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11637 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
11638 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
11639
11640 handler
11641 .handle_client_control_request(
11642 &ctx,
11643 frame,
11644 ClientControlRequest::SupervisorList {},
11645 &mut None,
11646 )
11647 .await
11648 .unwrap();
11649
11650 assert!(capture.events().into_iter().all(|event| {
11651 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11652 }));
11653 }
11654
11655 #[tokio::test(flavor = "current_thread")]
11656 async fn supervisor_rescan_preview_does_not_log_request_received() {
11657 let handler = ControlHandler::new(Arc::new(Registry::default()));
11658 let capture = EventCapture::default();
11659 let _subscriber =
11660 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11661 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
11662 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
11663
11664 handler
11665 .handle_client_control_request(
11666 &ctx,
11667 frame,
11668 ClientControlRequest::SupervisorRescan { preview: true },
11669 &mut None,
11670 )
11671 .await
11672 .unwrap();
11673
11674 assert!(capture.events().into_iter().all(|event| {
11675 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11676 }));
11677 }
11678
11679 struct ApproveOperator;
11680 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
11681 fn prompt(
11682 &self,
11683 _: &str,
11684 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
11685 ) -> crate::operator_confirm::ProviderResult {
11686 crate::operator_confirm::ProviderResult::Approved
11687 }
11688 }
11689
11690 #[tokio::test]
11691 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
11692 assert_eq!(
11693 ControlHandler::default().privileged_operator_prompt,
11694 PrivilegedOperatorPrompt::Log
11695 );
11696 for (initial, changed, running) in [
11697 ("log", "enforce", PrivilegedOperatorPrompt::Log),
11698 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
11699 ] {
11700 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
11701 let path = dir.join("subc.jsonc");
11702 std::fs::write(
11703 &path,
11704 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
11705 )
11706 .unwrap();
11707 let config = crate::daemon_config::load(&path).unwrap().unwrap();
11708 let registry = Arc::new(Registry::default());
11709 let handle = SupervisorHandle::new();
11710 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
11711 .with_handle(handle.clone());
11712 let handler = ControlHandler::new(registry)
11713 .with_supervisor(handle)
11714 .with_privileged_operator_prompt(config.privileged_operator_prompt)
11715 .with_supervisor_rescan(supervisor, &path, config.port);
11716 handler.forwarding.operator_confirms().configure(
11717 Arc::new(ApproveOperator),
11718 crate::operator_confirm::OperatorLimits::default(),
11719 );
11720 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
11721
11722 for (disk_mode, needs_restart) in [
11723 (Some(initial), false),
11724 (Some(changed), true),
11725 (Some(changed), true),
11726 (None, running == PrivilegedOperatorPrompt::Enforce),
11727 (Some(initial), false),
11728 ] {
11729 let mut doc = json!({"version":1});
11730 if let Some(mode) = disk_mode {
11731 doc["privileged_operator_prompt"] = json!(mode);
11732 }
11733 std::fs::write(&path, doc.to_string()).unwrap();
11734 for preview in [true, false] {
11735 let request = ClientControlRequest::SupervisorRescan { preview };
11736 let frame = Frame::build(
11737 FrameType::Request,
11738 control_flags(),
11739 0,
11740 0,
11741 1,
11742 serde_json::to_vec(&request).unwrap(),
11743 )
11744 .unwrap();
11745 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
11746 assert_eq!(frames.len(), 1);
11747 let ClientControlResponse::SupervisorRescan { result } =
11748 serde_json::from_slice(&frames[0].body).unwrap()
11749 else {
11750 panic!("expected a rescan response");
11751 };
11752 let expected = if needs_restart {
11753 vec!["privileged_operator_prompt"]
11754 } else {
11755 Vec::new()
11756 };
11757 assert_eq!(result.restart_required, expected);
11758 assert_eq!(handler.privileged_operator_prompt, running);
11759 assert_eq!(
11760 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
11761 running
11762 );
11763 }
11764 }
11765 }
11766 }
11767
11768 #[tokio::test]
11769 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
11770 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
11771 let path = dir.join("subc.jsonc");
11772 let registry = Arc::new(Registry::default());
11773 let handle = SupervisorHandle::new();
11774 let supervisor =
11775 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
11776 let handler = ControlHandler::new(registry)
11777 .with_supervisor(handle)
11778 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
11779 .with_supervisor_rescan(supervisor, &path, None);
11780 handler.forwarding.operator_confirms().configure(
11781 Arc::new(ApproveOperator),
11782 crate::operator_confirm::OperatorLimits::default(),
11783 );
11784 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
11785 for preview in [true, false] {
11786 std::fs::write(
11787 &path,
11788 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
11789 )
11790 .unwrap();
11791 let request = ClientControlRequest::SupervisorRescan { preview };
11792 let frame = Frame::build(
11793 FrameType::Request,
11794 control_flags(),
11795 0,
11796 0,
11797 1,
11798 serde_json::to_vec(&request).unwrap(),
11799 )
11800 .unwrap();
11801 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
11802 assert_eq!(frames.len(), 1);
11803 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
11804 assert_eq!(
11805 handler.privileged_operator_prompt,
11806 PrivilegedOperatorPrompt::Enforce
11807 );
11808 }
11809 }
11810
11811 #[tokio::test]
11816 async fn route_open_refusal_names_the_check_that_refused() {
11817 let handler = ControlHandler::new(Arc::new(Registry::default()));
11818 let capture = EventCapture::default();
11819 let _subscriber =
11820 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11821 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
11822 let response = handler
11823 .handle_control_frame(
11824 &ctx,
11825 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
11826 )
11827 .await
11828 .unwrap();
11829
11830 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11831 let event = capture
11832 .events()
11833 .into_iter()
11834 .find(|event| {
11835 event.target == "control"
11836 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
11837 })
11838 .expect("route.open refusal event");
11839 assert_eq!(
11840 event.fields.get("reason"),
11841 Some(&"\"not_registered\"".to_string())
11842 );
11843 }
11844
11845 #[tokio::test]
11846 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
11847 let registry = Arc::new(Registry::default());
11848 let supervisor_handle = SupervisorHandle::new();
11849 let supervisor =
11850 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11851 .with_handle(supervisor_handle.clone())
11852 .with_connection_file_path(std::env::temp_dir().join(format!(
11853 "subc-route-open-refusal-info-{}",
11854 std::process::id()
11855 )));
11856 let module = supervisor
11857 .supervise_configured(
11858 ModuleSpec {
11859 module_id: "warming".to_string(),
11860 program: fake_aft_stub_path(),
11861 args: Vec::new(),
11862 env: Vec::new(),
11863 reserved: false,
11864 reserved_prefixes: Vec::new(),
11865 protocol: ModuleProtocol::Subc,
11866 overlap: Default::default(),
11867 },
11868 true,
11869 )
11870 .unwrap();
11871 assert_eq!(module.state().unwrap(), ModuleState::Running);
11872
11873 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11874 assert!(handler
11875 .counters()
11876 .snapshot()
11877 .get("route_open_refused_by_code")
11878 .is_none());
11879 let capture = EventCapture::default();
11880 let _subscriber =
11881 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11882 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
11883 let response = handler
11884 .handle_control_frame(
11885 &ctx,
11886 route_open_frame(394, "warming", unique_project_root("refusal-info")),
11887 )
11888 .await
11889 .unwrap();
11890 module.stop().await.unwrap();
11891
11892 assert_eq!(parse_error(&response[0])["code"], "module_warming");
11893 let event = capture
11894 .events()
11895 .into_iter()
11896 .find(|event| {
11897 event.target == "control"
11898 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
11899 })
11900 .expect("route.open refusal event");
11901 assert_eq!(
11902 event.fields.get("module_id"),
11903 Some(&"\"warming\"".to_string())
11904 );
11905 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
11906 assert_eq!(
11907 event.fields.get("reason"),
11908 Some(&"\"supervised_not_registered\"".to_string())
11909 );
11910 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
11911 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
11912 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
11913 assert_eq!(
11914 handler.counters().snapshot()["route_open_refused_by_code"],
11915 json!({ "module_warming": 1 })
11916 );
11917 }
11918
11919 const OUTAGE_START: &str = "route.open refusing module: not serving";
11920 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
11921
11922 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
11923 capture
11924 .events()
11925 .into_iter()
11926 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
11927 .collect()
11928 }
11929
11930 fn supervise_stub(
11931 registry: &Arc<Registry>,
11932 module_id: &str,
11933 enabled: bool,
11934 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
11935 let supervisor_handle = SupervisorHandle::new();
11936 let supervisor =
11937 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
11938 .with_handle(supervisor_handle.clone())
11939 .with_connection_file_path(std::env::temp_dir().join(format!(
11940 "subc-route-outage-{module_id}-{}",
11941 std::process::id()
11942 )));
11943 let module = supervisor
11944 .supervise_configured(
11945 ModuleSpec {
11946 module_id: module_id.to_string(),
11947 program: fake_aft_stub_path(),
11948 args: Vec::new(),
11949 env: Vec::new(),
11950 reserved: false,
11951 reserved_prefixes: Vec::new(),
11952 protocol: ModuleProtocol::Subc,
11953 overlap: Default::default(),
11954 },
11955 enabled,
11956 )
11957 .unwrap();
11958 (supervisor_handle, module)
11959 }
11960
11961 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
11962 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
11963 module_id: module_id.to_string(),
11964 drain_timeout_ms: Some(50),
11965 })
11966 .unwrap();
11967 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11968 }
11969
11970 #[test]
11974 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
11975 let registry = Arc::new(Registry::default());
11976 let forwarding = Arc::new(ForwardingTable::default());
11977 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11978 let second = ControlHandler::with_forwarding(registry, forwarding);
11979 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
11980 }
11981
11982 #[tokio::test(flavor = "current_thread")]
11986 async fn route_open_unknown_module_refusals_add_no_outage_state() {
11987 let handler = ControlHandler::new(Arc::new(Registry::default()));
11988 let capture = EventCapture::default();
11989 let _subscriber =
11990 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11991 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
11992 for corr in 0..8 {
11993 let response = handler
11994 .handle_control_frame(
11995 &ctx,
11996 route_open_frame(
11997 380 + corr,
11998 &format!("nobody-{corr}"),
11999 unique_project_root("outage-unknown"),
12000 ),
12001 )
12002 .await
12003 .unwrap();
12004 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12005 }
12006
12007 assert_eq!(handler.route_outages.tracked_module_count(), 0);
12008 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
12009 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
12010 }
12011
12012 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12017 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
12018 let registry = Arc::new(Registry::default());
12019 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
12020 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12021 let capture = EventCapture::default();
12022 let _subscriber =
12023 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12024 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
12025 handler.route_outages.record_accepted("outage-restart");
12028
12029 let response = handler
12030 .handle_control_frame(
12031 &ctx,
12032 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
12033 )
12034 .await
12035 .unwrap();
12036 assert_eq!(response[0].header.ty, FrameType::Error);
12037 let starts = outage_lines(&capture, OUTAGE_START);
12038 assert_eq!(starts.len(), 1, "{starts:?}");
12039 assert_eq!(starts[0].level, tracing::Level::WARN);
12040 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
12041 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
12042 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
12043 handler.route_outages.record_accepted("outage-restart");
12044 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
12045
12046 let restart = handler
12047 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
12048 .await
12049 .unwrap();
12050 assert_eq!(
12051 restart[0].header.ty,
12052 FrameType::Response,
12053 "{:?}",
12054 parse_error(&restart[0])
12055 );
12056 handler
12057 .handle_control_frame(
12058 &ctx,
12059 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
12060 )
12061 .await
12062 .unwrap();
12063 module.stop().await.unwrap();
12064
12065 let starts = outage_lines(&capture, OUTAGE_START);
12066 assert_eq!(starts.len(), 2, "{starts:?}");
12067 assert_eq!(starts[1].level, tracing::Level::INFO);
12068 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
12069 }
12070
12071 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12075 async fn failed_operator_restart_leaves_no_operator_mark() {
12076 let registry = Arc::new(Registry::default());
12077 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
12078 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12079 let capture = EventCapture::default();
12080 let _subscriber =
12081 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12082 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
12083 handler.route_outages.record_accepted("outage-disabled");
12084
12085 let restart = handler
12086 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
12087 .await
12088 .unwrap();
12089 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
12090 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
12091
12092 handler
12093 .handle_control_frame(
12094 &ctx,
12095 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
12096 )
12097 .await
12098 .unwrap();
12099 let starts = outage_lines(&capture, OUTAGE_START);
12100 assert_eq!(starts.len(), 1, "{starts:?}");
12101 assert_eq!(starts[0].level, tracing::Level::WARN);
12102 }
12103
12104 #[tokio::test(flavor = "current_thread")]
12105 async fn route_open_unknown_module_escapes_target_module_id() {
12106 let handler = ControlHandler::new(Arc::new(Registry::default()));
12107 let capture = EventCapture::default();
12108 let _subscriber =
12109 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12110 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
12111 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
12112 let response = handler
12113 .handle_control_frame(
12114 &ctx,
12115 route_open_frame(
12116 395,
12117 hostile_module_id,
12118 unique_project_root("hostile-target-module-id"),
12119 ),
12120 )
12121 .await
12122 .unwrap();
12123
12124 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12125 let event = capture
12126 .events()
12127 .into_iter()
12128 .find(|event| {
12129 event.target == "control"
12130 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12131 })
12132 .expect("route.open unknown-module refusal event");
12133 let logged = event.fields.get("module_id").expect("module_id field");
12134 assert!(!logged.bytes().any(|byte| byte < 0x20));
12135 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12136 }
12137
12138 #[tokio::test(flavor = "current_thread")]
12139 async fn route_open_module_rejection_uses_daemon_counter_key() {
12140 let registry = Arc::new(Registry::default());
12141 let forwarding = Arc::new(ForwardingTable::default());
12142 let handler =
12143 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12144 let module_connection = ConnectionId::new(95);
12145 let (module_ctx, mut module_rx) = route_ctx(module_connection);
12146 hello_via_sink(
12147 &handler,
12148 &module_ctx,
12149 &mut module_rx,
12150 hello_frame("aft", PROTOCOL_VERSION, 395),
12151 )
12152 .await;
12153
12154 let client_connection = ConnectionId::new(96);
12155 let (client_ctx, _client_rx) = route_ctx(client_connection);
12156 let capture = EventCapture::default();
12157 let _subscriber =
12158 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12159 let (route_task, bind) = relay_route_open(
12160 &handler,
12161 client_connection,
12162 &client_ctx.egress,
12163 &mut module_rx,
12164 396,
12165 "aft",
12166 "hostile-module-code",
12167 )
12168 .await;
12169 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
12170 let rejection = Frame::build(
12171 FrameType::Error,
12172 control_flags(),
12173 0,
12174 0,
12175 bind.header.corr,
12176 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
12177 )
12178 .unwrap();
12179 handler
12180 .handle_control_frame(&module_ctx, rejection)
12181 .await
12182 .unwrap();
12183
12184 let response = route_task.await.unwrap();
12185 assert_eq!(parse_error(&response[0])["code"], hostile_code);
12186 let counters = handler.counters().snapshot();
12187 assert_eq!(
12188 counters["route_open_refused_by_code"],
12189 json!({ "module_rejected": 1 })
12190 );
12191 assert!(counters["route_open_refused_by_code"]
12192 .get(hostile_code)
12193 .is_none());
12194
12195 let event = capture
12196 .events()
12197 .into_iter()
12198 .find(|event| {
12199 event.target == "control"
12200 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
12201 })
12202 .expect("route.open module-rejection refusal event");
12203 let logged = event.fields.get("module_code").expect("module_code field");
12204 assert!(!logged.bytes().any(|byte| byte < 0x20));
12205 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12206 }
12207
12208 #[tokio::test]
12209 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
12210 let registry = Arc::new(Registry::default());
12211 let supervisor_handle = SupervisorHandle::new();
12212 let missing_program = std::env::temp_dir().join(format!(
12213 "subc-route-open-missing-program-{}",
12214 std::process::id()
12215 ));
12216 let supervisor =
12217 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12218 .with_handle(supervisor_handle.clone());
12219 let module = supervisor
12220 .supervise_configured(
12221 ModuleSpec {
12222 module_id: "failed".to_string(),
12223 program: missing_program,
12224 args: Vec::new(),
12225 env: Vec::new(),
12226 reserved: false,
12227 reserved_prefixes: Vec::new(),
12228 protocol: ModuleProtocol::Subc,
12229 overlap: Default::default(),
12230 },
12231 true,
12232 )
12233 .unwrap();
12234 assert_eq!(module.state().unwrap(), ModuleState::Failed);
12235
12236 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12237 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
12238 let response = handler
12239 .handle_control_frame(
12240 &ctx,
12241 route_open_frame(305, "failed", unique_project_root("failed")),
12242 )
12243 .await
12244 .unwrap();
12245
12246 assert_eq!(response[0].header.ty, FrameType::Error);
12247 let error = parse_error(&response[0]);
12248 assert_eq!(error["code"], "target_unavailable");
12249 assert!(error["message"]
12250 .as_str()
12251 .unwrap()
12252 .contains("state=failed, enabled=true, live=false"));
12253 }
12254
12255 #[tokio::test]
12256 async fn route_open_role_mismatch_remains_target_unavailable() {
12257 let registry = Arc::new(Registry::default());
12258 let handler = ControlHandler::new(Arc::clone(®istry));
12259 handler
12260 .handle_control(
12261 ConnectionId::new(41),
12262 non_routable_hello_frame_with_control_ops("health-only", 306, None),
12263 )
12264 .unwrap();
12265
12266 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
12267 let response = handler
12268 .handle_control_frame(
12269 &ctx,
12270 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
12271 )
12272 .await
12273 .unwrap();
12274
12275 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12276 assert!(parse_error(&response[0])["message"]
12277 .as_str()
12278 .unwrap()
12279 .contains("does not provide the requested target"));
12280 }
12281
12282 #[tokio::test]
12283 async fn route_open_inactive_registration_remains_target_unavailable() {
12284 let registry = Arc::new(Registry::default());
12285 let handler = ControlHandler::new(Arc::clone(®istry));
12286 handler
12287 .handle_control(
12288 ConnectionId::new(43),
12289 hello_frame("inactive", PROTOCOL_VERSION, 308),
12290 )
12291 .unwrap();
12292 assert!(registry
12293 .set_module_state_for_test("inactive", ChannelState::Closed)
12294 .unwrap());
12295
12296 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
12297 let response = handler
12298 .handle_control_frame(
12299 &ctx,
12300 route_open_frame(309, "inactive", unique_project_root("inactive")),
12301 )
12302 .await
12303 .unwrap();
12304
12305 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12306 assert!(parse_error(&response[0])["message"]
12307 .as_str()
12308 .unwrap()
12309 .contains("is not active"));
12310 }
12311
12312 #[tokio::test]
12313 async fn late_health_reply_is_recorded_through_the_module_response_path() {
12314 let registry = Arc::new(Registry::default());
12315 let forwarding = Arc::new(ForwardingTable::default());
12316 let supervisor_handle = SupervisorHandle::new();
12317 let supervisor =
12318 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
12319 .with_forwarding(Arc::clone(&forwarding))
12320 .with_handle(supervisor_handle.clone());
12321 let module = supervisor
12322 .supervise_configured(
12323 crate::ModuleSpec {
12324 module_id: "late-health-response".to_string(),
12325 program: PathBuf::from("disabled-module"),
12326 args: Vec::new(),
12327 env: Vec::new(),
12328 reserved: false,
12329 reserved_prefixes: Vec::new(),
12330 protocol: ModuleProtocol::Subc,
12331 overlap: Default::default(),
12332 },
12333 false,
12334 )
12335 .unwrap();
12336 let handler =
12337 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12338 .with_supervisor(supervisor_handle);
12339 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
12340 handler
12341 .handle_control_frame(
12342 &module_ctx,
12343 hello_frame_with_control_ops(
12344 "late-health-response",
12345 PROTOCOL_VERSION,
12346 7,
12347 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12348 ),
12349 )
12350 .await
12351 .unwrap();
12352 let probe_started_at = Instant::now() - Duration::from_millis(80);
12353 let pending = forwarding
12354 .begin_health_probe_rpc_for(
12355 "late-health-response",
12356 MODULE_CONTROL_OP_HEALTH_CHECK,
12357 probe_started_at,
12358 Instant::now() - Duration::from_millis(1),
12359 )
12360 .unwrap();
12361 assert!(forwarding
12362 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
12363 .unwrap());
12364
12365 let responses = handler
12366 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
12367 .await
12368 .unwrap();
12369
12370 assert!(responses.is_empty());
12371 let health = module.status().unwrap().health;
12372 assert_eq!(health.late_answer_count, 1);
12373 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
12374 }
12375
12376 #[tokio::test]
12377 async fn health_probe_timeout_and_module_death_are_typed() {
12378 let registry = Arc::new(Registry::default());
12379 let forwarding = Arc::new(ForwardingTable::default());
12380 let handler =
12381 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12382 .with_health_probe_timeout(Duration::from_millis(50));
12383 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
12384 hello_via_sink(
12385 &handler,
12386 &module_ctx,
12387 &mut module_rx,
12388 hello_frame_with_control_ops(
12389 "aft",
12390 PROTOCOL_VERSION,
12391 7,
12392 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12393 ),
12394 )
12395 .await;
12396
12397 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12398 let responses = handler
12399 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
12400 .await
12401 .unwrap();
12402 assert_eq!(responses[0].header.ty, FrameType::Error);
12403 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
12404 let _ = module_rx.try_recv();
12405
12406 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
12407 let health_handler = handler.clone();
12408 let death_task = tokio::spawn(async move {
12409 health_handler
12410 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
12411 .await
12412 .unwrap()
12413 });
12414 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12415 .await
12416 .unwrap()
12417 .unwrap();
12418 handler
12419 .cleanup_connection(module_ctx.connection_id)
12420 .unwrap();
12421 let responses = death_task.await.unwrap();
12422 assert_eq!(responses[0].header.ty, FrameType::Error);
12423 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
12424 }
12425
12426 #[test]
12427 fn hello_requires_exact_protocol_version() {
12428 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
12429 let registry = Arc::new(Registry::default());
12430 let handler = ControlHandler::new(Arc::clone(®istry));
12431 let responses = handler
12432 .handle_control(
12433 ConnectionId::new(connection),
12434 hello_frame("aft", offered, 9),
12435 )
12436 .unwrap();
12437
12438 assert_eq!(responses.len(), 1);
12439 assert_eq!(responses[0].header.ty, FrameType::Error);
12440 let error = parse_error(&responses[0]);
12441 assert_eq!(error["code"], "version_unsupported");
12442 assert!(registry.get_module("aft").unwrap().is_none());
12443 assert_eq!(registry.active_registration_count().unwrap(), 0);
12444 }
12445 }
12446
12447 #[test]
12448 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
12449 let registry = Arc::new(Registry::default());
12450 let forwarding = Arc::new(ForwardingTable::default());
12451 let handler =
12452 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12453 let module_connection = ConnectionId::new(301);
12454 let registration = registry
12455 .register_with_control_ops(
12456 manifest("aft-push", PROTOCOL_VERSION),
12457 PROTOCOL_VERSION,
12458 module_connection,
12459 module_baseline_control_ops(),
12460 )
12461 .unwrap();
12462 let (module_tx, _module_rx) = mpsc::channel(8);
12463 let endpoint = forwarding
12464 .register_module_connection(
12465 module_connection,
12466 "aft-push".to_string(),
12467 PROTOCOL_VERSION,
12468 manifest_concurrency(®istration.manifest),
12469 FrameSink::new(module_tx),
12470 )
12471 .unwrap();
12472
12473 let unknown = Frame::build(
12475 FrameType::Push,
12476 control_flags(),
12477 0,
12478 0,
12479 5,
12480 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
12481 )
12482 .unwrap();
12483 let out = handler.handle_status_update(endpoint, unknown).unwrap();
12484 assert!(
12485 out.is_empty(),
12486 "unknown push op must be ignored, got {out:?}"
12487 );
12488
12489 let malformed = Frame::build(
12491 FrameType::Push,
12492 control_flags(),
12493 0,
12494 0,
12495 6,
12496 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
12497 )
12498 .unwrap();
12499 let out = handler.handle_status_update(endpoint, malformed).unwrap();
12500 assert_eq!(out.len(), 1);
12501 assert_eq!(out[0].header.ty, FrameType::Error);
12502 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
12503 }
12504
12505 #[test]
12506 fn hello_rejected_when_connection_already_owns_client_routes() {
12507 let registry = Arc::new(Registry::default());
12508 let forwarding = Arc::new(ForwardingTable::default());
12509 let handler =
12510 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12511 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
12513 let client_connection = ConnectionId::new(202);
12514
12515 let responses = handler
12518 .handle_control(
12519 client_connection,
12520 hello_frame("aft-second", PROTOCOL_VERSION, 9),
12521 )
12522 .unwrap();
12523 assert_eq!(responses[0].header.ty, FrameType::Error);
12524 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
12525 assert!(registry.get_module("aft-second").unwrap().is_none());
12526 }
12527
12528 #[tokio::test]
12529 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
12530 let registry = Arc::new(Registry::default());
12531 let forwarding = Arc::new(ForwardingTable::default());
12532 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
12533 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
12534 hello_via_sink(
12535 &handler,
12536 &module_ctx,
12537 &mut module_rx,
12538 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
12539 )
12540 .await;
12541 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
12542 let pending = forwarding
12543 .begin_route_bind_relay_for_test(
12544 client_ctx.connection_id,
12545 client_ctx.egress.clone(),
12546 2,
12547 "alpha",
12548 )
12549 .unwrap();
12550 forwarding
12551 .complete_pending_relay(
12552 module_ctx.connection_id,
12553 pending.corr,
12554 RouteBindRelayOutcome::Accepted,
12555 )
12556 .unwrap();
12557 client_rx.try_recv().unwrap();
12558 for module_id in ["beta", "alpha"] {
12559 let replies = handler
12560 .handle_control_frame(
12561 &module_ctx,
12562 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
12563 )
12564 .await
12565 .unwrap();
12566 assert_eq!(replies.len(), 1, "second HELLO must be refused");
12567 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
12568 }
12569 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
12570 assert!(registry.get_module("beta").unwrap().is_none());
12571 assert!(matches!(
12572 forwarding
12573 .lookup_data_route(
12574 client_ctx.connection_id,
12575 pending.client_channel,
12576 pending.client_epoch,
12577 )
12578 .unwrap(),
12579 DataRoute::Client(DataRouteState::Bound(_))
12580 ));
12581 assert!(handler
12582 .hello_launch_nonces
12583 .lock()
12584 .unwrap()
12585 .presented(module_ctx.connection_id, Some("alpha-nonce")));
12586 assert!(module_rx.try_recv().is_err());
12587 }
12588
12589 #[test]
12590 fn reserved_module_hello_requires_matching_launch_nonce() {
12591 let registry = Arc::new(Registry::default());
12592 let supervisor = SupervisorHandle::new();
12593 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
12596 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12597
12598 let no_nonce = handler
12600 .handle_control(
12601 ConnectionId::new(1),
12602 hello_frame("vault", PROTOCOL_VERSION, 1),
12603 )
12604 .unwrap();
12605 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
12606 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
12607 assert!(registry.get_module("vault").unwrap().is_none());
12608
12609 let wrong = handler
12611 .handle_control(
12612 ConnectionId::new(2),
12613 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
12614 )
12615 .unwrap();
12616 assert_eq!(wrong[0].header.ty, FrameType::Error);
12617 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
12618 assert!(registry.get_module("vault").unwrap().is_none());
12619
12620 let ok = handler
12622 .handle_control(
12623 ConnectionId::new(3),
12624 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
12625 )
12626 .unwrap();
12627 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
12628 assert!(registry.get_module("vault").unwrap().is_some());
12629 }
12630
12631 #[test]
12632 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
12633 let registry = Arc::new(Registry::default());
12634 let supervisor = SupervisorHandle::new();
12635 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
12636 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
12637 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12638
12639 let squat = handler
12640 .handle_control(
12641 ConnectionId::new(1),
12642 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
12643 )
12644 .unwrap();
12645 assert_eq!(squat[0].header.ty, FrameType::Error);
12646 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
12647 assert!(parse_error(&squat[0])["message"]
12648 .as_str()
12649 .unwrap()
12650 .contains("fed:"));
12651
12652 let accepted_peer = handler
12653 .handle_control(
12654 ConnectionId::new(2),
12655 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
12656 )
12657 .unwrap();
12658 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
12659
12660 let accepted_short = handler
12661 .handle_control(
12662 ConnectionId::new(3),
12663 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
12664 )
12665 .unwrap();
12666 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
12667
12668 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
12669 let response = handler
12670 .handle_control(
12671 ConnectionId::new(conn),
12672 hello_frame(module_id, PROTOCOL_VERSION, conn),
12673 )
12674 .unwrap();
12675 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
12676 }
12677 }
12678
12679 #[test]
12680 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
12681 let registry = Arc::new(Registry::default());
12682 let supervisor = SupervisorHandle::new();
12683 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
12684 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
12685 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
12686 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12687
12688 let owner_nonce = handler
12689 .handle_control(
12690 ConnectionId::new(1),
12691 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
12692 )
12693 .unwrap();
12694 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
12695 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
12696 assert!(registry.get_module("fed:special").unwrap().is_none());
12697
12698 let exact_nonce = handler
12699 .handle_control(
12700 ConnectionId::new(2),
12701 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
12702 )
12703 .unwrap();
12704 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
12705 assert!(registry.get_module("fed:special").unwrap().is_some());
12706 }
12707
12708 #[test]
12709 fn non_reserved_module_ignores_launch_nonce() {
12710 let registry = Arc::new(Registry::default());
12711 let handler = ControlHandler::new(Arc::clone(®istry));
12714 let no_nonce = handler
12715 .handle_control(
12716 ConnectionId::new(1),
12717 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
12718 )
12719 .unwrap();
12720 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
12721 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
12722
12723 let echoed_nonce = handler
12724 .handle_control(
12725 ConnectionId::new(2),
12726 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
12727 )
12728 .unwrap();
12729 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
12730 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
12731 }
12732
12733 #[test]
12734 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
12735 let handler = ControlHandler::default();
12736 let conn = ConnectionId::new(1);
12737 let malformed = Frame::build(
12738 FrameType::Hello,
12739 control_flags(),
12740 0,
12741 0,
12742 3,
12743 b"{not json".to_vec(),
12744 )
12745 .unwrap();
12746
12747 let error = handler.handle_control(conn, malformed).unwrap();
12748 assert_eq!(error[0].header.ty, FrameType::Error);
12749 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
12750
12751 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
12752 let pong = handler.handle_control(conn, ping).unwrap();
12753 assert_eq!(pong[0].header.ty, FrameType::Pong);
12754 assert_eq!(pong[0].header.corr, 4);
12755 }
12756
12757 #[test]
12758 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
12759 let registry = Arc::new(Registry::default());
12760 let handler = ControlHandler::new(Arc::clone(®istry));
12761
12762 handler
12763 .handle_control(
12764 ConnectionId::new(1),
12765 hello_frame("aft", PROTOCOL_VERSION, 1),
12766 )
12767 .unwrap();
12768 let duplicate = handler
12769 .handle_control(
12770 ConnectionId::new(2),
12771 hello_frame("aft", PROTOCOL_VERSION, 2),
12772 )
12773 .unwrap();
12774
12775 assert_eq!(duplicate[0].header.ty, FrameType::Error);
12776 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
12777 let registration = registry.get_module("aft").unwrap().unwrap();
12778 assert_eq!(registration.connection_id, ConnectionId::new(1));
12779 }
12780
12781 #[test]
12782 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
12783 let registry = Arc::new(Registry::default());
12784 let forwarding = Arc::new(ForwardingTable::default());
12785 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
12786 let handler =
12787 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12788 .with_process_liveness(process_liveness);
12789 let (ctx, route_channel, route_epoch) =
12790 bind_liveness_route(®istry, &forwarding, "aft-dead");
12791 let responses = handler
12792 .handle_route_poll(
12793 &ctx,
12794 route_poll_frame(41, PollKind::Liveness, route_channel),
12795 route_channel,
12796 route_epoch,
12797 PollKind::Liveness,
12798 )
12799 .unwrap();
12800
12801 assert_eq!(responses.len(), 1);
12802 assert_eq!(responses[0].header.ty, FrameType::Response);
12803 assert_route_poll_liveness(&responses[0], false);
12804 }
12805
12806 #[test]
12807 fn liveness_poll_without_process_source_uses_bound_route() {
12808 let registry = Arc::new(Registry::default());
12809 let forwarding = Arc::new(ForwardingTable::default());
12810 let handler =
12811 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12812 let (ctx, route_channel, route_epoch) =
12813 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
12814 let responses = handler
12815 .handle_route_poll(
12816 &ctx,
12817 route_poll_frame(42, PollKind::Liveness, route_channel),
12818 route_channel,
12819 route_epoch,
12820 PollKind::Liveness,
12821 )
12822 .unwrap();
12823
12824 assert_route_poll_liveness(&responses[0], true);
12825 }
12826
12827 #[test]
12828 fn liveness_poll_untracked_process_source_uses_bound_route() {
12829 let registry = Arc::new(Registry::default());
12830 let forwarding = Arc::new(ForwardingTable::default());
12831 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
12832 let handler =
12833 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12834 .with_process_liveness(process_liveness);
12835 let (ctx, route_channel, route_epoch) =
12836 bind_liveness_route(®istry, &forwarding, "aft-untracked");
12837 let responses = handler
12838 .handle_route_poll(
12839 &ctx,
12840 route_poll_frame(43, PollKind::Liveness, route_channel),
12841 route_channel,
12842 route_epoch,
12843 PollKind::Liveness,
12844 )
12845 .unwrap();
12846
12847 assert_route_poll_liveness(&responses[0], true);
12848 }
12849
12850 #[tokio::test]
12851 async fn unknown_op_returns_unknown_control_op() {
12852 let handler = ControlHandler::default();
12853 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
12854 let request = Frame::build(
12855 FrameType::Request,
12856 control_flags(),
12857 0,
12858 0,
12859 55,
12860 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
12861 )
12862 .unwrap();
12863
12864 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12865
12866 assert_eq!(response.len(), 1);
12867 assert_eq!(response[0].header.ty, FrameType::Error);
12868 assert_eq!(response[0].header.corr, 55);
12869 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
12870 }
12871
12872 #[tokio::test]
12873 async fn supervisor_provenance_rejects_unknown_exact_module() {
12874 let handler = ControlHandler::default();
12875 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
12876 let request = Frame::build(
12877 FrameType::Request,
12878 control_flags(),
12879 0,
12880 0,
12881 57,
12882 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
12883 )
12884 .unwrap();
12885
12886 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12887
12888 assert_eq!(response.len(), 1);
12889 assert_eq!(response[0].header.ty, FrameType::Error);
12890 assert_eq!(response[0].header.corr, 57);
12891 let error = parse_error(&response[0]);
12892 assert_eq!(error["code"], "unknown_module");
12893 assert_eq!(error["message"], "module_id 'missing' is not supervised");
12894 }
12895
12896 #[test]
12897 fn provenance_probe_override_keeps_handler_tests_deterministic() {
12898 let expected = subc_control::RunningImageAgreement::Unavailable {
12899 reason: subc_control::RunningImageUnavailableReason::HashFailed,
12900 };
12901 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
12902 assert_eq!(handler.provenance_probe_override, Some(expected));
12903 }
12904
12905 #[test]
12906 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
12907 let verdict = reload_verdict(
12908 std::path::Path::new("/bin/new"),
12909 Some(std::path::Path::new("/bin/old")),
12910 subc_control::RunningImageAgreement::Unavailable {
12911 reason: subc_control::RunningImageUnavailableReason::HashFailed,
12912 },
12913 );
12914 assert!(matches!(
12915 verdict.path,
12916 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
12917 if configured == std::path::Path::new("/bin/new")
12918 && spawned_from == std::path::Path::new("/bin/old")
12919 ));
12920 }
12921
12922 #[cfg(windows)]
12923 #[test]
12924 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
12925 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
12926 let configured = dir.join("module.exe");
12927 std::fs::write(&configured, b"executable path fixture").unwrap();
12928 let spawned = std::fs::canonicalize(&configured).unwrap();
12929 assert_ne!(
12930 configured, spawned,
12931 "fixture must exercise different path spellings"
12932 );
12933 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
12934 let lower_case = configured.to_string_lossy().to_lowercase();
12935 for spelling in [
12936 &configured,
12937 Path::new(&forward_slashes),
12938 Path::new(&lower_case),
12939 ] {
12940 let verdict = reload_verdict(
12941 spelling,
12942 Some(&spawned),
12943 subc_control::RunningImageAgreement::Unavailable {
12944 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12945 },
12946 );
12947 assert_eq!(
12948 verdict.path,
12949 ReloadPathAgreement::Match,
12950 "configured {spelling:?}, spawned {spawned:?}"
12951 );
12952 }
12953 }
12954
12955 #[cfg(windows)]
12956 #[test]
12957 fn windows_reload_path_keeps_different_existing_program_pending() {
12958 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
12959 let old = dir.join("old.exe");
12960 let new = dir.join("new.exe");
12961 std::fs::write(&old, b"same executable content").unwrap();
12962 std::fs::copy(&old, &new).unwrap();
12963 let spawned = std::fs::canonicalize(&old).unwrap();
12964 let image = subc_control::RunningImageAgreement::Unavailable {
12965 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12966 };
12967 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
12968 assert!(
12969 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
12970 if configured == new && spawned_from == spawned)
12971 );
12972 let missing = dir.join("missing.exe");
12973 assert!(matches!(
12974 reload_verdict(&missing, Some(&spawned), image).path,
12975 ReloadPathAgreement::Mismatch { .. }
12976 ));
12977 }
12978
12979 #[test]
12980 fn reload_verdict_detects_replaced_image_at_same_path() {
12981 let image = subc_control::RunningImageAgreement::Mismatch {
12982 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
12983 digest: "old".into(),
12984 },
12985 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
12986 digest: "new".into(),
12987 },
12988 };
12989 let verdict = reload_verdict(
12990 std::path::Path::new("/bin/same"),
12991 Some(std::path::Path::new("/bin/same")),
12992 image.clone(),
12993 );
12994 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
12995 assert_eq!(verdict.image, image);
12996 }
12997
12998 #[test]
12999 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
13000 let image = subc_control::RunningImageAgreement::Unavailable {
13001 reason: subc_control::RunningImageUnavailableReason::NotRunning,
13002 };
13003 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
13004 assert_eq!(
13005 verdict.path,
13006 subc_control::ReloadPathAgreement::Unavailable {
13007 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
13008 }
13009 );
13010 assert_eq!(verdict.image, image);
13011
13012 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
13013 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13014 };
13015 let verdict = reload_verdict(
13016 std::path::Path::new("/bin/same"),
13017 Some(std::path::Path::new("/bin/same")),
13018 unconfirmed.clone(),
13019 );
13020 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13021 assert_eq!(verdict.image, unconfirmed);
13022 }
13023
13024 #[test]
13025 fn reload_verdict_preserves_each_image_unavailability_reason() {
13026 use subc_control::RunningImageUnavailableReason as Reason;
13027
13028 for reason in [
13029 Reason::NotRunning,
13030 Reason::UnsupportedPlatform,
13031 Reason::RunningExecutableUnreadable,
13032 Reason::SpawnedPathUnreadable,
13033 Reason::HashFailed,
13034 Reason::ProcessIdentityUnconfirmed,
13035 Reason::Unknown("future_probe_reason".to_string()),
13036 ] {
13037 let image = subc_control::RunningImageAgreement::Unavailable {
13038 reason: reason.clone(),
13039 };
13040 let verdict = reload_verdict(
13041 std::path::Path::new("/bin/same"),
13042 Some(std::path::Path::new("/bin/same")),
13043 image.clone(),
13044 );
13045 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13046 assert_eq!(verdict.image, image, "{reason:?}");
13047 }
13048 }
13049
13050 #[tokio::test]
13051 async fn malformed_control_bodies_return_invalid_control_body() {
13052 let handler = ControlHandler::default();
13053 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
13054
13055 for (corr, body) in [
13056 (56, br#"{"route_channel":1}"#.as_slice()),
13057 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
13058 (
13059 58,
13060 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
13061 ),
13062 ] {
13063 let request = Frame::build(
13064 FrameType::Request,
13065 control_flags(),
13066 0,
13067 0,
13068 corr,
13069 body.to_vec(),
13070 )
13071 .unwrap();
13072 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13073
13074 assert_eq!(response.len(), 1);
13075 assert_eq!(response[0].header.ty, FrameType::Error);
13076 assert_eq!(response[0].header.corr, corr);
13077 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
13078 }
13079 }
13080
13081 #[tokio::test]
13082 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
13083 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13084 let registry = Arc::new(Registry::default());
13085 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
13086 let router = Router::with_control_handler(Arc::clone(&control));
13087 let connection = router.begin_connection();
13088 let (ctx, mut rx) = route_ctx(connection.id());
13089
13090 router
13091 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
13092 .await
13093 .unwrap();
13094 let response = rx.recv().await.unwrap();
13095 let ack = parse_ack(&response);
13096 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13097 let channel = 1;
13098
13099 let goodbye =
13100 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
13101 router.route_for_connection(&ctx, goodbye).await.unwrap();
13102 assert!(rx.try_recv().is_err());
13103 assert!(registry.get_module("aft").unwrap().is_none());
13104
13105 router
13106 .route_for_connection(&ctx, channel_request(channel, 13))
13107 .await
13108 .unwrap();
13109 let error_frame = rx.recv().await.unwrap();
13110 assert_eq!(error_frame.header.ty, FrameType::Error);
13111 assert_eq!(error_frame.header.channel, channel);
13112 let captured = crate::router::test_log::captured_logs(&logs);
13113 assert_eq!(
13114 captured
13115 .lines()
13116 .filter(|line| {
13117 line.contains("module_id=aft")
13118 && line.contains("reason=explicit_goodbye")
13119 && line.contains("module registration ended")
13120 })
13121 .count(),
13122 1,
13123 "unexpected GOODBYE registry log: {captured}"
13124 );
13125 }
13126
13127 #[tokio::test]
13128 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
13129 let registry = Arc::new(Registry::default());
13130 let handler = ControlHandler::new(registry).with_capability_config(
13131 [("prov".to_string(), true), ("cons".to_string(), true)],
13132 BTreeMap::new(),
13133 );
13134 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
13135 register_capability_manifest(
13136 &handler,
13137 &provider_ctx,
13138 &mut provider_rx,
13139 capability_manifest("prov", &["thing/v1"], &[]),
13140 1,
13141 )
13142 .await;
13143 let mut consumer = capability_manifest("cons", &[], &[]);
13144 consumer.capabilities.as_mut().unwrap().requires.push(
13145 subc_protocol::manifest::CapabilityRequirement {
13146 capability: "thing/v1".to_string(),
13147 need: subc_protocol::manifest::CapabilityNeed::Required,
13148 },
13149 );
13150 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
13151 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
13152 assert_eq!(
13153 handler.capability_evaluator.verdict("cons", "thing/v1"),
13154 Some(CapabilityVerdict::Provided)
13155 );
13156 let (mut client_rx, _) = open_route_for_capability_test(
13157 &handler,
13158 &provider_ctx,
13159 &mut provider_rx,
13160 703,
13161 3,
13162 "prov",
13163 None,
13164 )
13165 .await;
13166 let goodbye =
13167 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13168 handler
13169 .handle_control_frame(&provider_ctx, goodbye)
13170 .await
13171 .unwrap();
13172 assert_eq!(
13173 handler.capability_evaluator.verdict("cons", "thing/v1"),
13174 Some(CapabilityVerdict::NeverProvided)
13175 );
13176 let closed = client_rx
13177 .try_recv()
13178 .expect("GOODBYE pushes route.closed before route GOODBYE");
13179 assert!(
13180 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
13181 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
13182 );
13183 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
13184 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
13185 }
13186
13187 struct RouteClosedFixture {
13188 handler: ControlHandler,
13189 #[cfg(unix)]
13190 supervisor: Supervisor,
13191 module_ctx: RouteCtx,
13192 #[cfg(unix)]
13193 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13194 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13195 route: Arc<crate::forwarding::RouteBinding>,
13196 }
13197
13198 async fn route_closed_fixture() -> RouteClosedFixture {
13199 let registry = Arc::new(Registry::default());
13200 let handler = ControlHandler::new(Arc::clone(®istry));
13201 #[cfg(unix)]
13202 let supervisor = Supervisor::new(registry, RestartPolicy::default())
13203 .with_forwarding(Arc::clone(&handler.forwarding));
13204 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
13205 register_capability_manifest(
13206 &handler,
13207 &module_ctx,
13208 &mut module_rx,
13209 capability_manifest("prov", &[], &[]),
13210 1,
13211 )
13212 .await;
13213 let (client_rx, bind) = open_route_for_capability_test(
13214 &handler,
13215 &module_ctx,
13216 &mut module_rx,
13217 702,
13218 2,
13219 "prov",
13220 None,
13221 )
13222 .await;
13223 let ModuleControlRequest::RouteBind {
13224 route_channel: channel,
13225 epoch: route_epoch,
13226 ..
13227 } = bind
13228 else {
13229 panic!("expected route.bind");
13230 };
13231 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
13232 handler
13233 .forwarding
13234 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
13235 .unwrap()
13236 else {
13237 panic!("the acknowledged route must be bound");
13238 };
13239 RouteClosedFixture {
13240 handler,
13241 #[cfg(unix)]
13242 supervisor,
13243 module_ctx,
13244 #[cfg(unix)]
13245 module_rx,
13246 client_rx,
13247 route,
13248 }
13249 }
13250
13251 impl RouteClosedFixture {
13252 #[cfg(unix)]
13253 async fn cut(&mut self) {
13254 let notices = async {
13255 let mut draining = self.module_rx.recv().await.unwrap();
13256 assert!(matches!(
13257 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
13258 &draining.body
13259 )
13260 .unwrap(),
13261 subc_protocol::session::ModuleControlCommand::Draining {
13262 reason: RouteCloseReason::Restart,
13263 ..
13264 }
13265 ));
13266 draining.flushed.take().unwrap().send(()).unwrap();
13267 let mut closing = self.client_rx.recv().await.unwrap();
13268 assert!(matches!(
13269 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
13270 ClientControlPush::RouteClosing {
13271 reason: RouteCloseReason::Restart,
13272 ..
13273 }
13274 ));
13275 closing.flushed.take().unwrap().send(()).unwrap();
13276 };
13277 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
13278 result.unwrap();
13279 }
13280
13281 fn disconnect(&mut self) -> ClientControlPush {
13282 self.handler
13283 .cleanup_connection(self.module_ctx.connection_id)
13284 .unwrap();
13285 let closed = self.client_rx.try_recv().unwrap();
13286 assert_eq!(closed.header.ty, FrameType::Push);
13287 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
13288 assert!(matches!(
13289 &push,
13290 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
13291 if channels == &[self.route.client_channel]
13292 ));
13293 assert_eq!(
13294 self.client_rx.try_recv().unwrap().header.ty,
13295 FrameType::Goodbye
13296 );
13297 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
13298 push
13299 }
13300 }
13301
13302 #[cfg(unix)]
13303 #[tokio::test(start_paused = true)]
13304 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
13305 let mut fixture = route_closed_fixture().await;
13306 fixture.cut().await;
13307 assert!(matches!(
13308 fixture.disconnect(),
13309 ClientControlPush::RouteClosed {
13310 reason: RouteCloseReason::Restart,
13311 drained: true,
13312 ..
13313 }
13314 ));
13315 }
13316
13317 #[cfg(unix)]
13318 #[tokio::test(start_paused = true)]
13319 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
13320 let mut fixture = route_closed_fixture().await;
13321 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
13322 let started = tokio::time::Instant::now();
13323 fixture.cut().await;
13324 assert!(started.elapsed() >= Duration::from_secs(2));
13325 assert_eq!(fixture.route.flow.in_flight(), 1);
13326 assert!(matches!(
13327 fixture.disconnect(),
13328 ClientControlPush::RouteClosed {
13329 reason: RouteCloseReason::Restart,
13330 drained: false,
13331 ..
13332 }
13333 ));
13334 }
13335
13336 #[tokio::test]
13337 async fn module_connection_crash_reports_crash_and_not_drained() {
13338 let mut fixture = route_closed_fixture().await;
13339 assert!(matches!(
13343 fixture.disconnect(),
13344 ClientControlPush::RouteClosed {
13345 reason: RouteCloseReason::Crash,
13346 drained: false,
13347 ..
13348 }
13349 ));
13350 }
13351
13352 #[tokio::test]
13353 async fn dropping_router_connection_releases_registration() {
13354 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13355 let registry = Arc::new(Registry::default());
13356 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
13357 let router = Router::with_control_handler(Arc::clone(&control));
13358 let connection = router.begin_connection();
13359 let connection_id = connection.id();
13360 let (ctx, mut rx) = route_ctx(connection_id);
13361
13362 router
13363 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
13364 .await
13365 .unwrap();
13366 let response = rx.recv().await.unwrap();
13367 let ack = parse_ack(&response);
13368 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13369 assert!(registry.get_module("aft").unwrap().is_some());
13370
13371 drop(connection);
13372
13373 assert!(registry.get_module("aft").unwrap().is_none());
13374 assert_eq!(registry.active_registration_count().unwrap(), 0);
13375
13376 control
13377 .cleanup_connection(ConnectionId::new(u64::MAX))
13378 .unwrap();
13379 let captured = crate::router::test_log::captured_logs(&logs);
13380 println!("captured registry lifecycle logs:\n{captured}");
13381 let events: Vec<_> = captured
13382 .lines()
13383 .filter(|line| {
13384 line.contains("module registered module_id=aft ")
13385 || line.contains("module registration ended")
13386 })
13387 .collect();
13388 assert_eq!(
13389 events.len(),
13390 2,
13391 "unexpected registry lifecycle logs: {captured}"
13392 );
13393 assert!(events[0].contains("module registered module_id=aft "));
13394 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
13395 assert!(events[1].contains(&format!(
13396 "module_id=aft connection_id={}",
13397 connection_id.get()
13398 )));
13399 assert!(events[1].contains("reason=connection_closed"));
13400 assert!(events[1].contains("module registration ended"));
13401 assert_eq!(
13402 captured
13403 .lines()
13404 .filter(|line| line.contains("module registered module_id=aft "))
13405 .count(),
13406 1,
13407 "legacy registration admission line must appear once: {captured}"
13408 );
13409 }
13410
13411 #[tokio::test]
13412 async fn hello_registration_keeps_legacy_module_registered_line_once() {
13413 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13414 let registry = Arc::new(Registry::default());
13415 let control = ControlHandler::new(Arc::clone(®istry));
13416 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
13417 hello_via_sink(
13418 &control,
13419 &ctx,
13420 &mut rx,
13421 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
13422 )
13423 .await;
13424
13425 let captured = crate::router::test_log::captured_logs(&logs);
13426 let admissions: Vec<_> = captured
13427 .lines()
13428 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
13429 .collect();
13430 assert_eq!(
13431 admissions.len(),
13432 1,
13433 "legacy admission line must remain exactly once: {captured}"
13434 );
13435 assert!(admissions[0].contains("routable_provider=true"));
13436 assert!(admissions[0].contains("connection_id=777"));
13437 }
13438
13439 fn capability_manifest(
13440 module_id: &str,
13441 provides: &[&str],
13442 must_never_reach: &[&str],
13443 ) -> ModuleManifest {
13444 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
13445 manifest.capabilities = Some(CapabilityDeclarations {
13446 provides: provides
13447 .iter()
13448 .map(|capability| (*capability).to_string())
13449 .collect(),
13450 requires: Vec::new(),
13451 must_never_reach: must_never_reach
13452 .iter()
13453 .map(|capability| (*capability).to_string())
13454 .collect(),
13455 });
13456 manifest
13457 }
13458
13459 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
13460 Frame::build(
13461 FrameType::Hello,
13462 control_flags(),
13463 0,
13464 0,
13465 corr,
13466 serde_json::to_vec(&ModuleHelloBody {
13467 protocol_ver: manifest.protocol_ver,
13468 manifest,
13469 control_ops: None,
13470 launch_nonce: None,
13471 })
13472 .expect("capability test HELLO serializes"),
13473 )
13474 .expect("capability test HELLO frame builds")
13475 }
13476
13477 fn catalog_update_with_capabilities_frame(
13478 corr: u64,
13479 capabilities: CapabilityDeclarations,
13480 ) -> Frame {
13481 Frame::build(
13482 FrameType::Request,
13483 control_flags(),
13484 0,
13485 0,
13486 corr,
13487 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
13488 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
13489 capabilities: Some(capabilities),
13490 ready: None,
13491 })
13492 .expect("capability catalog.update serializes"),
13493 )
13494 .expect("capability catalog.update frame builds")
13495 }
13496
13497 async fn register_capability_manifest(
13498 handler: &ControlHandler,
13499 ctx: &RouteCtx,
13500 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13501 manifest: ModuleManifest,
13502 corr: u64,
13503 ) {
13504 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
13505 }
13506
13507 async fn open_route_for_capability_test(
13508 handler: &ControlHandler,
13509 target_ctx: &RouteCtx,
13510 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13511 client_connection_id: u64,
13512 corr: u64,
13513 target_module_id: &str,
13514 consumer_identity: Option<ConsumerIdentity>,
13515 ) -> (
13516 mpsc::Receiver<crate::router::OutboundFrame>,
13517 ModuleControlRequest,
13518 ) {
13519 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
13520 let route_handler = handler.clone();
13521 let target_module_id = target_module_id.to_string();
13522 let route_task = tokio::spawn(async move {
13523 route_handler
13524 .handle_control_frame(
13525 &client_ctx,
13526 route_open_frame_with_admission_facts(
13527 corr,
13528 &target_module_id,
13529 unique_project_root("admission-facts"),
13530 consumer_identity,
13531 None,
13532 ),
13533 )
13534 .await
13535 .expect("capability test route.open succeeds")
13536 });
13537 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
13538 .await
13539 .expect("capability test route.open must reach route.bind")
13540 .expect("target control receiver stays open");
13541 let bind_request: ModuleControlRequest =
13542 serde_json::from_slice(&bind.body).expect("route.bind decodes");
13543 handler
13544 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
13545 .await
13546 .expect("capability test route.bind ACK succeeds");
13547 assert!(route_task.await.expect("route.open task joins").is_empty());
13548 let opened = client_rx
13549 .recv()
13550 .await
13551 .expect("successful route.open publishes a response");
13552 assert!(matches!(
13553 serde_json::from_slice::<ClientControlResponse>(&opened.body),
13554 Ok(ClientControlResponse::RouteOpen { .. })
13555 ));
13556 (client_rx, bind_request)
13557 }
13558
13559 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
13560 assert_eq!(frame.header.ty, FrameType::Push);
13561 assert_eq!(frame.header.channel, 0);
13562 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
13563 .expect("route.closed control push decodes");
13564 let ClientControlPush::RouteClosed { channels, .. } = &push else {
13565 panic!("expected route.closed");
13566 };
13567 assert_eq!(channels.len(), 1, "exactly one violating route closed");
13568 let channels = channels.clone();
13569 assert_eq!(
13570 push,
13571 ClientControlPush::RouteClosed {
13572 module_id: target_module_id.to_string(),
13573 channels,
13574 reason: RouteCloseReason::CapabilityDenied,
13575 drained: false,
13576 abandoned: 0,
13577 excluded_subscriptions: 0,
13578 terminal: Some(false),
13579 }
13580 );
13581 }
13582
13583 #[tokio::test]
13584 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
13585 let registry = Arc::new(Registry::default());
13586 let forwarding = Arc::new(ForwardingTable::default());
13587 let supervisor = SupervisorHandle::new();
13588 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13589 let handler =
13590 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13591 .with_supervisor(supervisor);
13592 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
13593 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
13594 register_capability_manifest(
13595 &handler,
13596 &target_ctx,
13597 &mut target_rx,
13598 capability_manifest("target", &["credentials-provider/v1"], &[]),
13599 1,
13600 )
13601 .await;
13602 register_capability_manifest(
13603 &handler,
13604 &opener_ctx,
13605 &mut opener_rx,
13606 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13607 2,
13608 )
13609 .await;
13610
13611 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
13612 let replies = handler
13613 .handle_control_frame(
13614 &client_ctx,
13615 route_open_frame_with_admission_facts(
13616 3,
13617 "target",
13618 unique_project_root("admission-facts"),
13619 Some(ConsumerIdentity {
13620 module_id: "opener".to_string(),
13621 launch_nonce: "opener-nonce".to_string(),
13622 }),
13623 None,
13624 ),
13625 )
13626 .await
13627 .expect("denied route.open returns a typed frame");
13628 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
13629 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13630 assert!(
13631 target_rx.try_recv().is_err(),
13632 "forbidden route.open must not relay route.bind"
13633 );
13634 }
13635
13636 #[tokio::test]
13637 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
13638 let registry = Arc::new(Registry::default());
13639 let forwarding = Arc::new(ForwardingTable::default());
13640 let supervisor = SupervisorHandle::new();
13641 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13642 let handler =
13643 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13644 .with_supervisor(supervisor);
13645 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
13646 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
13647 register_capability_manifest(
13648 &handler,
13649 &target_ctx,
13650 &mut target_rx,
13651 capability_manifest("target", &["credentials-provider/v1"], &[]),
13652 1,
13653 )
13654 .await;
13655 register_capability_manifest(
13656 &handler,
13657 &old_opener_ctx,
13658 &mut old_opener_rx,
13659 capability_manifest("opener", &[], &[]),
13660 2,
13661 )
13662 .await;
13663 let (mut client_rx, _) = open_route_for_capability_test(
13664 &handler,
13665 &target_ctx,
13666 &mut target_rx,
13667 712,
13668 3,
13669 "target",
13670 Some(ConsumerIdentity {
13671 module_id: "opener".to_string(),
13672 launch_nonce: "opener-nonce".to_string(),
13673 }),
13674 )
13675 .await;
13676 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13677
13678 handler
13679 .cleanup_connection(old_opener_ctx.connection_id)
13680 .expect("old opener registration cleans up");
13681 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
13682 register_capability_manifest(
13683 &handler,
13684 &new_opener_ctx,
13685 &mut new_opener_rx,
13686 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13687 4,
13688 )
13689 .await;
13690
13691 assert_capability_denied_push(
13692 client_rx
13693 .try_recv()
13694 .expect("HELLO deny addition must emit route.closed")
13695 .frame,
13696 "target",
13697 );
13698 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13699 assert!(matches!(
13700 target_rx.try_recv(),
13701 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
13702 ));
13703 }
13704
13705 #[tokio::test]
13706 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
13707 let registry = Arc::new(Registry::default());
13708 let forwarding = Arc::new(ForwardingTable::default());
13709 let supervisor = SupervisorHandle::new();
13710 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13711 let handler =
13712 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13713 .with_supervisor(supervisor);
13714 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
13715 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
13716 register_capability_manifest(
13717 &handler,
13718 &target_ctx,
13719 &mut target_rx,
13720 capability_manifest("target", &[], &[]),
13721 1,
13722 )
13723 .await;
13724 register_capability_manifest(
13725 &handler,
13726 &opener_ctx,
13727 &mut opener_rx,
13728 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13729 2,
13730 )
13731 .await;
13732 let (mut client_rx, _) = open_route_for_capability_test(
13733 &handler,
13734 &target_ctx,
13735 &mut target_rx,
13736 722,
13737 3,
13738 "target",
13739 Some(ConsumerIdentity {
13740 module_id: "opener".to_string(),
13741 launch_nonce: "opener-nonce".to_string(),
13742 }),
13743 )
13744 .await;
13745 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13746
13747 let replies = handler
13748 .handle_control_frame(
13749 &target_ctx,
13750 catalog_update_with_capabilities_frame(
13751 4,
13752 CapabilityDeclarations {
13753 provides: vec!["credentials-provider/v1".to_string()],
13754 requires: Vec::new(),
13755 must_never_reach: Vec::new(),
13756 },
13757 ),
13758 )
13759 .await
13760 .expect("claim catalog.update succeeds");
13761 assert!(matches!(
13762 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
13763 Ok(ModuleControlResponseToModule::CatalogUpdate {})
13764 ));
13765 assert_capability_denied_push(
13766 client_rx
13767 .try_recv()
13768 .expect("claim addition must emit route.closed")
13769 .frame,
13770 "target",
13771 );
13772 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13773 assert!(matches!(
13774 target_rx.try_recv(),
13775 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
13776 ));
13777 }
13778
13779 #[tokio::test]
13780 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
13781 let registry = Arc::new(Registry::default());
13782 let forwarding = Arc::new(ForwardingTable::default());
13783 let supervisor = SupervisorHandle::new();
13784 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13785 let handler =
13786 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13787 .with_supervisor(supervisor);
13788 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
13789 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
13790 register_capability_manifest(
13791 &handler,
13792 &target_ctx,
13793 &mut target_rx,
13794 capability_manifest("target", &["credentials-provider/v1"], &[]),
13795 1,
13796 )
13797 .await;
13798 register_capability_manifest(
13799 &handler,
13800 &opener_ctx,
13801 &mut opener_rx,
13802 capability_manifest("opener", &[], &[]),
13803 2,
13804 )
13805 .await;
13806 let (mut client_rx, _) = open_route_for_capability_test(
13807 &handler,
13808 &target_ctx,
13809 &mut target_rx,
13810 732,
13811 3,
13812 "target",
13813 Some(ConsumerIdentity {
13814 module_id: "opener".to_string(),
13815 launch_nonce: "opener-nonce".to_string(),
13816 }),
13817 )
13818 .await;
13819 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13820
13821 handler
13822 .handle_control_frame(
13823 &target_ctx,
13824 catalog_update_with_capabilities_frame(
13825 4,
13826 CapabilityDeclarations {
13827 provides: Vec::new(),
13828 requires: Vec::new(),
13829 must_never_reach: Vec::new(),
13830 },
13831 ),
13832 )
13833 .await
13834 .expect("claim removal catalog.update succeeds");
13835 assert_eq!(
13836 forwarding.active_binding_count().unwrap(),
13837 1,
13838 "removing an attested target claim must leave the route census unchanged"
13839 );
13840 assert!(
13841 client_rx.try_recv().is_err(),
13842 "claim removal must not emit route.closed capability_denied"
13843 );
13844 assert!(
13845 target_rx.try_recv().is_err(),
13846 "claim removal must not send the target a route GOODBYE"
13847 );
13848 }
13849
13850 #[tokio::test]
13853 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
13854 let registry = Arc::new(Registry::default());
13855 let forwarding = Arc::new(ForwardingTable::default());
13856 let supervisor = SupervisorHandle::new();
13857 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13858 let handler =
13859 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13860 .with_supervisor(supervisor);
13861 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
13862 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
13863 register_capability_manifest(
13864 &handler,
13865 &target_ctx,
13866 &mut target_rx,
13867 capability_manifest("target", &["credentials-provider/v1"], &[]),
13868 1,
13869 )
13870 .await;
13871 register_capability_manifest(
13872 &handler,
13873 &opener_ctx,
13874 &mut opener_rx,
13875 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13876 2,
13877 )
13878 .await;
13879
13880 let (_client_rx, bind) = open_route_for_capability_test(
13881 &handler,
13882 &target_ctx,
13883 &mut target_rx,
13884 742,
13885 3,
13886 "target",
13887 None,
13888 )
13889 .await;
13890 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
13891 panic!("direct scope-honesty route must bind");
13892 };
13893 assert_eq!(principal, Some(Principal::Direct));
13894 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13895 }
13896
13897 #[tokio::test]
13900 async fn must_never_reach_self_route_is_capability_forbidden() {
13901 let registry = Arc::new(Registry::default());
13902 let forwarding = Arc::new(ForwardingTable::default());
13903 let supervisor = SupervisorHandle::new();
13904 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
13905 let handler =
13906 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13907 .with_supervisor(supervisor);
13908 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
13909 register_capability_manifest(
13910 &handler,
13911 &self_ctx,
13912 &mut self_rx,
13913 capability_manifest(
13914 "self-provider",
13915 &["credentials-provider/v1"],
13916 &["credentials-provider/v1"],
13917 ),
13918 1,
13919 )
13920 .await;
13921
13922 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
13923 let replies = handler
13924 .handle_control_frame(
13925 &client_ctx,
13926 route_open_frame_with_admission_facts(
13927 2,
13928 "self-provider",
13929 unique_project_root("admission-facts"),
13930 Some(ConsumerIdentity {
13931 module_id: "self-provider".to_string(),
13932 launch_nonce: "self-nonce".to_string(),
13933 }),
13934 None,
13935 ),
13936 )
13937 .await
13938 .expect("self-route refusal returns a typed frame");
13939 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
13940 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13941 assert!(
13942 self_rx.try_recv().is_err(),
13943 "self denial must not relay route.bind"
13944 );
13945 }
13946
13947 #[test]
13948 fn unsupported_channel_zero_frame_returns_error() {
13949 let handler = ControlHandler::default();
13950 let request = Frame::build(
13951 FrameType::Request,
13952 control_flags(),
13953 0,
13954 0,
13955 21,
13956 b"opaque".to_vec(),
13957 )
13958 .unwrap();
13959
13960 let response = handler
13961 .handle_control(ConnectionId::new(1), request)
13962 .unwrap();
13963
13964 assert_eq!(response[0].header.ty, FrameType::Error);
13965 assert_eq!(
13966 parse_error(&response[0])["code"],
13967 "unsupported_control_frame"
13968 );
13969 }
13970
13971 mod swap {
13976 use super::*;
13977
13978 const INCUMBENT: ConnectionId = ConnectionId::new(30);
13979 const CANDIDATE: ConnectionId = ConnectionId::new(40);
13980
13981 struct Swap {
13982 registry: Arc<Registry>,
13983 forwarding: Arc<ForwardingTable>,
13984 handler: ControlHandler,
13985 incumbent_ctx: RouteCtx,
13986 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13987 candidate_ctx: RouteCtx,
13988 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13989 }
13990
13991 async fn swap_with_incumbent() -> Swap {
13992 let registry = Arc::new(Registry::default());
13993 let forwarding = Arc::new(ForwardingTable::default());
13994 let handler =
13995 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13996 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
13997 hello_via_sink(
13998 &handler,
13999 &incumbent_ctx,
14000 &mut incumbent_rx,
14001 hello_frame("aft", PROTOCOL_VERSION, 7),
14002 )
14003 .await;
14004 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
14005 Swap {
14006 registry,
14007 forwarding,
14008 handler,
14009 incumbent_ctx,
14010 incumbent_rx,
14011 candidate_ctx,
14012 candidate_rx,
14013 }
14014 }
14015
14016 fn register_candidate(swap: &Swap, ready: Option<bool>) {
14017 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
14018 candidate_manifest.ready = ready;
14019 let registration = swap
14020 .registry
14021 .register_candidate_with_control_ops(
14022 candidate_manifest,
14023 PROTOCOL_VERSION,
14024 CANDIDATE,
14025 module_baseline_control_ops(),
14026 )
14027 .unwrap();
14028 swap.forwarding
14029 .register_candidate_module_connection(
14030 CANDIDATE,
14031 "aft".to_string(),
14032 PROTOCOL_VERSION,
14033 manifest_concurrency(®istration.manifest),
14034 swap.candidate_ctx.egress.clone(),
14035 )
14036 .unwrap();
14037 }
14038
14039 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
14040 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
14041 swap.registry.promote_candidate("aft").unwrap().unwrap();
14042 cutover.incumbent.unwrap()
14043 }
14044
14045 #[tokio::test]
14054 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
14055 let mut swap = swap_with_incumbent().await;
14056 let hold = Arc::new(std::sync::Barrier::new(2));
14057 let handler = swap
14058 .handler
14059 .clone()
14060 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
14061 handler
14062 .supervisor
14063 .open_swap("aft", "candidate-nonce".to_string());
14064 let hello = {
14065 let handler = handler.clone();
14066 let candidate_ctx = swap.candidate_ctx.clone();
14067 std::thread::spawn(move || {
14068 handler
14069 .handle_hello(
14070 candidate_ctx.connection_id,
14071 Some(candidate_ctx.egress.clone()),
14072 hello_frame_with_nonce(
14073 "aft",
14074 PROTOCOL_VERSION,
14075 8,
14076 Some("candidate-nonce"),
14077 ),
14078 )
14079 .unwrap()
14080 })
14081 };
14082
14083 hold.wait();
14084 assert!(swap
14087 .registry
14088 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
14089 .unwrap()
14090 .is_some());
14091 assert!(swap
14092 .registry
14093 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
14094 .unwrap()
14095 .is_none());
14096 assert!(swap
14097 .forwarding
14098 .module_endpoint_for_connection(CANDIDATE)
14099 .unwrap()
14100 .is_none());
14101
14102 hold.wait();
14103 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
14104 assert_eq!(
14105 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
14106 FrameType::HelloAck
14107 );
14108 let published = swap
14109 .registry
14110 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
14111 .unwrap()
14112 .expect("candidate is published once its HELLO completes");
14113 assert_eq!(published.connection_id, CANDIDATE);
14114 assert!(swap
14115 .forwarding
14116 .module_endpoint_for_connection(CANDIDATE)
14117 .unwrap()
14118 .is_some());
14119 }
14120
14121 fn keyed_total(counters: &Value, key: &str) -> u64 {
14122 counters[key]
14123 .as_object()
14124 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
14125 .unwrap_or(0)
14126 }
14127
14128 #[tokio::test]
14129 async fn replacement_logs_old_end_and_new_admission_once() {
14130 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14131 let mut swap = swap_with_incumbent().await;
14132 swap.handler
14133 .supervisor
14134 .open_swap("aft", "candidate-nonce".to_string());
14135 hello_via_sink(
14136 &swap.handler,
14137 &swap.candidate_ctx,
14138 &mut swap.candidate_rx,
14139 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
14140 )
14141 .await;
14142 cutover(&swap);
14143 swap.handler.cleanup_connection(INCUMBENT).unwrap();
14144
14145 let captured = crate::router::test_log::captured_logs(&logs);
14146 println!("captured replacement registry logs:\n{captured}");
14147 let new_admissions: Vec<_> = captured
14148 .lines()
14149 .filter(|line| {
14150 line.contains("connection_id=40")
14151 && line.contains("swap candidate registered; not routable until cutover")
14152 })
14153 .collect();
14154 assert_eq!(
14155 new_admissions.len(),
14156 1,
14157 "unexpected admission logs: {captured}"
14158 );
14159 assert!(new_admissions[0].contains("module_id=aft"));
14160 assert!(new_admissions[0].contains("connection_id=40"));
14161 assert!(
14162 new_admissions[0].contains("swap candidate registered; not routable until cutover")
14163 );
14164 assert!(new_admissions[0].contains("ready=true"));
14165
14166 let old_admissions: Vec<_> = captured
14167 .lines()
14168 .filter(|line| {
14169 line.contains("module registered module_id=aft ")
14170 && line.contains("connection_id=30")
14171 })
14172 .collect();
14173 assert_eq!(
14174 old_admissions.len(),
14175 1,
14176 "unexpected incumbent admission: {captured}"
14177 );
14178
14179 let old_ends: Vec<_> = captured
14180 .lines()
14181 .filter(|line| {
14182 line.contains("connection_id=30") && line.contains("module registration ended")
14183 })
14184 .collect();
14185 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
14186 assert!(old_ends[0].contains("module_id=aft"));
14187 assert!(old_ends[0].contains("connection_id=30"));
14188 assert!(old_ends[0].contains("reason=replaced"));
14189 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
14190 assert_eq!(
14191 captured
14192 .lines()
14193 .filter(|line| line.contains("module registration promoted"))
14194 .count(),
14195 1,
14196 "unexpected promotion logs: {captured}"
14197 );
14198 }
14199
14200 #[tokio::test]
14206 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
14207 let mut swap = swap_with_incumbent().await;
14208 let handler = swap.handler.clone();
14209
14210 let cotenant = ConnectionId::new(31);
14212 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
14213 let (cotenant_task, cotenant_bind) = relay_route_open(
14214 &handler,
14215 cotenant,
14216 &cotenant_ctx.egress,
14217 &mut swap.incumbent_rx,
14218 100,
14219 "aft",
14220 "swap-cotenant",
14221 )
14222 .await;
14223 handler
14224 .handle_control_frame(
14225 &swap.incumbent_ctx,
14226 route_bind_ack(cotenant_bind.header.corr),
14227 )
14228 .await
14229 .unwrap();
14230 assert!(cotenant_task.await.unwrap().is_empty());
14231 let (cotenant_channel, cotenant_epoch) =
14232 published_route(&cotenant_rx.recv().await.unwrap());
14233
14234 let caller = ConnectionId::new(32);
14236 let (caller_ctx, mut caller_rx) = route_ctx(caller);
14237 let (caller_task, caller_bind) = relay_route_open(
14238 &handler,
14239 caller,
14240 &caller_ctx.egress,
14241 &mut swap.incumbent_rx,
14242 101,
14243 "aft",
14244 "swap-caller",
14245 )
14246 .await;
14247 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
14248
14249 register_candidate(&swap, None);
14250 cutover(&swap);
14251
14252 let ack = handler
14254 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
14255 .await;
14256 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
14257 if module_loop_error.is_some() {
14258 handler.cleanup_connection(INCUMBENT).unwrap();
14261 }
14262
14263 assert!(
14265 cotenant_rx.try_recv().is_err(),
14266 "the co-tenant route on the incumbent was torn down by one late ack: \
14267 {module_loop_error:?}"
14268 );
14269 assert!(matches!(
14270 swap.forwarding
14271 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
14272 .unwrap(),
14273 DataRoute::Client(DataRouteState::Bound(_))
14274 ));
14275 assert_eq!(module_loop_error, None);
14276 assert!(swap
14277 .registry
14278 .get_module_by_connection(INCUMBENT)
14279 .unwrap()
14280 .is_some());
14281
14282 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
14284 .await
14285 .expect("the incumbent is told to drop the abandoned binding")
14286 .unwrap()
14287 .frame;
14288 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14289 assert_eq!(goodbye.header.channel, abandoned_channel);
14290 assert_eq!(goodbye.header.epoch, abandoned_epoch);
14291 assert!(swap.incumbent_rx.try_recv().is_err());
14292
14293 let response = caller_task.await.unwrap();
14295 assert_eq!(response.len(), 1);
14296 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14297 assert!(caller_rx.try_recv().is_err());
14298
14299 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
14303 let counters = handler.counters().snapshot();
14304 assert_eq!(
14305 keyed_total(&counters, "route_open_accepted_by_principal"),
14306 1
14307 );
14308 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
14309 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
14310 }
14311
14312 #[tokio::test]
14316 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
14317 let mut swap = swap_with_incumbent().await;
14318 register_candidate(&swap, None);
14319 let incumbent = cutover(&swap);
14320 swap.forwarding
14321 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
14322 .unwrap()
14323 .expect("the incumbent is still registered");
14324
14325 let client = ConnectionId::new(33);
14326 let (client_ctx, mut client_rx) = route_ctx(client);
14327 let route_handler = swap.handler.clone();
14328 let open_ctx = RouteCtx {
14329 connection_id: client,
14330 egress: client_ctx.egress.clone(),
14331 };
14332 let mut route_task = tokio::spawn(async move {
14333 route_handler
14334 .handle_control_frame(
14335 &open_ctx,
14336 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
14337 )
14338 .await
14339 .unwrap()
14340 });
14341 let bind = tokio::select! {
14342 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
14343 response = &mut route_task => {
14344 let response = response.unwrap();
14345 panic!(
14346 "post-cutover route.open was refused instead of relayed to the candidate: {}",
14347 parse_error(&response[0])["code"]
14348 );
14349 }
14350 };
14351 swap.handler
14352 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
14353 .await
14354 .unwrap();
14355 assert!(route_task.await.unwrap().is_empty());
14356 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
14357 match swap
14358 .forwarding
14359 .lookup_data_route(client, channel, epoch)
14360 .unwrap()
14361 {
14362 DataRoute::Client(DataRouteState::Bound(route)) => {
14363 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
14364 }
14365 other => panic!("expected a bound route on the candidate, got {other:?}"),
14366 }
14367 assert!(swap.incumbent_rx.try_recv().is_err());
14368 }
14369
14370 #[tokio::test]
14375 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
14376 let swap = swap_with_incumbent().await;
14377 register_candidate(&swap, Some(false));
14378 let update = Frame::build(
14379 FrameType::Request,
14380 control_flags(),
14381 0,
14382 0,
14383 55,
14384 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14385 provides: manifest("aft", PROTOCOL_VERSION).provides,
14386 capabilities: None,
14387 ready: Some(true),
14388 })
14389 .unwrap(),
14390 )
14391 .unwrap();
14392
14393 let replies = swap
14394 .handler
14395 .handle_control_frame(&swap.candidate_ctx, update)
14396 .await
14397 .unwrap();
14398
14399 assert_eq!(replies.len(), 1);
14400 assert_eq!(
14401 replies[0].header.ty,
14402 FrameType::Response,
14403 "candidate catalog.update was refused: {:?}",
14404 serde_json::from_slice::<Value>(&replies[0].body).ok()
14405 );
14406 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
14407 assert_eq!(
14408 swap.registry
14409 .get_module("aft")
14410 .unwrap()
14411 .unwrap()
14412 .connection_id,
14413 INCUMBENT
14414 );
14415 }
14416 }
14417
14418 mod swap_admission {
14422 use super::*;
14423
14424 const INCUMBENT_NONCE: &str = "incumbent-nonce";
14425 const CANDIDATE_NONCE: &str = "candidate-nonce";
14426
14427 fn handler_with_incumbent(
14428 module_id: &str,
14429 reserved: bool,
14430 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
14431 let registry = Arc::new(Registry::default());
14432 let supervisor = SupervisorHandle::new();
14433 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
14434 if reserved {
14435 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
14436 }
14437 let handler =
14438 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
14439 let incumbent = handler
14440 .handle_control(
14441 ConnectionId::new(1),
14442 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
14443 )
14444 .unwrap();
14445 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
14446 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
14447 (registry, supervisor, handler)
14448 }
14449
14450 #[test]
14456 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
14457 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
14458
14459 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
14460 let replies = handler
14461 .handle_control(
14462 ConnectionId::new(connection),
14463 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
14464 )
14465 .unwrap();
14466 assert_eq!(replies[0].header.ty, FrameType::Error);
14467 assert_eq!(
14468 parse_error(&replies[0])["code"],
14469 "swap_token_invalid",
14470 "nonce {nonce:?}"
14471 );
14472 }
14473 assert!(registry.get_candidate("aft").unwrap().is_none());
14474 assert_eq!(
14475 registry.get_module("aft").unwrap().unwrap().connection_id,
14476 ConnectionId::new(1)
14477 );
14478
14479 let admitted = handler
14482 .handle_control(
14483 ConnectionId::new(4),
14484 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
14485 )
14486 .unwrap();
14487 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
14488 assert_eq!(
14489 registry
14490 .get_candidate("aft")
14491 .unwrap()
14492 .unwrap()
14493 .connection_id,
14494 ConnectionId::new(4)
14495 );
14496 assert_eq!(
14497 registry.get_module("aft").unwrap().unwrap().connection_id,
14498 ConnectionId::new(1),
14499 "the candidate must not take the active slot"
14500 );
14501 let replayed = handler
14502 .handle_control(
14503 ConnectionId::new(5),
14504 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
14505 )
14506 .unwrap();
14507 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
14508
14509 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
14513 let squatter = handler
14514 .handle_control(
14515 ConnectionId::new(6),
14516 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
14517 )
14518 .unwrap();
14519 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
14520 assert!(
14521 registry.get_module("aft").unwrap().is_none(),
14522 "a squatter took the active slot of an id being swapped"
14523 );
14524 }
14525
14526 #[test]
14531 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
14532 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
14533
14534 let replies = handler
14535 .handle_control(
14536 ConnectionId::new(2),
14537 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14538 )
14539 .unwrap();
14540
14541 assert_eq!(
14542 replies[0].header.ty,
14543 FrameType::HelloAck,
14544 "reserved candidate refused: {:?}",
14545 serde_json::from_slice::<Value>(&replies[0].body).ok()
14546 );
14547 assert_eq!(
14548 registry
14549 .get_candidate("vault")
14550 .unwrap()
14551 .unwrap()
14552 .connection_id,
14553 ConnectionId::new(2)
14554 );
14555 }
14556
14557 #[test]
14560 fn without_an_open_swap_the_ordinary_gates_decide() {
14561 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
14562 supervisor.close_swap("vault");
14563
14564 let candidate = handler
14565 .handle_control(
14566 ConnectionId::new(2),
14567 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14568 )
14569 .unwrap();
14570 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
14571 let duplicate = handler
14572 .handle_control(
14573 ConnectionId::new(3),
14574 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
14575 )
14576 .unwrap();
14577 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14578 assert!(registry.get_candidate("vault").unwrap().is_none());
14579 }
14580 }
14581
14582 mod scopes {
14586 mod idle_tests {
14587 use super::*;
14588 include!("scope_idle_tests.rs");
14589 }
14590 mod route_open_timings {
14591 use super::*;
14592 include!("route_open_timing_tests.rs");
14593 }
14594 mod activity {
14595 include!("activity_scope_tests.rs");
14596 }
14597 mod delta {
14598 include!("scope_handler_delta_tests.rs");
14599 }
14600 mod call_modes {
14601 include!("scope_call_mode_tests.rs");
14602 }
14603
14604 use subc_protocol::scope::{
14605 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
14606 ScopeStatus,
14607 };
14608
14609 use super::*;
14610
14611 const OWNER: &str = "prefrontal-core";
14612
14613 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
14614 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
14615 }
14616
14617 async fn call(
14618 handler: &ControlHandler,
14619 ctx: &RouteCtx,
14620 request: &ModuleControlRequestFromModule,
14621 ) -> Frame {
14622 let body = serde_json::to_vec(request).unwrap();
14623 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
14624 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
14625 assert_eq!(replies.len(), 1, "{replies:?}");
14626 replies.pop().unwrap()
14627 }
14628
14629 async fn sync(
14630 handler: &ControlHandler,
14631 ctx: &RouteCtx,
14632 generation: u64,
14633 scopes: Vec<ScopeRecord>,
14634 ) -> Result<ModuleControlResponseToModule, String> {
14635 let reply = call(
14636 handler,
14637 ctx,
14638 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
14639 )
14640 .await;
14641 match reply.header.ty {
14642 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
14643 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
14644 }
14645 }
14646
14647 async fn describe(
14648 handler: &ControlHandler,
14649 ctx: &RouteCtx,
14650 owner: &str,
14651 scope_ref: &str,
14652 ) -> ModuleControlResponseToModule {
14653 let reply = call(
14654 handler,
14655 ctx,
14656 &ModuleControlRequestFromModule::ScopeDescribe {
14657 owner: Principal::Reserved {
14658 module_id: owner.to_string(),
14659 },
14660 scope_ref: scope_ref.to_string(),
14661 },
14662 )
14663 .await;
14664 assert_eq!(
14665 reply.header.ty,
14666 FrameType::Response,
14667 "{:?}",
14668 parse_error(&reply)
14669 );
14670 serde_json::from_slice(&reply.body).unwrap()
14671 }
14672
14673 #[tokio::test]
14674 async fn scope_describe_never_echoes_bind_proof_or_hash() {
14675 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
14676 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
14677 let mut rig = rig().await;
14678 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
14679 .await;
14680 let _bound = rig
14681 .bound(
14682 Some(AFT),
14683 PLEXUS,
14684 Some(
14685 rig.selector("s", Some(1))
14686 .with_bind_proof(Some(PROOF.into())),
14687 ),
14688 )
14689 .await;
14690 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
14691 let output = serde_json::to_string(&reply).unwrap();
14692 assert!(output.contains("\"status\":\"live\""));
14693 assert!(!output.contains(PROOF));
14694 assert!(!output.contains(HASH));
14695 }
14696
14697 async fn module(
14699 handler: &ControlHandler,
14700 connection: u64,
14701 module_id: &str,
14702 nonce: Option<&str>,
14703 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
14704 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
14705 hello_via_sink(
14706 handler,
14707 &ctx,
14708 &mut rx,
14709 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
14710 )
14711 .await;
14712 (ctx, rx)
14713 }
14714
14715 #[tokio::test]
14718 async fn a_client_connection_cannot_sync_or_describe() {
14719 let handler = ControlHandler::new(Arc::new(Registry::default()));
14720 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
14721 for request in [
14722 ModuleControlRequestFromModule::ScopeSync {
14723 generation: 1,
14724 scopes: vec![head("s", 1)],
14725 },
14726 ModuleControlRequestFromModule::ScopeDescribe {
14727 owner: Principal::Direct,
14728 scope_ref: "s".to_string(),
14729 },
14730 ] {
14731 let reply = call(&handler, &ctx, &request).await;
14732 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
14733 }
14734 assert!(
14735 !handler
14736 .scopes
14737 .read()
14738 .unwrap()
14739 .describe(
14740 &Principal::Reserved {
14741 module_id: OWNER.to_string()
14742 },
14743 "s"
14744 )
14745 .owner_synced
14746 );
14747 }
14748
14749 #[tokio::test]
14752 async fn a_module_without_a_supervised_launch_cannot_sync() {
14753 let handler = ControlHandler::new(Arc::new(Registry::default()));
14754 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
14755 assert_eq!(
14756 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
14757 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14758 );
14759 }
14760
14761 #[tokio::test]
14762 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
14763 let supervisor = SupervisorHandle::new();
14764 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14765 let handler = ControlHandler::new(Arc::new(Registry::default()))
14766 .with_supervisor(supervisor.clone());
14767 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
14768 sync(&handler, &incumbent, 1, vec![head("s", 1)])
14769 .await
14770 .expect("the current launch syncs");
14771
14772 supervisor.open_swap(OWNER, "n2".to_string());
14775 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
14776 assert_eq!(
14777 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
14778 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14779 );
14780 sync(&handler, &incumbent, 2, vec![head("s", 1)])
14781 .await
14782 .expect("the serving owner syncs during the swap");
14783
14784 supervisor.close_swap(OWNER);
14787 assert_eq!(
14788 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
14789 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14790 );
14791 sync(&handler, &incumbent, 3, vec![head("s", 1)])
14792 .await
14793 .expect("the serving owner syncs after the rollback");
14794 handler.cleanup_connection(candidate.connection_id).unwrap();
14795
14796 supervisor.open_swap(OWNER, "n3".to_string());
14801 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
14802 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
14803 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
14804 .await
14805 .expect("the promoted launch takes authority");
14806 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
14807 panic!("unexpected reply {reply:?}");
14808 };
14809 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
14810 assert_eq!(
14811 sync(&handler, &incumbent, 4, Vec::new()).await,
14812 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14813 );
14814 }
14815
14816 #[tokio::test]
14819 async fn closing_the_authority_connection_frees_sync_authority() {
14820 let supervisor = SupervisorHandle::new();
14821 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14822 let handler = ControlHandler::new(Arc::new(Registry::default()))
14823 .with_supervisor(supervisor.clone());
14824 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
14825 sync(&handler, &first, 10, vec![head("s", 1)])
14826 .await
14827 .unwrap();
14828 handler.cleanup_connection(first.connection_id).unwrap();
14829
14830 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
14831 sync(&handler, &second, 1, vec![head("s", 1)])
14832 .await
14833 .expect("the next connection takes the released authority");
14834 }
14835
14836 #[tokio::test]
14837 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
14838 let supervisor = SupervisorHandle::new();
14839 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14840 let handler =
14841 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
14842 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
14843 sync(&handler, &first, 10, vec![head("s", 1)])
14844 .await
14845 .unwrap();
14846 handler
14847 .handle_control_frame(
14848 &first,
14849 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
14850 )
14851 .await
14852 .unwrap();
14853 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
14854 sync(&handler, &second, 1, vec![head("s", 1)])
14855 .await
14856 .expect("GOODBYE releases authority even if the old socket remains open");
14857 }
14858
14859 #[tokio::test]
14860 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
14861 let registry = Arc::new(Registry::default());
14862 let supervisor_handle = SupervisorHandle::new();
14863 let supervisor =
14864 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
14865 .with_handle(supervisor_handle.clone())
14866 .with_daemon_incarnation("incarnation-7".to_string());
14867 supervisor
14870 .supervise_configured(
14871 ModuleSpec {
14872 module_id: OWNER.to_string(),
14873 program: PathBuf::from("/nonexistent/prefrontal-core"),
14874 args: Vec::new(),
14875 env: Vec::new(),
14876 reserved: false,
14877 reserved_prefixes: Vec::new(),
14878 protocol: ModuleProtocol::Subc,
14879 overlap: Default::default(),
14880 },
14881 false,
14882 )
14883 .unwrap();
14884 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
14885 let handler =
14886 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14887 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
14888
14889 let ModuleControlResponseToModule::ScopeDescribe {
14891 status,
14892 daemon_incarnation,
14893 owner_synced,
14894 owner_configured,
14895 scope,
14896 ..
14897 } = describe(&handler, &reader, OWNER, "s").await
14898 else {
14899 panic!("not a describe reply");
14900 };
14901 assert_eq!(status, ScopeStatus::NotLive);
14902 assert_eq!(daemon_incarnation, "incarnation-7");
14903 assert!(!owner_synced);
14904 assert!(owner_configured);
14905 assert!(scope.is_none());
14906
14907 let ModuleControlResponseToModule::ScopeDescribe {
14910 status,
14911 owner_configured,
14912 ..
14913 } = describe(&handler, &reader, "ghost", "s").await
14914 else {
14915 panic!("not a describe reply");
14916 };
14917 assert_eq!(status, ScopeStatus::NotLive);
14918 assert!(!owner_configured);
14919
14920 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
14922 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
14923 let ModuleControlResponseToModule::ScopeDescribe {
14924 status,
14925 scope_epoch,
14926 owner_synced,
14927 scope,
14928 ..
14929 } = describe(&handler, &reader, OWNER, "s").await
14930 else {
14931 panic!("not a describe reply");
14932 };
14933 assert_eq!(status, ScopeStatus::Live);
14934 assert_eq!(scope_epoch, Some(4));
14935 assert!(owner_synced);
14936 let stamp = scope.expect("a live scope carries its stamp");
14937 assert!(
14938 stamp.owner_authorized,
14939 "prefrontal-core is the default authority"
14940 );
14941 assert_eq!(stamp.kind, ScopeKind::Head);
14942 }
14943
14944 #[tokio::test]
14945 async fn scope_authority_owners_decides_owner_authorized() {
14946 let supervisor = SupervisorHandle::new();
14947 supervisor.set_spawn_nonce("broca", "b1".to_string());
14948 let handler = ControlHandler::new(Arc::new(Registry::default()))
14949 .with_supervisor(supervisor)
14950 .with_scope_authority_owners(vec!["broca".to_string()]);
14951 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
14952 let mut gated = head("s", 1);
14953 gated.attributes.agent_id = Some("agent".to_string());
14954 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
14955 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
14956 describe(&handler, &broca, "broca", "s").await
14957 else {
14958 panic!("not a describe reply");
14959 };
14960 assert!(scope.unwrap().owner_authorized);
14961 }
14962
14963 #[tokio::test]
14967 async fn scope_ops_and_the_scopes_capability_are_advertised() {
14968 let handler = ControlHandler::new(Arc::new(Registry::default()));
14969 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
14970 let ack = hello_via_sink(
14971 &handler,
14972 &ctx,
14973 &mut rx,
14974 hello_frame("m", PROTOCOL_VERSION, 1),
14975 )
14976 .await;
14977 let ack = parse_ack(&ack);
14978 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
14979 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
14980 }
14981 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
14982
14983 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
14984 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
14985 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
14986 let reply = handler
14987 .handle_control_frame(&client, frame)
14988 .await
14989 .unwrap()
14990 .pop()
14991 .unwrap();
14992 let ClientControlResponse::ServerDescribe { capabilities, .. } =
14993 serde_json::from_slice(&reply.body).unwrap()
14994 else {
14995 panic!("not a server.describe reply");
14996 };
14997 assert!(
14998 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
14999 "{capabilities:?}"
15000 );
15001 }
15002
15003 const PLEXUS: &str = "plexus";
15006 const OTHER: &str = "other";
15007 const AFT: &str = "aft";
15008 const BROCA: &str = "broca";
15009 const MAGIC: &str = "magic-context";
15010
15011 fn nonce(module_id: &str) -> String {
15012 format!("nonce-{module_id}")
15013 }
15014
15015 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15016 let (tx, rx) = mpsc::channel(64);
15017 (
15018 RouteCtx {
15019 connection_id: ConnectionId::new(connection),
15020 egress: FrameSink::new(tx),
15021 },
15022 rx,
15023 )
15024 }
15025
15026 struct Rig {
15030 handler: ControlHandler,
15031 forwarding: Arc<ForwardingTable>,
15032 owner: RouteCtx,
15033 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15034 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
15035 generation: u64,
15036 next_connection: u64,
15037 _supervisor: Supervisor,
15038 }
15039
15040 async fn rig() -> Rig {
15041 rig_with_flow_support(true).await
15042 }
15043
15044 async fn rig_with_flow_support(flow_support: bool) -> Rig {
15045 rig_with_scope_capabilities(flow_support, false).await
15046 }
15047
15048 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
15049 let mut provides = Vec::new();
15050 if flow_support {
15051 provides.push("flow-scopes/v1");
15052 }
15053 if run_support {
15054 provides.push("agent-run-scopes/v1");
15055 }
15056 rig_with_target_capabilities(&provides).await
15057 }
15058
15059 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
15060 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
15061 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
15062 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
15065 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
15066 Frame::build(
15067 FrameType::Hello,
15068 control_flags(),
15069 0,
15070 0,
15071 connection,
15072 serde_json::to_vec(&body).unwrap(),
15073 )
15074 .unwrap()
15075 }
15076
15077 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
15078 let registry = Arc::new(Registry::default());
15079 let forwarding = Arc::new(ForwardingTable::default());
15080 let supervisor_handle = SupervisorHandle::new();
15081 let supervisor =
15082 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
15083 .with_handle(supervisor_handle.clone());
15084 supervisor
15085 .supervise_configured(
15086 ModuleSpec {
15087 module_id: OWNER.to_string(),
15088 program: PathBuf::from("/nonexistent/prefrontal-core"),
15089 args: Vec::new(),
15090 env: Vec::new(),
15091 reserved: false,
15092 reserved_prefixes: Vec::new(),
15093 protocol: ModuleProtocol::Subc,
15094 overlap: Default::default(),
15095 },
15096 false,
15097 )
15098 .unwrap();
15099 for module_id in [OWNER, AFT, BROCA, MAGIC] {
15100 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
15101 }
15102 let handler =
15103 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15104 .with_supervisor(supervisor_handle);
15105 let (owner, mut owner_rx) = wide_ctx(1);
15106 hello_via_sink(
15107 &handler,
15108 &owner,
15109 &mut owner_rx,
15110 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
15111 )
15112 .await;
15113 let mut modules = BTreeMap::new();
15114 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
15115 let (ctx, mut rx) = wide_ctx(connection);
15116 let hello = hello_with_capabilities(module_id, connection, provides);
15117 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
15118 modules.insert(module_id.to_string(), (ctx, rx));
15119 }
15120 Rig {
15121 handler,
15122 forwarding,
15123 owner,
15124 _owner_rx: owner_rx,
15125 modules,
15126 generation: 0,
15127 next_connection: 100,
15128 _supervisor: supervisor,
15129 }
15130 }
15131
15132 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
15133 ScopeCarrier::new(Principal::Reserved {
15134 module_id: module_id.to_string(),
15135 })
15136 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
15137 }
15138
15139 fn session(scope_epoch: u64) -> ScopeRecord {
15142 let mut record = head("s", scope_epoch);
15143 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
15144 record.attributes.agent_id = Some("agent-1".to_string());
15145 record.attributes.delegates = true;
15146 record
15147 }
15148
15149 impl Rig {
15150 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
15151 self.generation += 1;
15152 sync(&self.handler, &self.owner, self.generation, scopes)
15153 .await
15154 .expect("the owner's sync is accepted");
15155 }
15156
15157 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15158 ScopeSelector::new(
15159 Principal::Reserved {
15160 module_id: OWNER.to_string(),
15161 },
15162 scope_ref.to_string(),
15163 scope_epoch,
15164 )
15165 }
15166
15167 fn open_frame(
15168 &mut self,
15169 opener: Option<&str>,
15170 target: &str,
15171 scope: Option<ScopeSelector>,
15172 ) -> (
15173 RouteCtx,
15174 mpsc::Receiver<crate::router::OutboundFrame>,
15175 Frame,
15176 ) {
15177 self.next_connection += 1;
15178 let (ctx, rx) = wide_ctx(self.next_connection);
15179 let root = unique_project_root("scoped-open");
15180 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
15181 target: RouteTarget::ToolProvider {
15182 module_id: target.to_string(),
15183 },
15184 identity: BindIdentity::new(
15185 root.path().to_path_buf(),
15186 "unit".to_string(),
15187 "session".to_string(),
15188 ),
15189 consumer_identity: opener.map(|module_id| ConsumerIdentity {
15190 module_id: module_id.to_string(),
15191 launch_nonce: nonce(module_id),
15192 }),
15193 consumer_capabilities: None,
15194 role_versions: None,
15195 admission_facts: None,
15196 scope,
15197 })
15198 .unwrap();
15199 let frame = Frame::build(
15200 FrameType::Request,
15201 control_flags(),
15202 0,
15203 0,
15204 self.next_connection,
15205 body,
15206 )
15207 .unwrap();
15208 (ctx, rx, frame)
15209 }
15210
15211 async fn refused(
15213 &mut self,
15214 opener: Option<&str>,
15215 target: &str,
15216 scope: Option<ScopeSelector>,
15217 ) -> String {
15218 self.refusal_body(opener, target, scope).await["code"]
15219 .as_str()
15220 .unwrap()
15221 .to_string()
15222 }
15223
15224 async fn refusal_body(
15225 &mut self,
15226 opener: Option<&str>,
15227 target: &str,
15228 scope: Option<ScopeSelector>,
15229 ) -> Value {
15230 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
15231 let replies = tokio::time::timeout(
15232 Duration::from_secs(2),
15233 self.handler.handle_control_frame(&ctx, frame),
15234 )
15235 .await
15236 .expect("the open must be refused before waiting for a bind ack")
15237 .unwrap();
15238 assert_eq!(replies.len(), 1, "{replies:?}");
15239 assert_eq!(replies[0].header.ty, FrameType::Error);
15240 let (_, module_rx) = self.modules.get_mut(target).unwrap();
15241 assert!(
15242 module_rx.try_recv().is_err(),
15243 "a refused open relays nothing"
15244 );
15245 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
15246 parse_error(&replies[0])
15247 }
15248
15249 async fn relayed(
15251 &mut self,
15252 opener: Option<&str>,
15253 target: &str,
15254 scope: Option<ScopeSelector>,
15255 ) -> Relayed {
15256 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
15257 let handler = self.handler.clone();
15258 let task_ctx = ctx.clone();
15259 let task = tokio::spawn(async move {
15260 handler
15261 .handle_control_frame(&task_ctx, frame)
15262 .await
15263 .unwrap()
15264 });
15265 let (_, module_rx) = self.modules.get_mut(target).unwrap();
15266 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
15267 .await
15268 .expect("the target receives the relayed route.bind")
15269 .unwrap()
15270 .frame;
15271 Relayed {
15272 target: target.to_string(),
15273 client: ctx,
15274 client_rx: rx,
15275 task,
15276 bind,
15277 }
15278 }
15279
15280 async fn ack(&self, relayed: &Relayed) {
15281 let (module, _) = &self.modules[&relayed.target];
15282 self.handler
15283 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
15284 .await
15285 .unwrap();
15286 }
15287
15288 async fn bound(
15290 &mut self,
15291 opener: Option<&str>,
15292 target: &str,
15293 scope: Option<ScopeSelector>,
15294 ) -> Bound {
15295 let relayed = self.relayed(opener, target, scope).await;
15296 self.ack(&relayed).await;
15297 let Relayed {
15298 target,
15299 client,
15300 mut client_rx,
15301 task,
15302 bind,
15303 } = relayed;
15304 assert!(
15305 task.await.unwrap().is_empty(),
15306 "the open is answered by commit"
15307 );
15308 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
15309 Bound {
15310 target,
15311 client,
15312 client_rx,
15313 channel,
15314 epoch,
15315 bind,
15316 }
15317 }
15318
15319 fn live(&self, route: &Bound) -> bool {
15320 matches!(
15321 self.forwarding
15322 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
15323 .unwrap(),
15324 DataRoute::Client(DataRouteState::Bound(_))
15325 )
15326 }
15327 }
15328
15329 struct Relayed {
15330 target: String,
15331 client: RouteCtx,
15332 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15333 task: tokio::task::JoinHandle<Vec<Frame>>,
15334 bind: Frame,
15335 }
15336
15337 struct Bound {
15338 target: String,
15339 client: RouteCtx,
15340 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15341 channel: u16,
15342 epoch: u32,
15343 bind: Frame,
15344 }
15345
15346 impl Bound {
15347 fn closed_reason(&mut self) -> RouteCloseReason {
15350 let mut reason = None;
15351 let mut goodbye = false;
15352 while let Ok(outbound) = self.client_rx.try_recv() {
15353 let frame = outbound.frame;
15354 match frame.header.ty {
15355 FrameType::Goodbye => {
15356 assert_eq!(
15357 (frame.header.channel, frame.header.epoch),
15358 (self.channel, self.epoch)
15359 );
15360 goodbye = true;
15361 }
15362 FrameType::Push => {
15363 let ClientControlPush::RouteClosed {
15364 reason: r,
15365 module_id,
15366 ..
15367 } = serde_json::from_slice(&frame.body).unwrap()
15368 else {
15369 panic!("unexpected push");
15370 };
15371 assert_eq!(module_id, self.target);
15372 reason = Some(r);
15373 }
15374 other => panic!("unexpected frame {other:?}"),
15375 }
15376 }
15377 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
15378 reason.expect("the client is told why the route closed")
15379 }
15380
15381 fn untouched(&mut self) -> bool {
15382 self.client_rx.try_recv().is_err()
15383 }
15384
15385 fn stamp(&self) -> Option<ScopeStamp> {
15386 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
15387 ModuleControlRequest::RouteBind { scope, .. } => scope,
15388 other => panic!("expected a route.bind, got {other:?}"),
15389 }
15390 }
15391 }
15392
15393 #[tokio::test]
15394 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
15395 ) {
15396 let mut rig = rig().await;
15397 let mut record = session(1);
15398 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
15399 record.child_owners = vec![Principal::Reserved {
15400 module_id: MAGIC.to_string(),
15401 }];
15402 rig.sync(vec![record]).await;
15403 let scope = || Some(rig_selector("s", Some(1)));
15404
15405 rig.bound(Some(OWNER), PLEXUS, scope()).await;
15408 rig.bound(Some(AFT), OTHER, scope()).await;
15409 rig.bound(Some(BROCA), PLEXUS, scope()).await;
15410
15411 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
15415 assert_eq!(
15416 rig.refused(opener, target, scope()).await,
15417 error_codes::SCOPE_NOT_CARRIER,
15418 "{opener:?} -> {target}"
15419 );
15420 }
15421 }
15422
15423 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15424 ScopeSelector::new(
15425 Principal::Reserved {
15426 module_id: OWNER.to_string(),
15427 },
15428 scope_ref.to_string(),
15429 scope_epoch,
15430 )
15431 }
15432
15433 #[tokio::test]
15434 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
15435 let mut rig = rig().await;
15436 rig.sync(vec![session(1)]).await;
15437 for opener in [OWNER, AFT] {
15438 assert_eq!(
15439 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
15440 .await,
15441 error_codes::SCOPE_EPOCH_REQUIRED,
15442 "{opener}"
15443 );
15444 }
15445 }
15446
15447 #[tokio::test]
15448 async fn admission_separates_not_synced_not_live_and_ended() {
15449 let mut rig = rig().await;
15450 let code = rig
15452 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15453 .await;
15454 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
15455 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
15456
15457 let ghost = ScopeSelector::new(
15459 Principal::Reserved {
15460 module_id: "ghost".to_string(),
15461 },
15462 "s".to_string(),
15463 Some(1),
15464 );
15465 assert_eq!(
15466 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
15467 error_codes::SCOPE_NOT_LIVE
15468 );
15469
15470 rig.sync(vec![session(2)]).await;
15471 assert_eq!(
15472 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
15473 .await,
15474 error_codes::SCOPE_NOT_LIVE
15475 );
15476 for epoch in [1, 3] {
15477 assert_eq!(
15478 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
15479 .await,
15480 error_codes::SCOPE_ENDED,
15481 "epoch {epoch}"
15482 );
15483 }
15484 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
15486 .await;
15487 }
15488
15489 #[tokio::test]
15490 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
15491 let mut rig = rig().await;
15492 rig.sync(vec![session(1)]).await;
15493 let route = rig
15494 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15495 .await;
15496 let stamp = route.stamp().expect("a scoped bind carries the stamp");
15497 assert_eq!(stamp.scope_ref, "s");
15498 assert_eq!(stamp.scope_epoch, 1);
15499 assert_eq!(stamp.kind, ScopeKind::Head);
15500 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
15501 assert!(stamp.attributes.delegates);
15502 assert!(stamp.owner_authorized);
15503 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15504 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
15505
15506 let (broca, mut broca_rx) = wide_ctx(50);
15509 hello_via_sink(
15510 &rig.handler,
15511 &broca,
15512 &mut broca_rx,
15513 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
15514 )
15515 .await;
15516 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
15517 .await
15518 .unwrap();
15519 let own = ScopeSelector::new(
15520 Principal::Reserved {
15521 module_id: BROCA.to_string(),
15522 },
15523 "b".to_string(),
15524 Some(1),
15525 );
15526 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
15527 assert!(!route.stamp().unwrap().owner_authorized);
15528 }
15529
15530 #[tokio::test]
15531 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
15532 let mut rig = rig().await;
15533 let mut record = head("s", 1);
15534 record.carriers = vec![carrier(AFT, None)];
15535 let flow_id = "Flow:run-7/step_2!~";
15536 record.attributes.flow_id = Some(flow_id.to_string());
15537 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
15538 .await
15539 .unwrap();
15540 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
15541 panic!("not a sync reply");
15542 };
15543 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
15544 let route = rig
15545 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15546 .await;
15547 assert!(rig.live(&route), "the stamped bind committed");
15548 let stamp = route.stamp().expect("a flow scope carries a stamp");
15549 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
15550 assert_eq!(stamp.attributes.agent_id, None);
15551 assert!(!stamp.attributes.delegates);
15552 assert!(stamp.owner_authorized);
15553 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15554 assert_eq!(unscoped.stamp(), None);
15555 }
15556
15557 #[tokio::test]
15558 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
15559 let mut rig = rig_with_flow_support(false).await;
15560 let mut record = session(1);
15561 record.attributes.flow_id = Some("flow:7".to_string());
15562 rig.sync(vec![record]).await;
15563 for opener in [OWNER, AFT] {
15564 let body = rig
15565 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
15566 .await;
15567 assert_eq!(body["code"], "target_flow_unsupported");
15568 let message = body["message"].as_str().unwrap();
15569 for required in [PLEXUS, "flow-scopes/v1"] {
15570 assert!(message.contains(required), "{message}");
15571 }
15572 }
15573 }
15574
15575 #[tokio::test]
15576 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
15577 let mut rig = rig_with_flow_support(true).await;
15578 let mut record = session(1);
15579 record.attributes.flow_id = Some("flow:7".to_string());
15580 rig.sync(vec![record]).await;
15581 for opener in [OWNER, AFT] {
15582 let route = rig
15583 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
15584 .await;
15585 assert!(rig.live(&route));
15586 assert_eq!(
15587 route.stamp().unwrap().attributes.flow_id.as_deref(),
15588 Some("flow:7")
15589 );
15590 }
15591 }
15592
15593 #[tokio::test]
15594 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
15595 use std::future::Future;
15596
15597 let mut rig = rig_with_flow_support(true).await;
15598 let mut record = session(1);
15599 record.attributes.flow_id = Some("flow:7".to_string());
15600 rig.sync(vec![record]).await;
15601 let (client, mut client_rx, frame) =
15602 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
15603 for _ in 0..64 {
15606 client.egress.try_send(route_bind_ack(1)).unwrap();
15607 }
15608 let handler = rig.handler.clone();
15609 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15610 std::future::poll_fn(|cx| {
15611 assert!(open.as_mut().poll(cx).is_pending());
15612 std::task::Poll::Ready(())
15613 })
15614 .await;
15615 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15616
15617 let old_connection = rig.modules[PLEXUS].0.connection_id;
15618 rig.handler.cleanup_connection(old_connection).unwrap();
15619 let (replacement, mut replacement_rx) = wide_ctx(200);
15620 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
15621 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
15622 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
15623 let hello = Frame::build(
15624 FrameType::Hello,
15625 control_flags(),
15626 0,
15627 0,
15628 200,
15629 serde_json::to_vec(&body).unwrap(),
15630 )
15631 .unwrap();
15632 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
15633
15634 client_rx.try_recv().unwrap();
15635 let replies = tokio::time::timeout(Duration::from_secs(2), open)
15636 .await
15637 .expect("the replacement is refused without waiting for a bind ack")
15638 .unwrap();
15639 assert_eq!(replies.len(), 1);
15640 let body = parse_error(&replies[0]);
15641 assert_eq!(body["code"], "target_flow_unsupported");
15642 for required in [PLEXUS, "flow-scopes/v1"] {
15643 assert!(body["message"].as_str().unwrap().contains(required));
15644 }
15645 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
15646 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
15647 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15648 assert!(rig
15649 .handler
15650 .registry
15651 .get_module_by_connection(replacement.connection_id)
15652 .unwrap()
15653 .is_some());
15654 }
15655
15656 #[tokio::test]
15657 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
15658 use std::future::Future;
15659 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
15660 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15661 const NEXT_HASH: &str =
15662 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
15663 let mut rig = rig().await;
15664 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
15665 record.carriers.push(ScopeCarrier::new(Principal::Direct));
15666 rig.sync(vec![record.clone()]).await;
15667 let (client, mut rx, frame) = rig.open_frame(
15668 None,
15669 PLEXUS,
15670 Some(
15671 rig.selector("s", Some(1))
15672 .with_bind_proof(Some(PROOF.into())),
15673 ),
15674 );
15675 for _ in 0..64 {
15678 client.egress.try_send(route_bind_ack(1)).unwrap();
15679 }
15680 let handler = rig.handler.clone();
15681 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15682 std::future::poll_fn(|cx| {
15683 assert!(open.as_mut().poll(cx).is_pending());
15684 std::task::Poll::Ready(())
15685 })
15686 .await;
15687 record.bind_proof_sha256 = Some(NEXT_HASH.into());
15688 rig.sync(vec![record]).await;
15689 rx.try_recv().unwrap();
15690 let replies = tokio::time::timeout(Duration::from_secs(2), open)
15691 .await
15692 .unwrap()
15693 .unwrap();
15694 assert_eq!(
15695 parse_error(&replies[0])["code"],
15696 error_codes::SCOPE_BIND_PROOF_MISMATCH
15697 );
15698 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
15699 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
15700 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15701 }
15702
15703 #[tokio::test]
15704 async fn scope_bind_proof_reserved_openers_ignore_proof() {
15705 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15706 let mut rig = rig().await;
15707 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15708 .await;
15709 for opener in [OWNER, AFT] {
15710 for proof in [None, Some("malformed+proof=")] {
15711 let route = rig
15712 .bound(
15713 Some(opener),
15714 PLEXUS,
15715 Some(
15716 rig.selector("s", Some(1))
15717 .with_bind_proof(proof.map(str::to_string)),
15718 ),
15719 )
15720 .await;
15721 assert!(rig.live(&route));
15722 }
15723 }
15724 }
15725
15726 #[tokio::test]
15727 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
15728 use std::future::Future;
15729 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15730 const NEXT_HASH: &str =
15731 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
15732 let mut rig = rig().await;
15733 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15734 .await;
15735 let (client, mut rx, frame) =
15736 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
15737 for _ in 0..64 {
15738 client.egress.try_send(route_bind_ack(1)).unwrap();
15739 }
15740 let handler = rig.handler.clone();
15741 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15742 std::future::poll_fn(|cx| {
15743 assert!(open.as_mut().poll(cx).is_pending());
15744 std::task::Poll::Ready(())
15745 })
15746 .await;
15747 rig.sync(vec![
15748 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
15749 ])
15750 .await;
15751 rx.try_recv().unwrap();
15752 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15753 let bind = std::future::poll_fn(|cx| {
15754 assert!(open.as_mut().poll(cx).is_pending());
15755 module_rx.poll_recv(cx)
15756 })
15757 .await
15758 .unwrap()
15759 .frame;
15760 handler
15761 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
15762 .await
15763 .unwrap();
15764 let replies = tokio::time::timeout(Duration::from_secs(2), open)
15765 .await
15766 .unwrap()
15767 .unwrap();
15768 assert_eq!(replies.len(), 1);
15769 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
15772 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15773 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
15774 let route = rig
15775 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
15776 .await;
15777 assert!(rig.live(&route));
15778 }
15779
15780 #[tokio::test]
15781 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
15782 {
15783 let mut rig = rig_with_flow_support(false).await;
15784 rig.sync(vec![session(1)]).await;
15785 let route = rig
15786 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15787 .await;
15788 assert!(rig.live(&route));
15789 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
15790 let mut record = session(1);
15791 record.attributes.flow_id = Some("flow:7".to_string());
15792 rig.sync(vec![record]).await;
15793 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
15794 assert!(rig.live(&unscoped));
15795 assert_eq!(unscoped.stamp(), None);
15796 }
15797
15798 #[tokio::test]
15799 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
15800 let mut rig = rig().await;
15801 let mut record = session(1);
15802 record.attributes.flow_id = Some("flow:7".to_string());
15803 rig.sync(vec![record.clone()]).await;
15804 let before = rig.forwarding.published_scope_tag(OWNER, "s");
15805 let mut owner_route = rig
15806 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
15807 .await;
15808 let mut carrier_route = rig
15809 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
15810 .await;
15811 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15812 rig.sync(vec![record.clone()]).await;
15813 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
15814 assert!(rig.live(&owner_route) && owner_route.untouched());
15815 assert!(rig.live(&carrier_route) && carrier_route.untouched());
15816
15817 record.attributes.flow_id = Some("flow:8".to_string());
15818 rig.sync(vec![record]).await;
15819 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
15820 let before = before.unwrap();
15821 assert_eq!(after.scope_epoch, before.scope_epoch);
15822 assert!(after.version > before.version);
15823 for route in [&mut owner_route, &mut carrier_route] {
15824 assert!(!rig.live(route));
15825 assert_eq!(
15826 route.closed_reason(),
15827 RouteCloseReason::ScopeDelegationChanged
15828 );
15829 }
15830 assert!(rig.live(&unscoped) && unscoped.untouched());
15831 for target in [PLEXUS, OTHER] {
15834 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
15835 let goodbye = module_rx
15836 .try_recv()
15837 .expect("the provider sees the drain")
15838 .frame;
15839 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15840 assert!(module_rx.try_recv().is_err());
15841 }
15842 let rebound = rig
15843 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15844 .await;
15845 assert_eq!(
15846 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
15847 Some("flow:8")
15848 );
15849 }
15850
15851 #[tokio::test]
15856 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
15857 let mut rig = rig().await;
15858 rig.sync(vec![session(1)]).await;
15859 let mut cotenant = rig
15860 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
15861 .await;
15862
15863 let mut changed = session(1);
15864 changed.child_owners.push(Principal::Reserved {
15865 module_id: MAGIC.to_string(),
15866 });
15867 let mut ended = None;
15868 for (code, next) in [
15869 (error_codes::SCOPE_CHANGED, vec![changed]),
15870 (error_codes::SCOPE_ENDED, Vec::new()),
15871 ] {
15872 let relayed = rig
15873 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15874 .await;
15875 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
15876 ended = Some(next.is_empty());
15877 rig.sync(next).await;
15878 rig.ack(&relayed).await;
15879 let replies = relayed.task.await.unwrap();
15880 assert_eq!(replies.len(), 1, "{replies:?}");
15881 assert_eq!(parse_error(&replies[0])["code"], code);
15882 assert_eq!(
15883 subc_protocol::error_codes::is_retryable_route_open(code),
15884 code == error_codes::SCOPE_CHANGED
15885 );
15886 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15887 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15889 let mut goodbyes = Vec::new();
15892 while let Ok(outbound) = plexus_rx.try_recv() {
15893 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
15894 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
15895 }
15896 assert!(
15897 goodbyes.contains(&(bind_channel, bind_epoch)),
15898 "{goodbyes:?}"
15899 );
15900 assert!(rig
15901 .handler
15902 .registry
15903 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
15904 .unwrap()
15905 .is_some());
15906 }
15907 assert_eq!(ended, Some(true));
15908 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
15911 }
15912
15913 #[tokio::test]
15916 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
15917 struct Case {
15918 name: &'static str,
15919 change: fn(&mut ScopeRecord),
15920 closed: [Option<RouteCloseReason>; 4],
15923 }
15924 use RouteCloseReason::*;
15925 let cases = [
15926 Case {
15927 name: "a carrier entry removed",
15928 change: |r| {
15929 r.carriers.retain(|c| {
15930 c.principal
15931 != Principal::Reserved {
15932 module_id: AFT.to_string(),
15933 }
15934 })
15935 },
15936 closed: [None, Some(ScopeCarrierRemoved), None, None],
15937 },
15938 Case {
15939 name: "a target removed from a carrier",
15940 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
15941 closed: [None, None, None, Some(ScopeCarrierRemoved)],
15942 },
15943 Case {
15944 name: "a bare carrier narrowed to targets",
15945 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
15946 closed: [None, Some(ScopeCarrierRemoved), None, None],
15947 },
15948 Case {
15949 name: "delegates turned off",
15950 change: |r| r.attributes.delegates = false,
15951 closed: [Some(ScopeDelegationChanged); 4],
15952 },
15953 Case {
15954 name: "agent_id changed",
15955 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
15956 closed: [Some(ScopeDelegationChanged); 4],
15957 },
15958 Case {
15959 name: "a carrier added, child owners changed, the record re-sent",
15960 change: |r| {
15961 r.carriers.push(carrier(MAGIC, None));
15962 r.child_owners.push(Principal::Reserved {
15963 module_id: MAGIC.to_string(),
15964 });
15965 },
15966 closed: [None; 4],
15967 },
15968 Case {
15969 name: "a target added",
15970 change: |r| {
15971 r.carriers[1]
15972 .targets
15973 .as_mut()
15974 .unwrap()
15975 .push("third".to_string())
15976 },
15977 closed: [None; 4],
15978 },
15979 Case {
15980 name: "delegates turned on",
15981 change: |r| r.attributes.delegates = true,
15982 closed: [None; 4],
15983 },
15984 ];
15985 for case in cases {
15986 let mut rig = rig().await;
15987 rig.sync(vec![session(1)]).await;
15988 let scope = || Some(rig_selector("s", Some(1)));
15989 let mut routes = [
15990 rig.bound(Some(OWNER), PLEXUS, scope()).await,
15991 rig.bound(Some(AFT), PLEXUS, scope()).await,
15992 rig.bound(Some(BROCA), PLEXUS, scope()).await,
15993 rig.bound(Some(BROCA), OTHER, scope()).await,
15994 ];
15995 let mut record = session(1);
15996 (case.change)(&mut record);
15997 rig.sync(vec![record]).await;
15998 for (index, expected) in case.closed.iter().enumerate() {
15999 let route = &mut routes[index];
16000 match expected {
16001 Some(reason) => {
16002 assert!(!rig.live(route), "{}: route {index} still live", case.name);
16003 assert_eq!(
16004 route.closed_reason(),
16005 *reason,
16006 "{}: route {index}",
16007 case.name
16008 );
16009 }
16010 None => {
16011 assert!(rig.live(route), "{}: route {index} closed", case.name);
16012 assert!(
16013 route.untouched(),
16014 "{}: route {index} was told something",
16015 case.name
16016 );
16017 }
16018 }
16019 }
16020 }
16021 }
16022
16023 #[tokio::test]
16024 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
16025 for next in [Vec::new(), vec![session(2)]] {
16027 let mut rig = rig().await;
16028 rig.sync(vec![session(1)]).await;
16029 let mut route = rig
16030 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16031 .await;
16032 rig.sync(next).await;
16033 assert!(!rig.live(&route));
16034 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
16035 }
16036
16037 let mut rig = rig().await;
16040 let mut child = session(1);
16041 child.scope_ref = "child".to_string();
16042 child.kind = ScopeKind::Worker;
16043 child.parent = Some(ScopeParent::new(
16044 Principal::Reserved {
16045 module_id: OWNER.to_string(),
16046 },
16047 "s".to_string(),
16048 1,
16049 ));
16050 rig.sync(vec![session(1), child.clone()]).await;
16051 let mut child_route = rig
16052 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
16053 .await;
16054 assert_eq!(
16055 child_route.stamp().unwrap().parent_state,
16056 Some(ParentState::Linked)
16057 );
16058 rig.sync(vec![child]).await;
16059 assert!(!rig.live(&child_route));
16060 assert_eq!(
16061 child_route.closed_reason(),
16062 RouteCloseReason::ScopeParentEnded
16063 );
16064 }
16065
16066 #[tokio::test]
16067 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
16068 ) {
16069 let mut rig = rig().await;
16070 rig.sync(vec![session(1)]).await;
16071 let mut route = rig
16072 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16073 .await;
16074 let before = rig.forwarding.published_scope_tag(OWNER, "s");
16075 rig.sync(vec![session(1)]).await;
16076 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
16077 assert!(rig.live(&route) && route.untouched());
16078
16079 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
16083 .forwarding
16084 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16085 .unwrap()
16086 else {
16087 panic!("the route is bound");
16088 };
16089 binding.flow.acquire_tagged(9, false).await.unwrap();
16090 let mut widened = session(1);
16091 widened.carriers.push(carrier(MAGIC, None));
16092 rig.sync(vec![widened]).await;
16093 assert!(rig.live(&route) && route.untouched());
16094 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16095 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
16096 assert_eq!(binding.flow.in_flight(), 1);
16099 binding
16100 .flow
16101 .acquire_tagged(10, false)
16102 .await
16103 .expect("the flow is still open");
16104 }
16105
16106 #[tokio::test]
16109 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
16110 let mut rig = rig().await;
16111 rig.sync(vec![session(1)]).await;
16112 let mut on_incumbent = rig
16113 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16114 .await;
16115
16116 let (candidate, _candidate_rx) = wide_ctx(9);
16119 let registration = rig
16120 .handler
16121 .registry
16122 .register_candidate_with_control_ops(
16123 manifest(PLEXUS, PROTOCOL_VERSION),
16124 PROTOCOL_VERSION,
16125 candidate.connection_id,
16126 module_baseline_control_ops(),
16127 )
16128 .unwrap();
16129 rig.forwarding
16130 .register_candidate_module_connection(
16131 candidate.connection_id,
16132 PLEXUS.to_string(),
16133 PROTOCOL_VERSION,
16134 manifest_concurrency(®istration.manifest),
16135 candidate.egress.clone(),
16136 )
16137 .unwrap();
16138 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
16139 rig.handler
16140 .registry
16141 .promote_candidate(PLEXUS)
16142 .unwrap()
16143 .unwrap();
16144 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
16145
16146 rig.sync(Vec::new()).await;
16147 assert!(!rig.live(&on_incumbent));
16148 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
16149 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16150 let goodbye = incumbent_rx
16151 .try_recv()
16152 .expect("the superseded endpoint is told")
16153 .frame;
16154 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16155 }
16156 }
16157}
16158
16159#[cfg(test)]
16160mod concurrency_default_exposure_tests {
16161 use super::*;
16162
16163 fn hello_body(role_json: &str) -> Vec<u8> {
16164 format!(
16165 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
16166 )
16167 .into_bytes()
16168 }
16169
16170 fn manifest_from(body: &[u8]) -> ModuleManifest {
16171 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
16172 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
16173 .expect("manifest parses")
16174 }
16175
16176 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
16177
16178 #[test]
16179 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
16180 let body = hello_body(&format!(
16181 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
16182 ));
16183 let manifest = manifest_from(&body);
16184 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16187 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
16188 }
16189
16190 #[test]
16191 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
16192 let body = hello_body(&format!(
16193 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
16194 ));
16195 let manifest = manifest_from(&body);
16196 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16197 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16198 }
16199
16200 #[test]
16201 fn non_management_roles_are_never_reported() {
16202 let body = hello_body(
16203 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
16204 );
16205 let manifest = manifest_from(&body);
16206 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16207 }
16208}