1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_event_declarations,
25 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
26 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
27 },
28 scope::{
29 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
30 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
31 SCOPE_SYNC_OP,
32 },
33 session::{
34 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
35 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
36 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
37 ROLE_VERSIONS_FIELD,
38 },
39 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
40 Priority, RouteTarget, PROTOCOL_VERSION,
41};
42use tokio::time::{timeout_at, Instant};
43use tracing::{debug, info, warn};
44
45use crate::{
46 caller_identity::CallerIdentity,
47 capability_requirements::{
48 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
49 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
50 RuntimeModule,
51 },
52 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
53 forwarding::{
54 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
55 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
56 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
57 },
58 observability::{
59 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
60 },
61 privileged_prompt::{prompt_text, PrivilegedOperation},
62 provenance::{
63 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
64 },
65 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
66 router::{RouteCtx, RouterError},
67 scopes::{HelloLaunchNonces, ScopeTable},
68 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
69 stderr_tail::{CaptureState, TailEntry},
70 supervise::{
71 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
72 SupervisorHandle, SwapHelloAdmission,
73 },
74 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
75};
76
77pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
83
84const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
85const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
86const CAP_PING_PONG: &str = "ping_pong_v1";
87const CAP_SESSION_ATTACH: &str = "session_attach_v1";
88const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
89
90const SUBC_CONTROL_OPS: &[&str] = &[
91 ops::ACTIVITY_RECORD,
92 ops::ACTIVITY_QUERY,
93 ops::ACTIVITY_SUBSCRIBE,
94 ops::SERVER_DESCRIBE,
95 ops::CATALOG_LIST,
96 ops::ROUTE_OPEN,
97 ops::ROUTE_POLL,
98 ops::ROUTE_CLOSING,
99 ops::ROUTE_CLOSED,
100 ops::SUPERVISOR_LIST,
101 ops::SUPERVISOR_RESTART,
102 ops::SUPERVISOR_SWAP,
103 ops::SUPERVISOR_RELOAD,
104 ops::SUPERVISOR_RESCAN,
105 ops::SUPERVISOR_RELEASE_RESERVED,
106 ops::SUPERVISOR_SET_ENABLED,
107 ops::SUPERVISOR_HEALTH_PROBE,
108 ops::SUPERVISOR_HEALTH,
109 ops::SUPERVISOR_STDERR_TAIL,
110 ops::SUPERVISOR_TERMINALS,
111 ops::SUPERVISOR_ROUTES,
112 ops::SUPERVISOR_PROVENANCE,
113 ops::SUPERVISOR_SPAWN_SNAPSHOT,
114 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
115];
116
117const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
118 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
119 "supervisor.live_roots",
120 SCOPE_SYNC_OP,
121 SCOPE_APPLY_OP,
122 SCOPE_DESCRIBE_OP,
123 "operator.confirm",
124];
125
126const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
130
131const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
132
133pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
141
142pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
157
158pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
176
177const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
178const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
179
180fn reload_verdict(
181 configured: &Path,
182 spawned_from: Option<&Path>,
183 image: subc_control::RunningImageAgreement,
184) -> PendingReloadVerdict {
185 let path = match spawned_from {
186 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
187 ReloadPathAgreement::Match
188 }
189 Some(spawned_from) => ReloadPathAgreement::Mismatch {
190 configured: configured.to_path_buf(),
191 spawned_from: spawned_from.to_path_buf(),
192 },
193 None => ReloadPathAgreement::Unavailable {
194 reason: if matches!(
195 image,
196 subc_control::RunningImageAgreement::Unavailable {
197 reason: subc_control::RunningImageUnavailableReason::NotRunning
198 }
199 ) {
200 ReloadPathUnavailableReason::NotRunning
201 } else {
202 ReloadPathUnavailableReason::SpawnedPathUnavailable
203 },
204 },
205 };
206 PendingReloadVerdict { path, image }
207}
208
209fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
210 if configured == spawned_from {
211 return true;
212 }
213 #[cfg(windows)]
214 {
215 match (
221 std::fs::canonicalize(configured),
222 std::fs::canonicalize(spawned_from),
223 ) {
224 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
225 _ => false,
226 }
227 }
228 #[cfg(not(windows))]
229 {
230 false
231 }
232}
233
234#[derive(Clone)]
235struct DaemonProvenanceFacts {
236 build: DaemonBuildProvenance,
237 pid: Option<u32>,
238 started_at_ms: Option<u64>,
239 start_clock: Option<crate::clock::StartClock>,
240 executable_path: Option<PathBuf>,
241 executable_identity: Option<SpawnedFileIdentity>,
242 process_start_time: Option<u64>,
243 probe: ExecutableIdentityProbe,
244}
245
246impl Default for DaemonProvenanceFacts {
247 fn default() -> Self {
248 Self {
249 build: DaemonBuildProvenance {
250 build_git_sha: None,
251 build_lock_digest: None,
252 },
253 pid: None,
254 started_at_ms: None,
255 start_clock: None,
256 executable_path: None,
257 executable_identity: None,
258 process_start_time: None,
259 probe: ExecutableIdentityProbe::default(),
260 }
261 }
262}
263
264#[derive(Debug, Clone)]
265struct SupervisorRescanContext {
266 supervisor: Supervisor,
267 config_path: PathBuf,
268 configured_port: Option<u16>,
269 storage_config: Option<crate::daemon_config::StorageConfig>,
270 admission_facts_carrier_module_id: Option<String>,
271 admission_facts_targets: Option<Vec<String>>,
272 scope_authority_owners: Vec<String>,
273 privileged_operator_prompt: PrivilegedOperatorPrompt,
274}
275
276const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
297 "reloading",
298 "supervisor_not_live",
299 "registration_not_active",
300 "no_forwarding_connection",
301 "relay_send_failed",
302];
303
304#[derive(Clone)]
306pub struct ControlHandler {
307 registry: Arc<Registry>,
308 activity: Option<crate::activity::ActivityLedger>,
309 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
310 forwarding: Arc<ForwardingTable>,
311 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
312 supervisor: SupervisorHandle,
313 subc_capabilities: Arc<[String]>,
314 route_bind_relay_timeout: Duration,
318 route_bind_relay_timeouts: BTreeMap<String, Duration>,
322 route_bind_breakers: RouteBindBreakers,
325 route_bind_concurrency: RouteBindConcurrency,
328 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
333 route_bind_breaker_threshold: u32,
335 route_bind_breaker_cooldown: Duration,
337 health_probe_timeout: Duration,
338 storage_config: Option<crate::daemon_config::StorageConfig>,
341 machine_id: Option<crate::machine_id::MachineId>,
345 admission_facts_carrier_module_id: Option<String>,
346 admission_facts_targets: Option<Vec<String>>,
347 scopes: Arc<RwLock<ScopeTable>>,
351 scope_deadline_added: Arc<tokio::sync::Notify>,
355 scope_authority_owners: Vec<String>,
358 privileged_operator_prompt: PrivilegedOperatorPrompt,
360 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
363 #[cfg(test)]
364 caller_lookup: Option<TestCallerLookup>,
365 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
368 rescan: Option<SupervisorRescanContext>,
369 connected_clients: ConnectedClients,
370 counters: DaemonCounters,
371 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
372 daemon_provenance: DaemonProvenanceFacts,
373 #[cfg(test)]
374 control_dispatch_delay: Option<Duration>,
375 #[cfg(test)]
376 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
377 #[cfg(test)]
381 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
382 #[cfg(test)]
388 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
389}
390
391impl fmt::Debug for ControlHandler {
392 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
393 f.debug_struct("ControlHandler")
394 .field("registry", &self.registry)
395 .field("forwarding", &self.forwarding)
396 .field("process_liveness", &self.process_liveness.is_some())
397 .field("supervisor", &self.supervisor)
398 .field("subc_capabilities", &self.subc_capabilities)
399 .finish()
400 }
401}
402
403struct RouteOpenRequest {
404 target: RouteTarget,
405 identity: BindIdentity,
406 consumer_identity: Option<ConsumerIdentity>,
407 consumer_capabilities: Option<Vec<String>>,
408 role_versions: Option<BTreeMap<String, String>>,
409 admission_facts: Option<serde_json::Value>,
410 scope: Option<ScopeSelector>,
411}
412
413#[derive(Clone, Copy)]
414enum RouteOpenPhase {
415 Admission,
416 ProjectRoot,
417 EgressReservation,
418 BindSend,
419 BindAckWait,
420}
421
422struct RouteOpenTimings {
426 module_id: String,
427 principal: String,
428 admission: Duration,
429 project_root: Duration,
430 egress_reservation: Duration,
431 bind_send: Duration,
432 bind_ack_wait: Duration,
433 phase: Option<RouteOpenPhase>,
434 phase_started_at: Instant,
435}
436
437impl RouteOpenTimings {
438 fn new() -> Self {
439 Self {
440 module_id: String::new(),
441 principal: "unresolved".to_string(),
443 admission: Duration::ZERO,
444 project_root: Duration::ZERO,
445 egress_reservation: Duration::ZERO,
446 bind_send: Duration::ZERO,
447 bind_ack_wait: Duration::ZERO,
448 phase: Some(RouteOpenPhase::Admission),
449 phase_started_at: Instant::now(),
450 }
451 }
452
453 fn finish(&mut self) {
454 let elapsed = self.phase_started_at.elapsed();
455 match self.phase.take() {
456 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
457 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
458 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
459 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
460 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
461 None => {}
462 }
463 }
464
465 fn enter(&mut self, phase: RouteOpenPhase) {
466 self.finish();
467 self.phase = Some(phase);
468 self.phase_started_at = Instant::now();
469 }
470}
471
472struct RouteBindReservationGuard {
473 forwarding: Arc<ForwardingTable>,
474 endpoint: ModuleEndpointId,
475 relay_corr: u64,
476 armed: bool,
477}
478
479struct ModuleControlRpcGuard {
480 forwarding: Arc<ForwardingTable>,
481 endpoint: ModuleEndpointId,
482 corr: u64,
483 armed: bool,
484}
485
486impl ModuleControlRpcGuard {
487 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
488 Self {
489 forwarding,
490 endpoint,
491 corr,
492 armed: true,
493 }
494 }
495
496 fn disarm(&mut self) {
497 self.armed = false;
498 }
499}
500
501impl Drop for ModuleControlRpcGuard {
502 fn drop(&mut self) {
503 if self.armed {
504 let _ = self
505 .forwarding
506 .cancel_module_control_rpc(self.endpoint, self.corr);
507 }
508 }
509}
510
511impl RouteBindReservationGuard {
512 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
513 Self {
514 forwarding,
515 endpoint,
516 relay_corr,
517 armed: true,
518 }
519 }
520
521 fn release_and_disarm(&mut self) {
522 if !self.armed {
523 return;
524 }
525 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
526 self.endpoint,
527 self.relay_corr,
528 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
529 ) {
530 send_goodbye_target_best_effort(
531 &self.forwarding.counters(),
532 &target,
533 "canceled route.bind",
534 );
535 }
536 self.armed = false;
537 }
538
539 fn disarm(&mut self) {
540 self.armed = false;
541 }
542}
543
544impl Drop for RouteBindReservationGuard {
545 fn drop(&mut self) {
546 self.release_and_disarm();
547 }
548}
549
550#[derive(Debug, Clone, Default)]
580pub(crate) struct RouteBindBreakers {
581 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
582}
583
584#[derive(Debug, Clone, Default)]
585pub(crate) struct RouteBindConcurrency {
586 modules: Arc<Mutex<HashMap<String, usize>>>,
587}
588
589struct RouteBindConcurrencyGuard {
590 concurrency: RouteBindConcurrency,
591 module_id: String,
592}
593
594impl RouteBindConcurrency {
595 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
598 let mut modules = self
599 .modules
600 .lock()
601 .expect("route.bind concurrency mutex poisoned");
602 let in_flight = modules.entry(module_id.to_string()).or_default();
603 if *in_flight >= limit {
604 return Err(*in_flight);
605 }
606 *in_flight += 1;
607 Ok(RouteBindConcurrencyGuard {
608 concurrency: self.clone(),
609 module_id: module_id.to_string(),
610 })
611 }
612}
613
614impl Drop for RouteBindConcurrencyGuard {
615 fn drop(&mut self) {
616 let mut modules = self
617 .concurrency
618 .modules
619 .lock()
620 .expect("route.bind concurrency mutex poisoned");
621 let remove = {
622 let in_flight = modules
623 .get_mut(&self.module_id)
624 .expect("admitted route.bind has a concurrency entry");
625 *in_flight -= 1;
626 *in_flight == 0
627 };
628 if remove {
629 modules.remove(&self.module_id);
630 }
631 }
632}
633
634#[derive(Debug, Default)]
635struct ModuleBreakerState {
636 consecutive_timeouts: u32,
638 cooldown_until: Option<Instant>,
641 probe_in_flight: Option<Arc<()>>,
646}
647
648enum RouteBindAdmission<'a> {
650 Admitted {
651 guard: RouteBindBreakerGuard<'a>,
652 probe: bool,
655 },
656 Refused {
657 consecutive_timeouts: u32,
658 retry_in: Duration,
662 probe_in_flight: bool,
663 },
664}
665
666struct RouteBindBreakerGuard<'a> {
673 breakers: RouteBindBreakers,
674 module_id: &'a str,
675 probe_token: Option<Arc<()>>,
676 settled: bool,
677}
678
679impl RouteBindBreakerGuard<'_> {
680 fn record_accepted(&mut self) -> bool {
684 self.settled = true;
685 self.breakers.record_accepted(self.module_id)
686 }
687
688 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
691 self.settled = true;
692 self.breakers.record_timeout(
693 self.module_id,
694 self.probe_token.as_ref(),
695 threshold,
696 cooldown,
697 )
698 }
699
700 fn record_inconclusive(&mut self) {
709 self.settled = true;
710 self.breakers
711 .record_inconclusive(self.module_id, self.probe_token.as_ref());
712 }
713}
714
715impl Drop for RouteBindBreakerGuard<'_> {
716 fn drop(&mut self) {
717 if !self.settled {
718 self.breakers
719 .record_inconclusive(self.module_id, self.probe_token.as_ref());
720 }
721 }
722}
723
724struct BreakerOpened {
728 consecutive_timeouts: u32,
729 reopened_after_probe: bool,
732}
733
734impl RouteBindBreakers {
735 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
736 self.modules
737 .lock()
738 .expect("route.bind breaker mutex poisoned")
739 }
740
741 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
744 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
745 probe: probe_token.is_some(),
746 guard: RouteBindBreakerGuard {
747 breakers: self.clone(),
748 module_id,
749 probe_token,
750 settled: false,
751 },
752 };
753
754 let mut modules = self.lock();
755 let Some(state) = modules.get_mut(module_id) else {
756 return admitted(None);
757 };
758 let Some(cooldown_until) = state.cooldown_until else {
759 return admitted(None);
760 };
761 if state.probe_in_flight.is_some() {
762 return RouteBindAdmission::Refused {
763 consecutive_timeouts: state.consecutive_timeouts,
764 retry_in: Duration::ZERO,
765 probe_in_flight: true,
766 };
767 }
768 let now = Instant::now();
769 if now < cooldown_until {
770 return RouteBindAdmission::Refused {
771 consecutive_timeouts: state.consecutive_timeouts,
772 retry_in: cooldown_until - now,
773 probe_in_flight: false,
774 };
775 }
776 let token = Arc::new(());
777 state.probe_in_flight = Some(Arc::clone(&token));
778 admitted(Some(token))
779 }
780
781 fn record_accepted(&self, module_id: &str) -> bool {
782 self.lock()
783 .remove(module_id)
784 .is_some_and(|state| state.cooldown_until.is_some())
785 }
786
787 fn record_timeout(
788 &self,
789 module_id: &str,
790 probe_token: Option<&Arc<()>>,
791 threshold: u32,
792 cooldown: Duration,
793 ) -> Option<BreakerOpened> {
794 let mut modules = self.lock();
795 let state = modules.entry(module_id.to_string()).or_default();
796 let was_open = state.cooldown_until.is_some();
797 let was_probe = Self::owns_probe(state, probe_token);
798 if was_probe {
799 state.probe_in_flight = None;
800 }
801 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
802 if state.consecutive_timeouts < threshold {
803 return None;
804 }
805 state.cooldown_until = Some(Instant::now() + cooldown);
806 Some(BreakerOpened {
807 consecutive_timeouts: state.consecutive_timeouts,
808 reopened_after_probe: was_open && was_probe,
809 })
810 }
811
812 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
813 state
817 .probe_in_flight
818 .as_ref()
819 .zip(token)
820 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
821 }
822
823 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
824 if let Some(state) = self.lock().get_mut(module_id) {
825 if Self::owns_probe(state, probe_token) {
826 state.probe_in_flight = None;
827 }
828 }
829 }
830
831 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
846 self.lock()
847 .remove(module_id)
848 .map(|state| state.consecutive_timeouts)
849 .filter(|discarded| *discarded > 0)
850 }
851
852 fn open_snapshot(&self) -> Option<serde_json::Value> {
859 let now = Instant::now();
860 let modules = self.lock();
861 let open = modules
862 .iter()
863 .filter_map(|(module_id, state)| {
864 let cooldown_until = state.cooldown_until?;
865 Some((
866 module_id.clone(),
867 serde_json::json!({
868 "consecutive_timeouts": state.consecutive_timeouts,
869 "cooldown_remaining_ms":
870 cooldown_until.saturating_duration_since(now).as_millis() as u64,
871 "probe_in_flight": state.probe_in_flight.is_some(),
872 }),
873 ))
874 })
875 .collect::<serde_json::Map<String, serde_json::Value>>();
876 (!open.is_empty()).then_some(serde_json::Value::Object(open))
877 }
878}
879
880impl ControlHandler {
881 pub fn new(registry: Arc<Registry>) -> Self {
882 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
883 }
884
885 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
886 let counters = forwarding.counters();
887 let route_bind_breakers = forwarding.route_bind_breakers();
891 let route_bind_concurrency = forwarding.route_bind_concurrency();
892 let route_outages = forwarding.route_outages();
893 Self {
894 registry,
895 activity: None,
896 activity_streams: Arc::new(Mutex::new(HashMap::new())),
897 forwarding,
898 process_liveness: None,
899 supervisor: SupervisorHandle::new(),
900 subc_capabilities: Arc::from([
901 CAP_MANIFEST_REGISTRATION.to_string(),
902 CAP_CHANNEL_LIFECYCLE.to_string(),
903 CAP_PING_PONG.to_string(),
904 CAP_SESSION_ATTACH.to_string(),
905 CAP_ADMISSION_FACTS_RELAY.to_string(),
906 CAP_SCOPES_V1.to_string(),
907 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
908 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
909 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
910 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
911 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
912 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
913 ]),
914 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
915 route_bind_relay_timeouts: BTreeMap::new(),
916 route_bind_breakers,
917 route_bind_concurrency,
918 route_outages,
919 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
920 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
921 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
922 storage_config: None,
923 machine_id: None,
924 admission_facts_carrier_module_id: None,
925 admission_facts_targets: None,
926 scopes: Arc::new(RwLock::new(ScopeTable::new(
927 crate::daemon_config::default_scope_authority_owners(),
928 ))),
929 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
930 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
931 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
932 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
933 #[cfg(test)]
934 caller_lookup: None,
935 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
936 rescan: None,
937 connected_clients: ConnectedClients::new(),
938 counters,
939 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
940 daemon_provenance: DaemonProvenanceFacts::default(),
941 #[cfg(test)]
942 control_dispatch_delay: None,
943 #[cfg(test)]
944 provenance_probe_override: None,
945 #[cfg(test)]
946 route_bind_relay_sent_hold: None,
947 #[cfg(test)]
948 hello_registry_to_forwarding_hold: None,
949 }
950 }
951
952 pub fn with_storage_config(
955 mut self,
956 storage_config: Option<crate::daemon_config::StorageConfig>,
957 ) -> Self {
958 self.storage_config = storage_config;
959 self
960 }
961
962 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
965 self.activity = Some(crate::activity::ActivityLedger::open(
966 run_dir.join("activity.jsonl"),
967 incarnation,
968 ));
969 self.supervisor.set_activity(self.activity.clone());
970 self
971 }
972
973 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
976 self.machine_id = machine_id;
977 self
978 }
979
980 pub fn with_admission_facts_config(
984 mut self,
985 carrier_module_id: Option<String>,
986 targets: Option<Vec<String>>,
987 ) -> Self {
988 self.admission_facts_carrier_module_id = carrier_module_id;
989 self.admission_facts_targets = targets;
990 self
991 }
992
993 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
997 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
998 self.scope_authority_owners = owners;
999 self
1000 }
1001
1002 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1004 self.privileged_operator_prompt = mode;
1005 self
1006 }
1007
1008 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1011 self.route_bind_relay_timeout = timeout;
1012 self
1013 }
1014
1015 pub fn with_route_bind_relay_timeouts(
1021 mut self,
1022 timeouts: impl IntoIterator<Item = (String, Duration)>,
1023 ) -> Self {
1024 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1025 self
1026 }
1027
1028 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1034 self.route_bind_relay_timeouts
1035 .get(module_id)
1036 .copied()
1037 .unwrap_or(self.route_bind_relay_timeout)
1038 }
1039
1040 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1048 self.route_bind_breaker_threshold = threshold.max(1);
1049 self.route_bind_breaker_cooldown = cooldown;
1050 self
1051 }
1052
1053 #[cfg(test)]
1054 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1055 self.health_probe_timeout = timeout;
1056 self
1057 }
1058
1059 #[cfg(test)]
1060 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1061 self.control_dispatch_delay = Some(delay);
1062 self
1063 }
1064
1065 #[cfg(test)]
1069 pub(crate) fn with_route_bind_relay_sent_hold(
1070 mut self,
1071 hold: Arc<tokio::sync::Barrier>,
1072 ) -> Self {
1073 self.route_bind_relay_sent_hold = Some(hold);
1074 self
1075 }
1076
1077 #[cfg(test)]
1082 pub(crate) fn with_hello_registry_to_forwarding_hold(
1083 mut self,
1084 hold: Arc<std::sync::Barrier>,
1085 ) -> Self {
1086 self.hello_registry_to_forwarding_hold = Some(hold);
1087 self
1088 }
1089
1090 #[cfg(test)]
1091 fn hold_between_registry_and_forwarding(&self) {
1092 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1093 hold.wait();
1094 hold.wait();
1095 }
1096 }
1097
1098 pub fn with_process_liveness(
1099 mut self,
1100 process_liveness: Arc<dyn ModuleProcessLiveness>,
1101 ) -> Self {
1102 self.process_liveness = Some(process_liveness);
1103 self
1104 }
1105
1106 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1107 self.supervisor = supervisor;
1108 self.supervisor.set_activity(self.activity.clone());
1109 self
1110 }
1111
1112 pub fn with_daemon_provenance(
1113 mut self,
1114 pid: u32,
1115 started_at_ms: u64,
1116 executable_path: Option<PathBuf>,
1117 build_git_sha: Option<String>,
1118 build_lock_digest: Option<String>,
1119 ) -> Self {
1120 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1121 let process_start_time = process_start_time(pid);
1122 self.daemon_provenance = DaemonProvenanceFacts {
1123 build: DaemonBuildProvenance {
1124 build_git_sha,
1125 build_lock_digest,
1126 },
1127 pid: Some(pid),
1128 started_at_ms: Some(started_at_ms),
1129 start_clock: None,
1130 executable_path,
1131 executable_identity,
1132 process_start_time,
1133 probe: ExecutableIdentityProbe::default(),
1134 };
1135 self
1136 }
1137
1138 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1139 self.daemon_provenance.start_clock = Some(clock);
1140 self
1141 }
1142
1143 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1145 self.scopes
1146 .write()
1147 .unwrap_or_else(|poisoned| poisoned.into_inner())
1148 .set_wall_clock(Arc::new(clock));
1149 self
1150 }
1151
1152 #[cfg(test)]
1153 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1154 self.provenance_probe_override = Some(result);
1155 self
1156 }
1157
1158 pub fn with_capability_config(
1162 self,
1163 modules: impl IntoIterator<Item = (String, bool)>,
1164 reserved_capabilities: BTreeMap<String, String>,
1165 ) -> Self {
1166 self.capability_evaluator
1167 .configure(modules, reserved_capabilities);
1168 self
1169 }
1170
1171 pub fn with_supervisor_rescan(
1172 mut self,
1173 supervisor: Supervisor,
1174 config_path: impl Into<PathBuf>,
1175 configured_port: Option<u16>,
1176 ) -> Self {
1177 self.rescan = Some(SupervisorRescanContext {
1178 supervisor,
1179 config_path: config_path.into(),
1180 configured_port,
1181 storage_config: self.storage_config.clone(),
1182 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1183 admission_facts_targets: self.admission_facts_targets.clone(),
1184 scope_authority_owners: self.scope_authority_owners.clone(),
1185 privileged_operator_prompt: self.privileged_operator_prompt,
1186 });
1187 self
1188 }
1189
1190 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1191 self.connected_clients = connected_clients;
1192 self
1193 }
1194
1195 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1196 Arc::clone(&self.forwarding)
1197 }
1198
1199 pub(crate) fn counters(&self) -> DaemonCounters {
1200 self.counters.clone()
1201 }
1202
1203 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1206 tokio::spawn(async move {
1207 loop {
1208 self.capability_evaluator
1209 .wait_for_change_or_deadline()
1210 .await;
1211 self.refresh_capability_requirements();
1212 }
1213 });
1214 }
1215
1216 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1222 tokio::spawn(async move {
1223 loop {
1224 let has_deadlines = self
1225 .scopes
1226 .read()
1227 .unwrap_or_else(|poisoned| poisoned.into_inner())
1228 .has_deadlines();
1229 if !has_deadlines {
1230 self.scope_deadline_added.notified().await;
1231 continue;
1232 }
1233 tokio::time::sleep(Duration::from_secs(1)).await;
1234 if let Err(error) = self.sweep_expired_scopes() {
1235 warn!(%error, "scope expiry sweep failed");
1236 }
1237 }
1238 });
1239 }
1240
1241 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1242 let mut table = self
1243 .scopes
1244 .write()
1245 .unwrap_or_else(|poisoned| poisoned.into_inner());
1246 let swept = table.sweep_expired();
1247 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1248 drop(table);
1249 self.close_scope_drained_routes(drained);
1250 Ok(())
1251 }
1252
1253 fn publish_scope_changes(
1254 &self,
1255 changes: &[crate::scopes::ScopeTagChange],
1256 expired: &[crate::scopes::ScopeExpired],
1257 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1258 if changes.is_empty() {
1259 return Ok(Vec::new());
1260 }
1261 let drained = self
1262 .forwarding
1263 .publish_scope_changes(changes)
1264 .map_err(RouterError::Forwarding)?;
1265 let mut counts = HashMap::new();
1266 for route in &drained {
1267 *counts
1268 .entry((
1269 route.scope.owner.as_str(),
1270 route.scope.scope_ref.as_str(),
1271 route.scope.tag.scope_epoch,
1272 ))
1273 .or_insert(0usize) += 1;
1274 }
1275 for scope in expired {
1276 let routes_closed = counts
1277 .get(&(
1278 scope.owner.as_str(),
1279 scope.scope_ref.as_str(),
1280 scope.scope_epoch,
1281 ))
1282 .copied()
1283 .unwrap_or(0);
1284 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1285 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1286 routes_closed, "scope expired");
1287 }
1288 Ok(drained)
1289 }
1290
1291 fn runtime_capability_snapshot(
1292 &self,
1293 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1294 let runtime = self
1295 .supervisor
1296 .list()
1297 .into_iter()
1298 .map(|module| {
1299 let status = module.status().map_err(|err| {
1300 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1301 })?;
1302 Ok(RuntimeModule {
1303 module_id: status.module_id,
1304 state: status.state,
1305 enabled: status.enabled,
1306 })
1307 })
1308 .collect::<Result<Vec<_>, RouterError>>()?;
1309 let registrations = self.registry.list_registered_modules().map_err(|err| {
1315 RouterError::backend(
1316 0,
1317 0,
1318 format!("failed to list capability registrations: {err}"),
1319 )
1320 })?;
1321 let registrations = registrations
1322 .into_iter()
1323 .map(|registration| RegisteredModule {
1324 module_id: registration.manifest.module_id,
1325 module_version: registration.manifest.module_version,
1326 capabilities: registration.manifest.capabilities,
1327 })
1328 .collect();
1329 Ok((runtime, registrations))
1330 }
1331
1332 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1339 let cached_registration = RegisteredModule {
1340 module_id: registration.manifest.module_id.clone(),
1341 module_version: registration.manifest.module_version.clone(),
1342 capabilities: registration.manifest.capabilities.clone(),
1343 };
1344 if self.capability_evaluator.record_hello(&cached_registration) {
1345 warn!(
1346 module_id = %cached_registration.module_id,
1347 "capability claims drifted from the cached manifest"
1348 );
1349 }
1350 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1351 self.enforce_capability_denies();
1352 }
1353 self.refresh_capability_requirements();
1354 }
1355
1356 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1360 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1361 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1362 self.supervisor.set_swap_promotion_observer(observer);
1363 }
1364
1365 pub fn refresh_capability_requirements(&self) {
1366 match self.runtime_capability_snapshot() {
1367 Ok((runtime, registrations)) => {
1368 log_requirement_events(
1369 self.capability_evaluator
1370 .evaluate_now(&runtime, ®istrations),
1371 );
1372 }
1373 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1374 }
1375 }
1376
1377 fn enforce_capability_denies(&self) {
1381 let registrations = match self.registry.list_registered_modules() {
1382 Ok(snapshot) => snapshot,
1383 Err(err) => {
1384 warn!(error = %err, "failed to read registrations for capability deny census");
1385 return;
1386 }
1387 };
1388 let manifests = registrations
1389 .into_iter()
1390 .map(|registration| {
1391 (
1392 registration.manifest.module_id.clone(),
1393 registration.manifest,
1394 )
1395 })
1396 .collect::<BTreeMap<_, _>>();
1397 let census = match self.forwarding.route_census(None) {
1398 Ok(census) => census,
1399 Err(err) => {
1400 warn!(error = %err, "failed to read route census for capability deny enforcement");
1401 return;
1402 }
1403 };
1404
1405 for (target_module_id, routes) in census {
1406 let Some(target_manifest) = manifests.get(&target_module_id) else {
1407 continue;
1408 };
1409 let mut closed_routes = Vec::new();
1410 let mut module_goodbyes = Vec::new();
1411 for route in routes {
1412 let Principal::Reserved {
1413 module_id: opening_module_id,
1414 } = &route.principal
1415 else {
1416 continue;
1417 };
1418 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1419 continue;
1420 };
1421 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1422 continue;
1423 };
1424
1425 match self.forwarding.release_client_route(
1426 route.goodbye_target.connection_id,
1427 route.goodbye_target.channel,
1428 route.goodbye_target.epoch,
1429 ) {
1430 Ok(RouteRelease::Removed(module_goodbye)) => {
1431 warn!(
1432 opening_module_id,
1433 target_module_id,
1434 capability,
1435 "force-closing route because an attested capability deny edge now matches"
1436 );
1437 closed_routes.push(route);
1438 module_goodbyes.push(module_goodbye);
1439 }
1440 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1441 Err(err) => warn!(
1442 opening_module_id,
1443 target_module_id,
1444 capability,
1445 error = %err,
1446 "failed to force-close capability-denied route"
1447 ),
1448 }
1449 }
1450
1451 if closed_routes.is_empty() {
1452 continue;
1453 }
1454 send_route_control_pushes(
1455 &self.forwarding,
1456 closed_routes,
1457 ClientControlPush::RouteClosed {
1458 module_id: target_module_id,
1459 channels: Vec::new(),
1460 reason: RouteCloseReason::CapabilityDenied,
1461 drained: false,
1462 abandoned: 0,
1463 excluded_subscriptions: 0,
1464 terminal: Some(false),
1465 },
1466 );
1467 self.emit_route_goodbyes(module_goodbyes);
1468 }
1469 }
1470
1471 fn not_ready_reason(
1477 &self,
1478 registration: &crate::registry::ModuleRegistration,
1479 ) -> Option<NotReadyReason> {
1480 if !registration.ready {
1481 return Some(NotReadyReason {
1482 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1483 capability: None,
1484 });
1485 }
1486 self.first_unprovided_required_capability(registration)
1487 .map(|capability| NotReadyReason {
1488 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1489 capability: Some(capability),
1490 })
1491 }
1492
1493 fn first_unprovided_required_capability(
1511 &self,
1512 registration: &crate::registry::ModuleRegistration,
1513 ) -> Option<String> {
1514 let required = registration
1515 .manifest
1516 .capabilities
1517 .iter()
1518 .flat_map(|declarations| declarations.requires.iter())
1519 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1520 .map(|requirement| requirement.capability.as_str())
1521 .collect::<BTreeSet<_>>();
1522 if required.is_empty() {
1523 return None;
1524 }
1525 let module_id = registration.manifest.module_id.as_str();
1526 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1527 if required
1528 .iter()
1529 .any(|capability| verdict(capability).is_none())
1530 {
1531 self.refresh_capability_requirements();
1532 }
1533 required
1534 .into_iter()
1535 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1536 .map(str::to_string)
1537 }
1538
1539 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1540 self.capability_evaluator
1541 .statuses()
1542 .into_iter()
1543 .map(capability_requirement_status)
1544 .collect()
1545 }
1546
1547 fn deregister_connection(
1554 &self,
1555 connection_id: ConnectionId,
1556 reason: RegistrationEndReason,
1557 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1558 self.registry
1559 .deregister_connection_with_reason(connection_id, reason)
1560 }
1561
1562 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1563 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1564 return None;
1565 }
1566 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1567 parse_client_control_request(&frame.body)
1568 else {
1569 return None;
1570 };
1571 Some(target_module_id(&target).to_string())
1572 }
1573
1574 pub(crate) fn route_open_capacity_refusal(
1575 &self,
1576 ctx: &RouteCtx,
1577 frame: &Frame,
1578 target_module_id: &str,
1579 in_flight: usize,
1580 limit: usize,
1581 ) -> Result<Frame, RouterError> {
1582 self.route_open_admission_refusal_frame(
1583 ctx,
1584 frame,
1585 target_module_id,
1586 "open_admission_full",
1587 (in_flight, limit),
1588 format!(
1589 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1590 ),
1591 )
1592 }
1593
1594 fn route_open_target_capacity_refusal(
1595 &self,
1596 ctx: &RouteCtx,
1597 frame: &Frame,
1598 target_module_id: &str,
1599 in_flight: usize,
1600 ) -> Result<Frame, RouterError> {
1601 self.route_open_admission_refusal_frame(
1602 ctx,
1603 frame,
1604 target_module_id,
1605 "target_binds_full",
1606 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1607 format!(
1608 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1609 ),
1610 )
1611 }
1612
1613 fn route_open_admission_refusal_frame(
1621 &self,
1622 ctx: &RouteCtx,
1623 frame: &Frame,
1624 target_module_id: &str,
1625 reason: &'static str,
1626 (in_flight, limit): (usize, usize),
1627 message: impl Into<String>,
1628 ) -> Result<Frame, RouterError> {
1629 let code = error_codes::TARGET_UNAVAILABLE;
1630 self.counters.increment_route_open_refused(code);
1631 info!(
1632 target: "control",
1633 code,
1634 reason,
1635 module_id = ?target_module_id,
1636 connection_id = ctx.connection_id.get(),
1637 in_flight,
1638 limit,
1639 "route.open refused"
1640 );
1641 control_error_frame(frame, code, message.into())
1642 }
1643
1644 #[cfg(test)]
1650 pub fn handle_control(
1651 &self,
1652 connection_id: ConnectionId,
1653 frame: Frame,
1654 ) -> Result<Vec<Frame>, RouterError> {
1655 match frame.header.ty {
1656 FrameType::Ping => Ok(vec![pong(&frame)?]),
1657 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1658 FrameType::Goodbye => self.handle_goodbye(connection_id),
1659 ty => Ok(vec![control_error_frame(
1660 &frame,
1661 "unsupported_control_frame",
1662 format!("unsupported channel-0 frame {ty:?}"),
1663 )?]),
1664 }
1665 }
1666
1667 pub async fn handle_control_frame(
1668 &self,
1669 ctx: &RouteCtx,
1670 frame: Frame,
1671 ) -> Result<Vec<Frame>, RouterError> {
1672 self.handle_control_frame_timed(ctx, frame, None).await
1673 }
1674
1675 pub(crate) fn record_caller_endpoints(
1676 &self,
1677 connection: ConnectionId,
1678 endpoints: Option<(SocketAddr, SocketAddr)>,
1679 ) {
1680 let mut callers = self
1681 .caller_endpoints
1682 .lock()
1683 .unwrap_or_else(|p| p.into_inner());
1684 callers.remove(&connection);
1685 if let Some(endpoints) = endpoints {
1686 callers.insert(connection, endpoints);
1687 }
1688 }
1689
1690 #[cfg(test)]
1691 pub(crate) fn with_caller_lookup_for_test(
1692 mut self,
1693 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1694 ) -> Self {
1695 self.caller_lookup = Some(Arc::new(lookup));
1696 self
1697 }
1698
1699 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1700 let endpoints = self
1701 .caller_endpoints
1702 .lock()
1703 .unwrap_or_else(|p| p.into_inner())
1704 .get(&connection)
1705 .copied();
1706 #[cfg(test)]
1707 let lookup = self.caller_lookup.clone();
1708 tokio::task::spawn_blocking(move || {
1709 #[cfg(test)]
1710 if let Some(lookup) = lookup {
1711 return lookup(endpoints);
1712 }
1713 endpoints
1714 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1715 .unwrap_or_default()
1716 })
1717 .await
1718 .unwrap_or_default()
1719 }
1720
1721 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1722 frame.header.channel == 0
1723 && frame.header.ty == FrameType::Request
1724 && self
1725 .forwarding
1726 .module_endpoint_for_connection(ctx.connection_id)
1727 .is_ok_and(|endpoint| endpoint.is_none())
1728 && parse_client_control_request(&frame.body)
1729 .ok()
1730 .is_some_and(|request| privileged_operation(&request).is_some())
1731 }
1732
1733 pub(crate) async fn handle_control_frame_timed(
1734 &self,
1735 ctx: &RouteCtx,
1736 frame: Frame,
1737 dispatch_started_at: Option<StdInstant>,
1738 ) -> Result<Vec<Frame>, RouterError> {
1739 match frame.header.ty {
1740 FrameType::Ping => Ok(vec![pong(&frame)?]),
1741 FrameType::Hello => {
1742 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1743 }
1744 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1745 FrameType::Cancel => {
1746 if self
1749 .forwarding
1750 .operator_confirms()
1751 .cancel(ctx.connection_id, frame.header.corr)
1752 || self
1753 .supervisor
1754 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1755 || self.cancel_activity_subscription(ctx, &frame)?
1756 {
1757 Ok(Vec::new())
1758 } else {
1759 Ok(vec![control_error_frame(
1760 &frame,
1761 "unknown_subscription",
1762 "no subscription has this correlation id",
1763 )?])
1764 }
1765 }
1766 FrameType::Request => {
1767 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1770 if op
1771 .as_ref()
1772 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1773 {
1774 return self.handle_activity_record(ctx, frame).await;
1775 }
1776 if op.as_ref().is_some_and(|probe| {
1777 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1778 }) {
1779 return self.handle_activity_read(ctx, frame).await;
1780 }
1781 if op
1782 .as_ref()
1783 .is_some_and(|probe| probe.op == "operator.confirm")
1784 {
1785 return self.handle_operator_confirm(ctx, frame);
1786 }
1787 if self
1788 .forwarding
1789 .module_endpoint_for_connection(ctx.connection_id)
1790 .map_err(RouterError::Forwarding)?
1791 .is_some()
1792 {
1793 if !is_known_module_request_op(&frame.body) {
1794 return Ok(vec![control_error_frame(
1795 &frame,
1796 "unsupported_control_frame",
1797 "module-originated channel-0 REQUEST is not supported",
1798 )?]);
1799 }
1800 let request = match parse_module_control_request_from_module(&frame.body) {
1801 Ok(request) => request,
1802 Err((err, ControlRequestBodyError::UnknownOp)) => {
1803 return Ok(vec![control_error_frame(
1804 &frame,
1805 "unsupported_control_frame",
1806 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1807 )?])
1808 }
1809 Err((err, ControlRequestBodyError::InvalidBody)) => {
1810 return Ok(vec![control_error_frame(
1811 &frame,
1812 "invalid_control_body",
1813 format!("malformed module control body: {err}"),
1814 )?])
1815 }
1816 };
1817 let op = module_control_request_op(&request);
1818 let corr = frame.header.corr;
1819 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1820 let result =
1821 self.handle_module_control_request(ctx.connection_id, frame, request);
1822 log_slow_control_dispatch(
1823 dispatch_started_at,
1824 op,
1825 ctx.connection_id,
1826 corr,
1827 None,
1828 );
1829 return result;
1830 }
1831
1832 if is_known_module_request_op(&frame.body) {
1833 return Ok(vec![control_error_frame(
1834 &frame,
1835 "not_registered",
1836 "catalog.update requires an active module registration owned by this connection",
1837 )?]);
1838 }
1839
1840 let request = match parse_client_control_request(&frame.body) {
1841 Ok(request) => request,
1842 Err((err, ControlRequestBodyError::UnknownOp)) => {
1843 return Ok(vec![control_error_frame(
1844 &frame,
1845 "unknown_control_op",
1846 format!("unknown client control op: {err}"),
1847 )?])
1848 }
1849 Err((err, ControlRequestBodyError::InvalidBody)) => {
1850 return Ok(vec![control_error_frame(
1851 &frame,
1852 "invalid_control_body",
1853 format!("malformed client control body: {err}"),
1854 )?])
1855 }
1856 };
1857 let op = client_control_request_op(&request);
1858 let corr = frame.header.corr;
1859 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1860 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1861 #[cfg(test)]
1862 if let Some(delay) = self.control_dispatch_delay {
1863 tokio::time::sleep(delay).await;
1864 }
1865 let result = self
1866 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1867 .await;
1868 if let Some(timings) = route_open_timings.as_mut() {
1869 timings.finish();
1870 }
1871 log_slow_control_dispatch(
1872 dispatch_started_at,
1873 op,
1874 ctx.connection_id,
1875 corr,
1876 route_open_timings.as_ref(),
1877 );
1878 result
1879 }
1880 FrameType::Push => {
1881 let Some(endpoint) = self
1882 .forwarding
1883 .module_endpoint_for_connection(ctx.connection_id)
1884 .map_err(RouterError::Forwarding)?
1885 else {
1886 return Ok(vec![control_error_frame(
1887 &frame,
1888 "unsupported_control_frame",
1889 "client-originated channel-0 PUSH is not supported",
1890 )?]);
1891 };
1892 self.handle_status_update(endpoint, frame)
1893 }
1894 FrameType::Response | FrameType::Error
1895 if self
1896 .forwarding
1897 .module_endpoint_for_connection(ctx.connection_id)
1898 .map_err(RouterError::Forwarding)?
1899 .is_some() =>
1900 {
1901 self.handle_module_relay_response(ctx.connection_id, frame)
1902 }
1903 ty => Ok(vec![control_error_frame(
1904 &frame,
1905 "unsupported_control_frame",
1906 format!("unsupported channel-0 frame {ty:?}"),
1907 )?]),
1908 }
1909 }
1910
1911 pub fn cleanup_connection(
1912 &self,
1913 connection_id: ConnectionId,
1914 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1915 self.caller_endpoints
1916 .lock()
1917 .unwrap_or_else(|p| p.into_inner())
1918 .remove(&connection_id);
1919 self.forwarding
1920 .operator_confirms()
1921 .connection_closed(connection_id);
1922 self.cleanup_connection_with_end_reason(
1923 connection_id,
1924 RegistrationEndReason::ConnectionClosed,
1925 )
1926 }
1927
1928 fn cleanup_connection_with_end_reason(
1929 &self,
1930 connection_id: ConnectionId,
1931 requested_reason: RegistrationEndReason,
1932 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1933 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1934 self.registry
1935 .get_module_by_connection(connection_id)?
1936 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
1937 .and_then(|module| module.registration_end_reason().ok().flatten())
1938 .unwrap_or(requested_reason)
1939 } else {
1940 requested_reason
1941 };
1942 let crash_closed = self
1943 .registry
1944 .get_module_by_connection(connection_id)?
1945 .and_then(|registration| {
1946 self.forwarding
1947 .module_endpoint_for_connection(connection_id)
1948 .ok()
1949 .flatten()
1950 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1951 .map(|routes| (registration.manifest.module_id, routes))
1952 });
1953 let crash_closed = crash_closed.map(|(module_id, routes)| {
1954 let terminal = match self.supervisor.get(&module_id) {
1955 None => false,
1956 Some(module) => match module.will_recover_after_connection_loss() {
1957 Ok(will_recover) => !will_recover,
1958 Err(err) => {
1959 warn!(
1960 %module_id,
1961 error = %err,
1962 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1963 );
1964 false
1965 }
1966 },
1967 };
1968 let (reason, drained) = match self
1972 .forwarding
1973 .daemon_drain_result_for_connection(connection_id)
1974 {
1975 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
1976 Ok(None) => (RouteCloseReason::Crash, false),
1977 Err(err) => {
1978 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1979 (RouteCloseReason::Crash, false)
1980 }
1981 };
1982 (module_id, routes, reason, drained, terminal)
1983 });
1984 let registrations = self.deregister_connection(connection_id, end_reason);
1985 let cleanup = if crash_closed.is_some() {
1986 self.forwarding.cleanup_connection_counted(connection_id)
1987 } else {
1988 self.forwarding
1994 .cleanup_connection(connection_id)
1995 .map(|released| crate::forwarding::ConnectionCleanup {
1996 released,
1997 abandoned_relays: 0,
1998 })
1999 };
2000 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2005 let abandoned = cleanup
2006 .as_ref()
2007 .map_or(0, |cleanup| cleanup.abandoned_relays);
2008 send_route_control_pushes(
2009 &self.forwarding,
2010 routes,
2011 ClientControlPush::RouteClosed {
2012 module_id,
2013 channels: Vec::new(),
2014 reason,
2015 drained,
2016 abandoned,
2017 excluded_subscriptions: 0,
2018 terminal: Some(terminal),
2019 },
2020 );
2021 }
2022 if let Ok(cleanup) = cleanup {
2023 self.emit_route_goodbyes(cleanup.released);
2024 }
2025 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2029 crate::supervise::notify_registration_release();
2030 self.capability_evaluator.wake_deadline_loop();
2031 self.refresh_capability_requirements();
2032 }
2033 self.supervisor.remove_spawn_subscribers(connection_id);
2034 self.activity_streams
2035 .lock()
2036 .unwrap_or_else(|p| p.into_inner())
2037 .retain(|(id, _), task| {
2038 if *id == connection_id {
2039 task.abort();
2040 false
2041 } else {
2042 true
2043 }
2044 });
2045 self.hello_launch_nonces
2048 .lock()
2049 .unwrap_or_else(|poisoned| poisoned.into_inner())
2050 .forget(connection_id);
2051 self.scopes
2052 .write()
2053 .unwrap_or_else(|poisoned| poisoned.into_inner())
2054 .release_connection(connection_id);
2055 registrations
2056 }
2057
2058 pub(crate) fn handle_route_goodbye(
2059 &self,
2060 connection_id: ConnectionId,
2061 route_channel: u16,
2062 route_epoch: u32,
2063 ) -> Result<bool, RouterError> {
2064 debug!(
2065 connection_id = connection_id.get(),
2066 route_channel, route_epoch, "handling route GOODBYE"
2067 );
2068 let RouteRelease::Removed(released_route) = self
2069 .forwarding
2070 .release_client_route(connection_id, route_channel, route_epoch)
2071 .map_err(RouterError::Forwarding)?
2072 else {
2073 return Ok(false);
2074 };
2075 self.emit_route_goodbyes(vec![released_route]);
2076 Ok(true)
2077 }
2078
2079 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2080 for released in released_routes {
2081 let frame = match Frame::build_with_version(
2082 released.negotiated_ver,
2083 FrameType::Goodbye,
2084 control_flags(),
2085 released.channel,
2086 released.epoch,
2087 0,
2088 Vec::new(),
2089 ) {
2090 Ok(frame) => frame,
2091 Err(err) => {
2092 warn!(
2093 route_channel = released.channel,
2094 error = %err,
2095 "failed to build route GOODBYE frame"
2096 );
2097 continue;
2098 }
2099 };
2100 if !released.close_on_delivery_failure() {
2101 crate::forwarding::send_module_route_goodbye(
2102 &self.counters,
2103 &released.sink,
2104 frame,
2105 released.module_id.as_deref(),
2106 "client route released",
2107 );
2108 continue;
2109 }
2110 if let Err(err) = released.sink.try_send(frame) {
2111 warn!(
2112 target_connection_id = released.connection_id.get(),
2113 route_channel = released.channel,
2114 error = %err,
2115 "route GOODBYE was not delivered to client; closing target connection"
2116 );
2117 if self
2118 .forwarding
2119 .escalate_client_delivery_failure(
2120 released.connection_id,
2121 released.channel,
2122 released.epoch,
2123 CloseReason::new(
2124 "route_goodbye_delivery_failed",
2125 format!(
2126 "failed to enqueue route GOODBYE for channel {}: {err}",
2127 released.channel
2128 ),
2129 ),
2130 crate::forwarding::UndeliveredFrame {
2131 module_id: released.module_id.as_deref(),
2132 sink: &released.sink,
2133 },
2134 )
2135 .unwrap_or(false)
2136 {
2137 self.counters.increment_goodbye_relay_client_failed();
2138 }
2139 }
2140 }
2141 }
2142
2143 fn send_abandoned_route_bind_goodbye(
2155 &self,
2156 module_sink: &crate::FrameSink,
2157 negotiated_ver: u8,
2158 module_channel: u16,
2159 module_epoch: u32,
2160 ) {
2161 let frame = match Frame::build_with_version(
2162 negotiated_ver,
2163 FrameType::Goodbye,
2164 control_flags(),
2165 module_channel,
2166 module_epoch,
2167 0,
2168 Vec::new(),
2169 ) {
2170 Ok(frame) => frame,
2171 Err(err) => {
2172 warn!(
2173 route_channel = module_channel,
2174 error = %err,
2175 "failed to build GOODBYE for abandoned route.bind"
2176 );
2177 return;
2178 }
2179 };
2180 crate::forwarding::send_module_route_goodbye(
2181 &self.counters,
2182 module_sink,
2183 frame,
2184 None,
2185 "abandoned route.bind",
2186 );
2187 }
2188
2189 fn handle_hello(
2190 &self,
2191 connection_id: ConnectionId,
2192 sink: Option<crate::FrameSink>,
2193 frame: Frame,
2194 ) -> Result<Vec<Frame>, RouterError> {
2195 debug!(
2196 connection_id = connection_id.get(),
2197 corr = frame.header.corr,
2198 "handling HELLO"
2199 );
2200 if self
2204 .registry
2205 .get_module_by_connection(connection_id)
2206 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2207 .is_some()
2208 {
2209 return Ok(vec![control_error_frame(
2210 &frame,
2211 "invalid_hello",
2212 "connection is already registered as a module",
2213 )?]);
2214 }
2215 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2216 Ok(value) => value,
2217 Err(err) => {
2218 return Ok(vec![control_error_frame(
2219 &frame,
2220 "invalid_hello",
2221 format!("malformed HELLO body: {err}"),
2222 )?])
2223 }
2224 };
2225 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2226 return Ok(vec![control_error_frame(
2227 &frame,
2228 "invalid_capability_grammar",
2229 err.to_string(),
2230 )?]);
2231 }
2232 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2233 return Ok(vec![control_error_frame(
2234 &frame,
2235 "invalid_manifest",
2236 err.to_string(),
2237 )?]);
2238 }
2239 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2240 return Ok(vec![control_error_frame(
2241 &frame,
2242 "invalid_event_declaration",
2243 err.to_string(),
2244 )?]);
2245 }
2246 if let Some(provenance) = hello_value
2247 .get("manifest")
2248 .and_then(|manifest| manifest.get("provenance"))
2249 {
2250 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2251 return Ok(vec![control_error_frame(
2252 &frame,
2253 "invalid_manifest",
2254 format!("malformed manifest provenance: {err}"),
2255 )?]);
2256 }
2257 }
2258 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2259 Ok(hello) => hello,
2260 Err(err) => {
2261 return Ok(vec![control_error_frame(
2262 &frame,
2263 "invalid_hello",
2264 format!("malformed HELLO body: {err}"),
2265 )?])
2266 }
2267 };
2268
2269 if hello.protocol_ver != hello.manifest.protocol_ver {
2270 return Ok(vec![control_error_frame(
2271 &frame,
2272 "invalid_manifest",
2273 format!(
2274 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2275 hello.protocol_ver, hello.manifest.protocol_ver
2276 ),
2277 )?]);
2278 }
2279
2280 if hello.manifest.module_id.trim().is_empty() {
2281 return Ok(vec![control_error_frame(
2282 &frame,
2283 "invalid_manifest",
2284 "manifest module_id must not be empty",
2285 )?]);
2286 }
2287
2288 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2289 Ok(negotiated_ver) => negotiated_ver,
2290 Err(message) => {
2291 return Ok(vec![control_error_frame(
2292 &frame,
2293 "version_unsupported",
2294 message,
2295 )?])
2296 }
2297 };
2298
2299 let swap_admission = self
2307 .supervisor
2308 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2309 if swap_admission == SwapHelloAdmission::Refused {
2310 warn!(
2311 module_id = %hello.manifest.module_id,
2312 connection_id = connection_id.get(),
2313 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2314 );
2315 return Ok(vec![control_error_frame(
2316 &frame,
2317 "swap_token_invalid",
2318 format!(
2319 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2320 hello.manifest.module_id
2321 ),
2322 )?]);
2323 }
2324 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2325
2326 if let Some(rejection) = (!swap_candidate)
2334 .then(|| {
2335 self.supervisor.reserved_hello_rejection(
2336 &hello.manifest.module_id,
2337 hello.launch_nonce.as_deref(),
2338 )
2339 })
2340 .flatten()
2341 {
2342 let message = match rejection {
2343 ReservedHelloRejection::Exact { module_id } => format!(
2344 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2345 ),
2346 ReservedHelloRejection::Prefix {
2347 prefix,
2348 owner_module_id,
2349 } => format!(
2350 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2351 hello.manifest.module_id
2352 ),
2353 };
2354 return Ok(vec![control_error_frame(
2355 &frame,
2356 "reserved_module",
2357 message,
2358 )?]);
2359 }
2360
2361 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2362 &hello.manifest.module_id,
2363 hello.manifest.capabilities.as_ref(),
2364 );
2365 if let Some(refusal) = reserved_capability_refusals.first() {
2366 let capability = refusal.capability.clone();
2367 let bound_module = refusal.claimants[0].clone();
2368 log_duplicate_claim_events(reserved_capability_refusals);
2369 return Ok(vec![control_error_frame(
2370 &frame,
2371 "reserved_capability",
2372 format!(
2373 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2374 capability, bound_module, hello.manifest.module_id
2375 ),
2376 )?]);
2377 }
2378
2379 if self
2382 .forwarding
2383 .connection_has_client_routes(connection_id)
2384 .map_err(RouterError::Forwarding)?
2385 {
2386 return Ok(vec![control_error_frame(
2387 &frame,
2388 "invalid_hello",
2389 "connection has open client routes and cannot also register as a module",
2390 )?]);
2391 }
2392
2393 self.hello_launch_nonces
2398 .lock()
2399 .unwrap_or_else(|poisoned| poisoned.into_inner())
2400 .record(connection_id, hello.launch_nonce.as_deref());
2401 let control_ops = effective_module_control_ops(hello.control_ops);
2402 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2405 if swap_candidate {
2406 return self.register_swap_candidate(
2407 connection_id,
2408 sink,
2409 &frame,
2410 hello.manifest,
2411 negotiated_ver,
2412 control_ops,
2413 hello_ack,
2414 );
2415 }
2416 let registration = match self.registry.register_unpublished_with_control_ops(
2423 hello.manifest,
2424 negotiated_ver,
2425 connection_id,
2426 control_ops,
2427 ) {
2428 Ok(registration) => registration,
2429 Err(RegistryError::DuplicateModuleId { module_id }) => {
2430 return Ok(vec![control_error_frame(
2431 &frame,
2432 "duplicate_module_id",
2433 format!(
2434 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2435 ),
2436 )?])
2437 }
2438 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2439 return Ok(vec![control_error_frame(
2440 &frame,
2441 "invalid_module_id",
2442 err.to_string(),
2443 )?])
2444 }
2445 Err(err) => {
2446 return Ok(vec![control_error_frame(
2447 &frame,
2448 "registry_error",
2449 err.to_string(),
2450 )?])
2451 }
2452 };
2453 #[cfg(test)]
2454 self.hold_between_registry_and_forwarding();
2455
2456 let reply = if let Some(sink) = sink {
2457 let concurrency = manifest_concurrency(®istration.manifest);
2473 if let Err(err) = self.forwarding.register_module_connection_acked(
2474 connection_id,
2475 registration.manifest.module_id.clone(),
2476 negotiated_ver,
2477 concurrency,
2478 sink,
2479 hello_ack,
2480 ) {
2481 if matches!(
2486 self.deregister_connection(
2487 connection_id,
2488 RegistrationEndReason::RegistrationFailed,
2489 ),
2490 Ok(r) if !r.is_empty()
2491 ) {
2492 crate::supervise::notify_registration_release();
2493 }
2494 return Ok(vec![control_error_frame(
2495 &frame,
2496 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2497 "invalid_hello"
2498 } else {
2499 forwarding_error_code(&err)
2500 },
2501 err.to_string(),
2502 )?]);
2503 }
2504 Vec::new()
2505 } else {
2506 vec![hello_ack]
2509 };
2510
2511 if !self
2516 .registry
2517 .publish_registration(connection_id)
2518 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2519 {
2520 debug!(
2521 module_id = %registration.manifest.module_id,
2522 connection_id = connection_id.get(),
2523 "registration was removed before it could be published"
2524 );
2525 }
2526
2527 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2536 info!(
2537 module_id = %registration.manifest.module_id,
2538 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2539 );
2540 }
2541
2542 self.apply_registration_capabilities(®istration);
2543
2544 self.supervisor
2545 .record_activity_registration(®istration.manifest.module_id);
2546
2547 info!(
2548 module_id = %registration.manifest.module_id,
2549 module_version = %registration.manifest.module_version,
2550 negotiated_ver,
2551 routable_provider = manifest_provides_routable_role(®istration.manifest),
2552 connection_id = connection_id.get(),
2553 "module registered"
2554 );
2555
2556 Ok(reply)
2557 }
2558
2559 #[allow(clippy::too_many_arguments)]
2576 fn register_swap_candidate(
2577 &self,
2578 connection_id: ConnectionId,
2579 sink: Option<crate::FrameSink>,
2580 frame: &Frame,
2581 manifest: ModuleManifest,
2582 negotiated_ver: u8,
2583 control_ops: Vec<String>,
2584 hello_ack: Frame,
2585 ) -> Result<Vec<Frame>, RouterError> {
2586 let module_id = manifest.module_id.clone();
2587 let registration = match self.registry.register_unpublished_candidate_with_control_ops(
2588 manifest,
2589 negotiated_ver,
2590 connection_id,
2591 control_ops,
2592 ) {
2593 Ok(registration) => registration,
2594 Err(RegistryError::DuplicateModuleId { module_id }) => {
2595 return Ok(vec![control_error_frame(
2596 frame,
2597 "duplicate_module_id",
2598 format!(
2599 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2600 ),
2601 )?])
2602 }
2603 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2604 return Ok(vec![control_error_frame(
2605 frame,
2606 "invalid_module_id",
2607 err.to_string(),
2608 )?])
2609 }
2610 Err(err) => {
2611 return Ok(vec![control_error_frame(
2612 frame,
2613 "registry_error",
2614 err.to_string(),
2615 )?])
2616 }
2617 };
2618 #[cfg(test)]
2619 self.hold_between_registry_and_forwarding();
2620 let reply = if let Some(sink) = sink {
2621 let concurrency = manifest_concurrency(®istration.manifest);
2625 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2626 connection_id,
2627 module_id.clone(),
2628 negotiated_ver,
2629 concurrency,
2630 sink,
2631 hello_ack,
2632 ) {
2633 if matches!(
2634 self.deregister_connection(
2635 connection_id,
2636 RegistrationEndReason::RegistrationFailed,
2637 ),
2638 Ok(r) if !r.is_empty()
2639 ) {
2640 crate::supervise::notify_registration_release();
2641 }
2642 return Ok(vec![control_error_frame(
2643 frame,
2644 forwarding_error_code(&err),
2645 err.to_string(),
2646 )?]);
2647 }
2648 Vec::new()
2649 } else {
2650 vec![hello_ack]
2651 };
2652 if !self
2653 .registry
2654 .publish_registration(connection_id)
2655 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2656 {
2657 debug!(
2658 module_id = %module_id,
2659 connection_id = connection_id.get(),
2660 "swap candidate registration was removed before it could be published"
2661 );
2662 }
2663 self.supervisor.mark_swap_candidate_admitted(&module_id);
2664 info!(
2665 module_id = %module_id,
2666 module_version = %registration.manifest.module_version,
2667 negotiated_ver,
2668 ready = registration.ready,
2669 connection_id = connection_id.get(),
2670 "swap candidate registered; not routable until cutover"
2671 );
2672 Ok(reply)
2673 }
2674
2675 fn build_hello_ack(
2676 &self,
2677 frame: &Frame,
2678 negotiated_ver: u8,
2679 module_id: &str,
2680 ) -> Result<Frame, RouterError> {
2681 let ack = ModuleHelloAckBody {
2682 negotiated_ver,
2683 subc_ops: module_subc_ops(),
2684 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2685 storage: self
2686 .storage_config
2687 .as_ref()
2688 .map(|cfg| cfg.descriptor_for(module_id)),
2689 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2690 };
2691 let body = serde_json::to_vec(&ack).map_err(|err| {
2692 RouterError::backend(
2693 0,
2694 frame.header.corr,
2695 format!("failed to encode HELLO_ACK: {err}"),
2696 )
2697 })?;
2698
2699 Frame::build_with_version(
2700 negotiated_ver,
2701 FrameType::HelloAck,
2702 control_flags(),
2703 0,
2704 0,
2705 frame.header.corr,
2706 body,
2707 )
2708 .map_err(RouterError::FrameBuild)
2709 }
2710
2711 async fn handle_client_control_request(
2712 &self,
2713 ctx: &RouteCtx,
2714 frame: Frame,
2715 request: ClientControlRequest,
2716 route_open_timings: &mut Option<RouteOpenTimings>,
2717 ) -> Result<Vec<Frame>, RouterError> {
2718 if let Some(operation) = privileged_operation(&request) {
2719 let caller = self.resolve_caller(ctx.connection_id).await;
2720 let op = client_control_request_op(&request);
2721 let text = prompt_text(operation, &caller);
2722 match self.privileged_operator_prompt {
2723 PrivilegedOperatorPrompt::Log => {
2724 info!(target: "subc_daemon::privileged_prompt", operation = op,
2725 target = ?operation, caller = %caller.description(), summary = %text,
2726 "privileged request would prompt the operator");
2727 }
2728 PrivilegedOperatorPrompt::Enforce => {
2729 let key =
2730 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2731 let outcome = self
2732 .forwarding
2733 .operator_confirms()
2734 .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2735 .await;
2736 if outcome.code != "confirmed" {
2737 return Ok(vec![outcome.frame(&frame)]);
2738 }
2739 }
2740 }
2741 }
2742 match request {
2743 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2744 ClientControlRequest::CatalogList { module_id } => {
2745 self.handle_catalog_list(frame, module_id)
2746 }
2747 ClientControlRequest::RouteOpen {
2748 target,
2749 identity,
2750 consumer_identity,
2751 consumer_capabilities,
2752 role_versions,
2753 admission_facts,
2754 scope,
2755 } => {
2756 self.handle_route_open(
2757 ctx,
2758 frame,
2759 RouteOpenRequest {
2760 target,
2761 identity,
2762 consumer_identity,
2763 consumer_capabilities,
2764 role_versions,
2765 admission_facts,
2766 scope,
2767 },
2768 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2769 )
2770 .await
2771 }
2772 ClientControlRequest::RoutePoll {
2773 route_channel,
2774 route_epoch,
2775 kind,
2776 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2777 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2778 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2779 self.handle_supervisor_spawn_snapshot(frame)
2780 }
2781 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2782 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2783 }
2784 ClientControlRequest::SupervisorRestart {
2785 module_id,
2786 drain_timeout_ms,
2787 } => {
2788 self.log_supervisor_request_received(
2789 ctx,
2790 frame.header.corr,
2791 ops::SUPERVISOR_RESTART,
2792 Some(&module_id),
2793 None,
2794 )?;
2795 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2796 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2797 .await
2798 }
2799 ClientControlRequest::SupervisorSwap {
2800 module_id,
2801 ready_timeout_ms,
2802 } => {
2803 self.log_supervisor_request_received(
2804 ctx,
2805 frame.header.corr,
2806 ops::SUPERVISOR_SWAP,
2807 Some(&module_id),
2808 None,
2809 )?;
2810 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2811 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2812 .await
2813 }
2814 ClientControlRequest::SupervisorReload { module_id } => {
2815 self.log_supervisor_request_received(
2816 ctx,
2817 frame.header.corr,
2818 ops::SUPERVISOR_RELOAD,
2819 Some(&module_id),
2820 None,
2821 )?;
2822 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2823 self.handle_supervisor_reload(frame, module_id, requester)
2824 .await
2825 }
2826 ClientControlRequest::SupervisorRescan { preview } => {
2827 if !preview {
2828 self.log_supervisor_request_received(
2829 ctx,
2830 frame.header.corr,
2831 ops::SUPERVISOR_RESCAN,
2832 None,
2833 None,
2834 )?;
2835 }
2836 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2837 self.handle_supervisor_rescan(frame, preview, requester)
2838 .await
2839 }
2840 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2841 self.log_supervisor_request_received(
2842 ctx,
2843 frame.header.corr,
2844 ops::SUPERVISOR_RELEASE_RESERVED,
2845 Some(&module_id),
2846 None,
2847 )?;
2848 self.handle_supervisor_release_reserved(frame, module_id)
2849 .await
2850 }
2851 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2852 self.log_supervisor_request_received(
2853 ctx,
2854 frame.header.corr,
2855 ops::SUPERVISOR_SET_ENABLED,
2856 Some(&module_id),
2857 Some(enabled),
2858 )?;
2859 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2860 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2861 .await
2862 }
2863 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2864 self.handle_supervisor_health_probe(frame, module_id).await
2865 }
2866 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2867 ClientControlRequest::SupervisorRoutes { module_id } => {
2868 self.handle_supervisor_routes(frame, module_id)
2869 }
2870 ClientControlRequest::SupervisorProvenance { module_id } => {
2871 self.handle_supervisor_provenance(frame, module_id).await
2872 }
2873 ClientControlRequest::SupervisorStderrTail {
2874 module_id,
2875 max_lines,
2876 max_bytes,
2877 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2878 ClientControlRequest::SupervisorTerminals { module_id } => {
2879 self.handle_supervisor_terminals(frame, module_id).await
2880 }
2881 }
2882 }
2883
2884 fn log_supervisor_request_received(
2885 &self,
2886 ctx: &RouteCtx,
2887 corr: u64,
2888 op: &'static str,
2889 module_id: Option<&str>,
2890 enabled: Option<bool>,
2891 ) -> Result<(), RouterError> {
2892 let caller = self.supervisor_requester(ctx, corr)?;
2893 let caller = principal_label(&caller);
2894 info!(
2897 target: "control",
2898 op,
2899 module_id,
2900 enabled,
2901 connection_id = ctx.connection_id.get(),
2902 caller = %caller,
2903 "supervisor request received"
2904 );
2905 Ok(())
2906 }
2907
2908 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
2909 Ok(self
2910 .registry
2911 .get_module_by_connection(ctx.connection_id)
2912 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2913 .map(|registration| Principal::Reserved {
2914 module_id: registration.manifest.module_id,
2915 })
2916 .unwrap_or(Principal::Direct))
2917 }
2918
2919 fn handle_module_control_request(
2920 &self,
2921 connection_id: ConnectionId,
2922 frame: Frame,
2923 request: ModuleControlRequestFromModule,
2924 ) -> Result<Vec<Frame>, RouterError> {
2925 match request {
2926 ModuleControlRequestFromModule::CatalogUpdate {
2927 provides,
2928 capabilities,
2929 ready,
2930 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2931 ModuleControlRequestFromModule::LiveRoots {} => {
2932 let registered = self
2933 .registry
2934 .get_module_by_connection(connection_id)
2935 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2936 let Some(registration) = registered else {
2937 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2938 };
2939 let response = self
2940 .forwarding
2941 .live_roots(®istration.manifest.module_id)
2942 .map_err(RouterError::Forwarding)?;
2943 Ok(vec![control_response_body_frame(
2944 &frame,
2945 &response,
2946 "ModuleControlResponseToModule::LiveRoots",
2947 )?])
2948 }
2949 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2950 self.handle_scope_sync(connection_id, frame, generation, scopes)
2951 }
2952 ModuleControlRequestFromModule::ScopeApply {
2953 generation,
2954 upsert,
2955 end,
2956 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
2957 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2958 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2959 }
2960 }
2961 }
2962
2963 fn handle_operator_confirm(
2964 &self,
2965 ctx: &RouteCtx,
2966 frame: Frame,
2967 ) -> Result<Vec<Frame>, RouterError> {
2968 use crate::operator_confirm::{audit, Outcome};
2969 let registration = self
2970 .registry
2971 .get_module_by_connection(ctx.connection_id)
2972 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2973 let Some(registration) = registration else {
2974 let outcome = Outcome::refusal("not_registered");
2975 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2976 return Ok(vec![outcome.frame(&frame)]);
2977 };
2978 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
2979 Ok(request) => request,
2980 Err(_) => {
2981 let outcome = Outcome::refusal("invalid_control_body");
2982 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2983 return Ok(vec![outcome.frame(&frame)]);
2984 }
2985 };
2986 let module_id = registration.manifest.module_id;
2987 let nonce = self
2991 .hello_launch_nonces
2992 .lock()
2993 .unwrap_or_else(|p| p.into_inner())
2994 .nonce(ctx.connection_id)
2995 .map(str::to_owned);
2996 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
2997 self.supervisor
2998 .spawned_consumer_authorized(&module_id, nonce)
2999 });
3000 let confirms = self.forwarding.operator_confirms();
3001 self.forwarding
3002 .with_operator_route(
3003 ctx.connection_id,
3004 request.route_channel,
3005 request.route_epoch,
3006 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3007 )
3008 .map_err(RouterError::Forwarding)
3009 }
3010
3011 fn handle_scope_sync(
3015 &self,
3016 connection_id: ConnectionId,
3017 frame: Frame,
3018 generation: u64,
3019 scopes: Vec<ScopeRecord>,
3020 ) -> Result<Vec<Frame>, RouterError> {
3021 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3022 }
3023
3024 fn handle_scope_change(
3025 &self,
3026 connection_id: ConnectionId,
3027 frame: Frame,
3028 generation: u64,
3029 scopes: Vec<ScopeRecord>,
3030 end: Option<Vec<ScopeEnd>>,
3031 ) -> Result<Vec<Frame>, RouterError> {
3032 let is_apply = end.is_some();
3033 let Some(registration) = self
3034 .registry
3035 .get_module_by_connection(connection_id)
3036 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3037 else {
3038 return Ok(vec![control_error_frame(
3039 &frame,
3040 "not_registered",
3041 "scope.sync and scope.apply require an active module registration owned by this connection",
3042 )?]);
3043 };
3044 let owner = registration.manifest.module_id;
3045 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3046 let is_current_launch = |connection: ConnectionId| {
3047 self.hello_launch_nonces
3048 .lock()
3049 .unwrap_or_else(|poisoned| poisoned.into_inner())
3050 .presented(connection, current_nonce.as_deref())
3051 };
3052 let mut table = self
3057 .scopes
3058 .write()
3059 .unwrap_or_else(|poisoned| poisoned.into_inner());
3060 let outcome = match end {
3061 Some(end) => table.apply(
3062 &owner,
3063 connection_id,
3064 is_current_launch,
3065 generation,
3066 scopes,
3067 end,
3068 ),
3069 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3070 };
3071 let drained = match &outcome {
3072 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3073 Err(_) => Vec::new(),
3074 };
3075 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3076 drop(table);
3077 if has_deadlines {
3078 self.scope_deadline_added.notify_one();
3079 }
3080 match outcome {
3081 Ok(applied) => {
3082 let counts = ScopeOutcomeCounts::of(&applied.results);
3083 info!(
3084 owner = %owner,
3085 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3086 generation,
3087 records = applied.results.len(),
3088 created = counts.created,
3089 replaced = counts.replaced,
3090 updated = counts.updated,
3091 unchanged = counts.unchanged,
3092 refused = counts.refused,
3093 ended = applied.ended.len(),
3094 tag_changes = applied.tag_changes.len(),
3095 routes_closed = drained.len(),
3096 "scope change accepted"
3097 );
3098 for refused in applied
3104 .results
3105 .iter()
3106 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3107 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3108 {
3109 warn!(
3110 owner = %owner,
3111 generation,
3112 scope_ref = %refused.scope_ref,
3113 scope_epoch = refused.scope_epoch,
3114 code = refused.code.as_deref().unwrap_or(""),
3115 "scope record refused"
3116 );
3117 }
3118 self.close_scope_drained_routes(drained);
3119 let response = if is_apply {
3120 ModuleControlResponseToModule::ScopeApply {
3121 generation,
3122 results: applied.results,
3123 end_results: applied.end_results,
3124 ended: applied.ended,
3125 }
3126 } else {
3127 ModuleControlResponseToModule::ScopeSync {
3128 generation,
3129 results: applied.results,
3130 ended: applied.ended,
3131 }
3132 };
3133 Ok(vec![control_response_body_frame(
3134 &frame,
3135 &response,
3136 "scope change response",
3137 )?])
3138 }
3139 Err(refusal) => {
3140 info!(
3141 owner = %owner,
3142 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3143 generation,
3144 code = refusal.code,
3145 "scope change refused"
3146 );
3147 Ok(vec![control_error_frame(
3148 &frame,
3149 refusal.code,
3150 refusal.message,
3151 )?])
3152 }
3153 }
3154 }
3155
3156 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3162 if drained.is_empty() {
3163 return;
3164 }
3165 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3166 BTreeMap::new();
3167 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3168 for route in drained {
3169 warn!(
3170 module_id = %route.module_id,
3171 reason = ?route.reason,
3172 client_connection_id = route.client.connection_id.get(),
3173 route_channel = route.client.channel,
3174 "closing route because its scope changed"
3175 );
3176 pushes
3177 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3178 .or_insert_with(|| (route.reason, Vec::new()))
3179 .1
3180 .push(EndpointRoute {
3181 goodbye_target: route.client.clone(),
3182 principal: Principal::Unverified,
3183 bound_at: Instant::now(),
3184 draining: false,
3185 drain_reason: None,
3186 });
3187 goodbyes.push(route.module);
3188 goodbyes.push(route.client);
3189 }
3190 for ((module_id, _), (reason, routes)) in pushes {
3191 send_route_control_pushes(
3192 &self.forwarding,
3193 routes,
3194 ClientControlPush::RouteClosed {
3195 module_id,
3196 channels: Vec::new(),
3197 reason,
3198 drained: false,
3199 abandoned: 0,
3200 excluded_subscriptions: 0,
3201 terminal: Some(false),
3202 },
3203 );
3204 }
3205 self.emit_route_goodbyes(goodbyes);
3206 }
3207
3208 fn handle_scope_describe(
3211 &self,
3212 connection_id: ConnectionId,
3213 frame: Frame,
3214 owner: Principal,
3215 scope_ref: String,
3216 ) -> Result<Vec<Frame>, RouterError> {
3217 let registered = self
3218 .registry
3219 .get_module_by_connection(connection_id)
3220 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3221 if registered.is_none() {
3222 return Ok(vec![control_error_frame(
3223 &frame,
3224 "not_registered",
3225 "scope.describe requires an active module registration owned by this connection",
3226 )?]);
3227 }
3228 let description = self
3229 .scopes
3230 .read()
3231 .unwrap_or_else(|poisoned| poisoned.into_inner())
3232 .describe(&owner, &scope_ref);
3233 let owner_configured = match &owner {
3234 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3239 _ => false,
3240 };
3241 let response = ModuleControlResponseToModule::ScopeDescribe {
3242 status: description.status,
3243 scope_epoch: description.scope_epoch,
3244 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3245 owner_synced: description.owner_synced,
3246 owner_configured,
3247 scope: description.stamp,
3248 };
3249 Ok(vec![control_response_body_frame(
3250 &frame,
3251 &response,
3252 "ModuleControlResponseToModule::ScopeDescribe",
3253 )?])
3254 }
3255
3256 fn handle_catalog_update(
3257 &self,
3258 connection_id: ConnectionId,
3259 frame: Frame,
3260 provides: Vec<ProviderRole>,
3261 capabilities: Option<CapabilityDeclarations>,
3262 ready: Option<bool>,
3263 ) -> Result<Vec<Frame>, RouterError> {
3264 self.refresh_capability_requirements();
3265 let Some(registration) = self
3266 .registry
3267 .get_module_by_connection(connection_id)
3268 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3269 else {
3270 return Ok(vec![control_error_frame(
3271 &frame,
3272 "not_registered",
3273 "catalog.update requires an active module registration owned by this connection",
3274 )?]);
3275 };
3276
3277 if let Some(message) =
3278 catalog_update_frozen_field_message(®istration.manifest, &provides)
3279 {
3280 return Ok(vec![control_error_frame(
3281 &frame,
3282 "catalog_update_frozen_field",
3283 message,
3284 )?]);
3285 }
3286
3287 let mut candidate = registration.manifest.clone();
3288 candidate.provides = provides.clone();
3289 candidate.capabilities = capabilities
3290 .clone()
3291 .or_else(|| registration.manifest.capabilities.clone());
3292 if let Err(err) = candidate.validate_capability_grammar() {
3293 return Ok(vec![control_error_frame(
3294 &frame,
3295 "invalid_capability_grammar",
3296 err.to_string(),
3297 )?]);
3298 }
3299
3300 let mut conflicts = self
3303 .capability_evaluator
3304 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3305 if let Some(conflict) = conflicts.first() {
3306 let message = format!(
3307 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3308 conflict.capability, conflict.claimants[0], candidate.module_id
3309 );
3310 for conflict in &mut conflicts {
3311 conflict.source = DuplicateClaimSource::CatalogUpdate;
3312 }
3313 log_duplicate_claim_events(conflicts);
3314 return Ok(vec![control_error_frame(
3315 &frame,
3316 "reserved_capability",
3317 message,
3318 )?]);
3319 }
3320
3321 let updated = self
3322 .registry
3323 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3324 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3325 if updated.is_none() {
3326 return Ok(vec![control_error_frame(
3327 &frame,
3328 "not_registered",
3329 "catalog.update requires an active module registration owned by this connection",
3330 )?]);
3331 }
3332 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3333 log_duplicate_claim_events(
3334 self.capability_evaluator
3335 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3336 );
3337 }
3338 if capability_census_trigger(
3339 registration.manifest.capabilities.as_ref(),
3340 updated
3341 .as_ref()
3342 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3343 ) {
3344 self.enforce_capability_denies();
3345 }
3346 self.refresh_capability_requirements();
3347
3348 let response = ModuleControlResponseToModule::CatalogUpdate {};
3349 control_response_body_frame(
3350 &frame,
3351 &response,
3352 "ModuleControlResponseToModule::CatalogUpdate",
3353 )
3354 .map(|frame| vec![frame])
3355 }
3356
3357 pub(crate) fn with_activity_ledger_config(
3358 mut self,
3359 run_dir: &Path,
3360 incarnation: String,
3361 config: crate::activity::ActivityConfig,
3362 ) -> Self {
3363 self.activity = Some(crate::activity::ActivityLedger::open_config(
3364 run_dir.join("activity.jsonl"),
3365 incarnation,
3366 config,
3367 ));
3368 self.supervisor.set_activity(self.activity.clone());
3369 self
3370 }
3371
3372 pub(crate) async fn configure_activity(
3373 &self,
3374 config: crate::activity::ActivityConfig,
3375 ) -> Result<(), &'static str> {
3376 match &self.activity {
3377 Some(ledger) => ledger.configure(config).await,
3378 None => Ok(()),
3379 }
3380 }
3381
3382 fn cancel_activity_subscription(
3383 &self,
3384 ctx: &RouteCtx,
3385 frame: &Frame,
3386 ) -> Result<bool, RouterError> {
3387 let task = self
3388 .activity_streams
3389 .lock()
3390 .unwrap_or_else(|p| p.into_inner())
3391 .remove(&(ctx.connection_id, frame.header.corr));
3392 let Some(task) = task else {
3393 return Ok(false);
3394 };
3395 task.abort();
3396 let end = Frame::build_with_version(
3397 response_version(frame),
3398 FrameType::StreamEnd,
3399 control_flags(),
3400 0,
3401 0,
3402 frame.header.corr,
3403 Vec::new(),
3404 )
3405 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3406 let sink = ctx.egress.clone();
3407 tokio::spawn(async move {
3408 let _ = sink.send(end).await;
3409 });
3410 Ok(true)
3411 }
3412
3413 async fn handle_activity_read(
3414 &self,
3415 ctx: &RouteCtx,
3416 frame: Frame,
3417 ) -> Result<Vec<Frame>, RouterError> {
3418 use subc_control::{
3419 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3420 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3421 };
3422 #[derive(Deserialize)]
3423 struct Identity {
3424 #[serde(default)]
3425 consumer_identity: Option<ConsumerIdentity>,
3426 }
3427 let principal = match self
3431 .forwarding
3432 .module_id_for_connection(ctx.connection_id)
3433 .map_err(RouterError::Forwarding)?
3434 {
3435 Some(module_id) => Ok(Principal::Reserved { module_id }),
3436 None => {
3437 let identity: Identity = match serde_json::from_slice(&frame.body) {
3438 Ok(identity) => identity,
3439 Err(_) => {
3440 return Ok(vec![control_error_frame(
3441 &frame,
3442 ACTIVITY_MALFORMED,
3443 "malformed activity read",
3444 )?])
3445 }
3446 };
3447 self.route_open_principal(&frame, identity.consumer_identity)?
3448 }
3449 };
3450 let admitted = matches!(principal, Ok(Principal::Direct))
3451 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3452 if !admitted {
3453 return Ok(vec![control_error_frame(
3454 &frame,
3455 ACTIVITY_READ_FORBIDDEN,
3456 "activity reads require an operator or trusted mirror principal",
3457 )?]);
3458 }
3459 let Some(ledger) = &self.activity else {
3460 return Ok(vec![control_error_frame(
3461 &frame,
3462 ACTIVITY_UNAVAILABLE,
3463 "activity ledger unavailable",
3464 )?]);
3465 };
3466 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3467 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3468 {
3469 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3470 Ok(request) => ledger.query(request).await,
3471 Err(_) => Err(ACTIVITY_MALFORMED),
3472 };
3473 return match result {
3474 Ok(reply) => Ok(vec![control_response_body_frame(
3475 &frame,
3476 &reply,
3477 ops::ACTIVITY_QUERY,
3478 )?]),
3479 Err(code) => Ok(vec![control_error_frame(
3480 &frame,
3481 code,
3482 "activity query refused",
3483 )?]),
3484 };
3485 }
3486 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3487 Ok(request) => request,
3488 Err(_) => {
3489 return Ok(vec![control_error_frame(
3490 &frame,
3491 ACTIVITY_MALFORMED,
3492 "malformed activity subscription",
3493 )?])
3494 }
3495 };
3496 let mut subscription = match ledger.subscribe(request.since).await {
3497 Ok(subscription) => subscription,
3498 Err(code) => {
3499 return Ok(vec![control_error_frame(
3500 &frame,
3501 code,
3502 "activity subscription refused",
3503 )?])
3504 }
3505 };
3506 let key = (ctx.connection_id, frame.header.corr);
3507 let sink = ctx.egress.clone();
3508 let mut streams = self
3509 .activity_streams
3510 .lock()
3511 .unwrap_or_else(|p| p.into_inner());
3512 streams.retain(|_, task| !task.is_finished());
3513 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3514 return Ok(vec![control_error_frame(
3515 &frame,
3516 "duplicate_subscription",
3517 "activity subscription correlation id already in use",
3518 )?]);
3519 }
3520 let task = tokio::spawn(async move {
3521 loop {
3522 let entry = if let Some(entry) = subscription.replay.pop_front() {
3523 Some(entry)
3524 } else {
3525 subscription.receiver.recv().await
3526 };
3527 let Some(entry) = entry else {
3528 break;
3529 };
3530 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3531 return;
3532 };
3533 let Ok(next) = Frame::build_with_version(
3534 response_version(&frame),
3535 FrameType::StreamData,
3536 control_flags(),
3537 0,
3538 0,
3539 frame.header.corr,
3540 body,
3541 ) else {
3542 return;
3543 };
3544 if sink.send(next).await.is_err() {
3545 return;
3546 }
3547 }
3548 if let Some(code) = subscription.terminal_code() {
3551 if let Ok(end) =
3552 control_error_frame(&frame, code, "activity subscriber fell behind")
3553 {
3554 let _ = sink.send(end).await;
3555 }
3556 }
3557 });
3558 streams.insert(key, task.abort_handle());
3559 Ok(Vec::new())
3560 }
3561
3562 async fn handle_activity_record(
3563 &self,
3564 ctx: &RouteCtx,
3565 frame: Frame,
3566 ) -> Result<Vec<Frame>, RouterError> {
3567 use subc_control::{
3568 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3569 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3570 };
3571 let result = async {
3572 let module = self
3573 .forwarding
3574 .module_id_for_connection(ctx.connection_id)
3575 .ok()
3576 .flatten()
3577 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3578 let request: ActivityRecordRequest =
3579 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3580 crate::activity::validate(&module, &request)?;
3581 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3582 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3583 let at_ms = std::time::SystemTime::now()
3584 .duration_since(std::time::UNIX_EPOCH)
3585 .unwrap_or_default()
3586 .as_millis() as u64;
3587 ledger.record(module, request, actor, at_ms).await
3588 }
3589 .await;
3590 match result {
3591 Ok(entry) => Ok(vec![control_response_body_frame(
3592 &frame,
3593 &ActivityRecordResponse { seq: entry.seq },
3594 "activity.record",
3595 )?]),
3596 Err(code) => Ok(vec![control_error_frame(
3597 &frame,
3598 code,
3599 "activity recording refused",
3600 )?]),
3601 }
3602 }
3603
3604 fn activity_actor(
3605 &self,
3606 connection: ConnectionId,
3607 module: &str,
3608 on: Option<&subc_control::ActivityOn>,
3609 ) -> Result<subc_control::ActivityActor, &'static str> {
3610 use crate::forwarding::{DataRoute, DataRouteState};
3611 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3612 let principal = |id: &str| Principal::Reserved {
3613 module_id: id.to_owned(),
3614 };
3615 let mut actor = ActivityActor {
3616 principal: principal(module),
3617 scope_owner: None,
3618 agent_id: None,
3619 run_id: None,
3620 flow_id: None,
3621 project_root: None,
3622 };
3623 let scope = match on {
3624 None => return Ok(actor),
3625 Some(ActivityOn::Scope(scope)) => {
3626 let selector = subc_protocol::scope::ScopeSelector::new(
3627 principal(module),
3628 scope.scope_ref.clone(),
3629 Some(scope.epoch),
3630 );
3631 self.scopes
3632 .read()
3633 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3634 .attribute(&principal(module), module, &selector, true)
3635 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3636 }
3637 Some(ActivityOn::Route(route)) => {
3638 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3639 .forwarding
3640 .lookup_data_route(connection, route.channel, route.epoch)
3641 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3642 else {
3643 return Err(ACTIVITY_ACTOR_UNKNOWN);
3644 };
3645 if binding.module_id != module {
3646 return Err(ACTIVITY_ACTOR_UNKNOWN);
3647 }
3648 actor.principal = binding.principal.clone();
3649 actor.project_root = binding
3650 .project_root
3651 .as_ref()
3652 .map(|root| root.as_path().to_path_buf());
3653 let Some(bound) = &binding.scope else {
3654 return Ok(actor);
3655 };
3656 let owner = principal(&bound.owner);
3657 let selector = subc_protocol::scope::ScopeSelector::new(
3658 owner.clone(),
3659 bound.scope_ref.clone(),
3660 Some(bound.tag.scope_epoch),
3661 );
3662 let admitted = self
3663 .scopes
3664 .read()
3665 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3666 .attribute(&owner, module, &selector, true)
3667 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3668 if admitted.tag != bound.tag {
3669 return Err(ACTIVITY_ACTOR_UNKNOWN);
3670 }
3671 admitted
3672 }
3673 };
3674 actor.scope_owner = Some(scope.stamp.owner);
3675 actor.agent_id = scope.stamp.attributes.agent_id;
3676 actor.run_id = scope.stamp.attributes.run_id;
3677 actor.flow_id = scope.stamp.attributes.flow_id;
3678 Ok(actor)
3679 }
3680
3681 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3682 self.refresh_capability_requirements();
3683 let mut counters = self.counters.snapshot();
3689 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3690 obj.insert(
3691 "activity_unparseable_lines".into(),
3692 ledger.read_errors().into(),
3693 );
3694 obj.insert(
3695 "activity_lifecycle_append_failures".into(),
3696 ledger.lifecycle_failures().into(),
3697 );
3698 obj.insert(
3699 "activity_started_digest_skips".into(),
3700 ledger.lifecycle_digest_skips().into(),
3701 );
3702 }
3703 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3704 self.forwarding.client_route_concentration(),
3705 counters.as_object_mut(),
3706 ) {
3707 obj.insert(
3708 "client_connections_with_routes".into(),
3709 connections_with_routes.into(),
3710 );
3711 obj.insert("max_routes_on_one_connection".into(), max.into());
3712 }
3713 if let (Some(open_breakers), Some(obj)) = (
3719 self.route_bind_breakers.open_snapshot(),
3720 counters.as_object_mut(),
3721 ) {
3722 obj.insert("route_bind_breakers_open".into(), open_breakers);
3723 }
3724 let response = ClientControlResponse::ServerDescribe {
3725 protocol_ver: PROTOCOL_VERSION,
3726 subc_ops: subc_ops(),
3727 capabilities: self.subc_capabilities.as_ref().to_vec(),
3728 connected_clients: self.connected_clients.count(),
3729 counters: Some(counters),
3730 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3731 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3732 capability_requirements: self.capability_requirement_statuses(),
3733 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3734 };
3735 Ok(vec![control_response_body_frame(
3736 &frame,
3737 &response,
3738 "ClientControlResponse::ServerDescribe",
3739 )?])
3740 }
3741
3742 fn handle_catalog_list(
3743 &self,
3744 frame: Frame,
3745 module_id: Option<String>,
3746 ) -> Result<Vec<Frame>, RouterError> {
3747 let (generation, modules) = self.registry.list_modules().map_err(|err| {
3748 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3749 })?;
3750 let entries = modules
3751 .into_iter()
3752 .filter(|registration| {
3753 module_id
3754 .as_deref()
3755 .map(|wanted| registration.manifest.module_id == wanted)
3756 .unwrap_or(true)
3757 })
3758 .map(|registration| {
3759 let not_ready = self.not_ready_reason(®istration);
3760 let roles = registration.manifest.provides;
3761 CatalogEntry::new(
3762 registration.manifest.module_id,
3763 roles,
3764 registration.control_ops,
3765 )
3766 .with_ready(not_ready.is_none())
3767 .with_not_ready(not_ready)
3768 .with_module_version(Some(registration.manifest.module_version))
3769 .with_capabilities(registration.manifest.capabilities)
3770 .with_self_signals(registration.manifest.self_signals)
3771 .with_events(registration.manifest.events)
3772 })
3773 .collect();
3774 let response = ClientControlResponse::CatalogList {
3775 generation,
3776 modules: entries,
3777 subc_ops: subc_ops(),
3778 };
3779 Ok(vec![control_response_body_frame(
3780 &frame,
3781 &response,
3782 "ClientControlResponse::CatalogList",
3783 )?])
3784 }
3785
3786 fn route_open_principal(
3787 &self,
3788 frame: &Frame,
3789 consumer_identity: Option<ConsumerIdentity>,
3790 ) -> Result<Result<Principal, Frame>, RouterError> {
3791 let Some(consumer_identity) = consumer_identity else {
3792 return Ok(Ok(Principal::Direct));
3793 };
3794
3795 if self.supervisor.spawned_consumer_authorized(
3796 &consumer_identity.module_id,
3797 &consumer_identity.launch_nonce,
3798 ) {
3799 return Ok(Ok(Principal::Reserved {
3800 module_id: consumer_identity.module_id,
3801 }));
3802 }
3803
3804 Ok(Err(control_error_frame(
3805 frame,
3806 "bad_consumer_identity",
3807 format!(
3808 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3809 consumer_identity.module_id
3810 ),
3811 )?))
3812 }
3813
3814 fn route_open_refusal_frame(
3821 &self,
3822 ctx: &RouteCtx,
3823 frame: &Frame,
3824 module_id: &str,
3825 reason: &'static str,
3826 code: &'static str,
3827 message: impl Into<String>,
3828 ) -> Result<Frame, RouterError> {
3829 self.observe_route_open_refusal(ctx, module_id, reason, code);
3830 control_error_frame(frame, code, message.into())
3831 }
3832
3833 fn route_open_breaker_refusal_frame(
3851 &self,
3852 ctx: &RouteCtx,
3853 frame: &Frame,
3854 module_id: &str,
3855 consecutive_timeouts: u32,
3856 retry_in: Duration,
3857 probe_in_flight: bool,
3858 ) -> Result<Frame, RouterError> {
3859 self.counters
3860 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
3861 debug!(
3862 target: "control",
3863 code = "module_timeout",
3864 module_id = ?module_id,
3865 connection_id = ctx.connection_id.get(),
3866 consecutive_timeouts,
3867 retry_in_ms = retry_in.as_millis() as u64,
3868 probe_in_flight,
3869 "route.open refused by open bind-relay breaker"
3870 );
3871 let detail = if probe_in_flight {
3878 "one test route is already being tried; retry once it settles".to_string()
3879 } else {
3880 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
3881 };
3882 control_error_frame(
3883 frame,
3884 "module_timeout",
3885 format!(
3886 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
3887 timed out in a row); {detail}; its established routes are unaffected"
3888 ),
3889 )
3890 }
3891
3892 fn observe_route_open_refusal(
3905 &self,
3906 ctx: &RouteCtx,
3907 module_id: &str,
3908 reason: &'static str,
3909 code: &'static str,
3910 ) {
3911 self.counters.increment_route_open_refused(code);
3912 info!(
3913 target: "control",
3914 code,
3915 reason,
3916 module_id = ?module_id,
3917 connection_id = ctx.connection_id.get(),
3918 "route.open refused"
3919 );
3920 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
3921 self.route_outages.record_not_serving(module_id, reason);
3922 }
3923 }
3924
3925 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
3970 self.route_outages.record_accepted(module_id);
3971 self.counters.increment_route_open_accepted(principal);
3972 info!(
3973 target: "control",
3974 principal,
3975 module_id,
3976 connection_id = ctx.connection_id.get(),
3977 "route.open accepted"
3978 );
3979 }
3980
3981 fn supervised_absent_route_open_refusal_frame(
3982 &self,
3983 ctx: &RouteCtx,
3984 frame: &Frame,
3985 module_id: &str,
3986 code: &'static str,
3987 status: &crate::supervise::ModuleStatus,
3988 ) -> Result<Frame, RouterError> {
3989 self.counters.increment_route_open_refused(code);
3990 info!(
3991 target: "control",
3992 code,
3993 reason = "supervised_not_registered",
3994 module_id = ?module_id,
3995 connection_id = ctx.connection_id.get(),
3996 state = %status.state,
3997 enabled = status.enabled,
3998 live = status.live,
3999 "route.open refused"
4000 );
4001 self.route_outages
4005 .record_not_serving(module_id, "supervised_not_registered");
4006 let message = if let Some(pid) = status.waiting_for_previous_pid {
4007 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4008 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4009 })
4010 } else {
4011 format!(
4012 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4013 status.state, status.enabled, status.live
4014 )
4015 };
4016 control_error_frame(frame, code, message)
4017 }
4018
4019 async fn handle_route_open(
4020 &self,
4021 ctx: &RouteCtx,
4022 frame: Frame,
4023 request: RouteOpenRequest,
4024 timings: &mut RouteOpenTimings,
4025 ) -> Result<Vec<Frame>, RouterError> {
4026 let RouteOpenRequest {
4027 target,
4028 mut identity,
4029 consumer_identity,
4030 consumer_capabilities,
4031 role_versions,
4032 admission_facts,
4033 scope,
4034 } = request;
4035 let target_module_id = target_module_id(&target).to_string();
4036 timings.module_id.clone_from(&target_module_id);
4037 if self
4038 .registry
4039 .get_module_by_connection(ctx.connection_id)
4040 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4041 .is_some()
4042 {
4043 return Ok(vec![control_error_frame(
4044 &frame,
4045 "invalid_request",
4046 "module connections cannot open client routes",
4047 )?]);
4048 }
4049 debug!(
4050 connection_id = ctx.connection_id.get(),
4051 corr = frame.header.corr,
4052 module_id = %target_module_id,
4053 "handling route.open"
4054 );
4055
4056 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4062 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4063 self.observe_route_open_refusal(
4064 ctx,
4065 &target_module_id,
4066 "invalid_role_versions",
4067 error_codes::INVALID_REQUEST,
4068 );
4069 return Ok(vec![control_error_body_frame(
4070 &frame,
4071 ErrorBody {
4072 code: error_codes::INVALID_REQUEST.to_string(),
4073 message: error.to_string(),
4074 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4075 },
4076 )?]);
4077 }
4078
4079 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4080 if status.waiting_for_previous_pid.is_some() {
4081 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4082 ctx,
4083 &frame,
4084 &target_module_id,
4085 "module_warming",
4086 &status,
4087 )?]);
4088 }
4089 }
4090
4091 let Some(registration) = self
4110 .registry
4111 .get_module(&target_module_id)
4112 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4113 else {
4114 if let Some((status, warming)) =
4115 self.supervisor_status(&target_module_id, frame.header.corr)?
4116 {
4117 if status.protocol == ModuleProtocol::None {
4124 return Ok(vec![self.route_open_refusal_frame(
4125 ctx,
4126 &frame,
4127 &target_module_id,
4128 "protocol_none",
4129 error_codes::MODULE_NO_PROTOCOL,
4130 format!(
4131 "module_id '{target_module_id}' is declared protocol: none; \
4132 it speaks no subc wire and serves no routes"
4133 ),
4134 )?]);
4135 }
4136 let code = if warming {
4137 "module_warming"
4138 } else {
4139 "target_unavailable"
4140 };
4141 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4142 ctx,
4143 &frame,
4144 &target_module_id,
4145 code,
4146 &status,
4147 )?]);
4148 }
4149 if let Some(removed_ago_ms) =
4150 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4151 {
4152 return Ok(vec![self.route_open_refusal_frame(
4153 ctx,
4154 &frame,
4155 &target_module_id,
4156 "removed",
4157 error_codes::MODULE_REMOVED,
4158 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4159 )?]);
4160 }
4161 return Ok(vec![self.route_open_refusal_frame(
4162 ctx,
4163 &frame,
4164 &target_module_id,
4165 "not_registered",
4166 error_codes::UNKNOWN_MODULE,
4167 format!("module_id '{target_module_id}' is not registered"),
4168 )?]);
4169 };
4170
4171 if !registration.ready {
4175 self.counters
4176 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4177 info!(
4178 target: "control",
4179 code = error_codes::MODULE_WARMING,
4180 module_id = ?target_module_id,
4181 connection_id = ctx.connection_id.get(),
4182 reason = "declared_not_ready",
4183 "route.open refused"
4184 );
4185 self.route_outages
4188 .record_not_serving(&target_module_id, "declared_not_ready");
4189 return Ok(vec![control_error_body_frame(
4190 &frame,
4191 ErrorBody {
4192 code: error_codes::MODULE_WARMING.to_string(),
4193 message: format!(
4194 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4195 ),
4196 detail: Some(serde_json::json!({
4197 "reason": "declared_not_ready"
4198 })),
4199 },
4200 )?]);
4201 }
4202
4203 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4222 self.counters
4223 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4224 info!(
4225 target: "control",
4226 code = error_codes::MODULE_WARMING,
4227 module_id = ?target_module_id,
4228 connection_id = ctx.connection_id.get(),
4229 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4230 capability = %capability,
4231 "route.open refused"
4232 );
4233 return Ok(vec![control_error_body_frame(
4234 &frame,
4235 ErrorBody {
4236 code: error_codes::MODULE_WARMING.to_string(),
4237 message: format!(
4238 "module_id '{target_module_id}' requires capability '{capability}', \
4239 which no registered module provides; retry"
4240 ),
4241 detail: Some(serde_json::json!({
4242 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4243 "capability": capability,
4244 })),
4245 },
4246 )?]);
4247 }
4248
4249 if !target_has_required_role(&target, ®istration.manifest.provides) {
4250 return Ok(vec![self.route_open_refusal_frame(
4251 ctx,
4252 &frame,
4253 &target_module_id,
4254 "role_not_provided",
4255 "target_unavailable",
4256 format!("module_id '{target_module_id}' does not provide the requested target"),
4257 )?]);
4258 }
4259
4260 if registration.state != ChannelState::Active {
4261 return Ok(vec![self.route_open_refusal_frame(
4262 ctx,
4263 &frame,
4264 &target_module_id,
4265 "registration_not_active",
4266 "target_unavailable",
4267 format!("module_id '{target_module_id}' is not active"),
4268 )?]);
4269 }
4270
4271 if self
4272 .forwarding
4273 .module_is_draining(&target_module_id)
4274 .map_err(RouterError::Forwarding)?
4275 {
4276 return Ok(vec![self.route_open_refusal_frame(
4277 ctx,
4278 &frame,
4279 &target_module_id,
4280 "reloading",
4281 "module_reloading",
4282 format!("module_id '{target_module_id}' is reloading"),
4283 )?]);
4284 }
4285
4286 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4287 process_liveness.process_live(&target_module_id) == Some(false)
4288 }) {
4289 if process_liveness.process_replacing(&target_module_id) {
4296 return Ok(vec![self.route_open_refusal_frame(
4297 ctx,
4298 &frame,
4299 &target_module_id,
4300 "reloading",
4301 "module_reloading",
4302 format!("module_id '{target_module_id}' is reloading"),
4303 )?]);
4304 }
4305 return Ok(vec![self.route_open_refusal_frame(
4306 ctx,
4307 &frame,
4308 &target_module_id,
4309 "supervisor_not_live",
4310 "target_unavailable",
4311 format!("module_id '{target_module_id}' is not live"),
4312 )?]);
4313 }
4314
4315 if !self
4316 .forwarding
4317 .has_live_module_connection(&target_module_id)
4318 .map_err(RouterError::Forwarding)?
4319 {
4320 return Ok(vec![self.route_open_refusal_frame(
4321 ctx,
4322 &frame,
4323 &target_module_id,
4324 "no_forwarding_connection",
4325 "target_unavailable",
4326 format!("module_id '{target_module_id}' has no live forwarding connection"),
4327 )?]);
4328 }
4329
4330 if let Some(error) =
4331 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4332 {
4333 self.observe_route_open_refusal(
4334 ctx,
4335 &target_module_id,
4336 "op_not_allowed",
4337 "op_not_allowed",
4338 );
4339 return Ok(vec![error]);
4340 }
4341
4342 let principal = match self.route_open_principal(&frame, consumer_identity)? {
4343 Ok(principal) => principal,
4344 Err(error) => {
4345 self.observe_route_open_refusal(
4346 ctx,
4347 &target_module_id,
4348 "bad_consumer_identity",
4349 "bad_consumer_identity",
4350 );
4351 return Ok(vec![error]);
4352 }
4353 };
4354 timings.principal = principal_label(&principal);
4355
4356 if let Principal::Reserved {
4360 module_id: opening_module_id,
4361 } = &principal
4362 {
4363 if let Some(opening_registration) = self
4364 .registry
4365 .get_module(opening_module_id)
4366 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4367 {
4368 if let Some(capability) =
4369 denied_capability(&opening_registration.manifest, ®istration.manifest)
4370 {
4371 warn!(
4372 opening_module_id,
4373 target_module_id,
4374 capability,
4375 "refusing route.open because an attested capability deny edge matches"
4376 );
4377 return Ok(vec![self.route_open_refusal_frame(
4378 ctx,
4379 &frame,
4380 &target_module_id,
4381 "capability_deny_edge",
4382 "capability_forbidden",
4383 format!(
4384 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4385 ),
4386 )?]);
4387 }
4388 }
4389 }
4390
4391 if admission_facts.is_some() {
4392 let carrier_matches = matches!(
4393 &principal,
4394 Principal::Reserved { module_id }
4395 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4396 );
4397 if !carrier_matches {
4398 return Ok(vec![self.route_open_refusal_frame(
4399 ctx,
4400 &frame,
4401 &target_module_id,
4402 "admission_facts_carrier_not_permitted",
4403 "admission_facts_not_permitted",
4404 "admission facts may only be carried by the configured reserved module",
4405 )?]);
4406 }
4407
4408 let target_allowed = self
4409 .admission_facts_targets
4410 .as_ref()
4411 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4412 if !target_allowed {
4413 return Ok(vec![self.route_open_refusal_frame(
4414 ctx,
4415 &frame,
4416 &target_module_id,
4417 "admission_facts_target_not_listed",
4418 "admission_facts_target_not_allowed",
4419 format!(
4420 "admission facts are not permitted for target module_id '{target_module_id}'"
4421 ),
4422 )?]);
4423 }
4424
4425 }
4429
4430 let (bound_scope, mut scope_stamp) = match &scope {
4436 None => (None, None),
4437 Some(selector) => {
4438 let owner_configured = match &selector.owner {
4439 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4445 _ => false,
4446 };
4447 let mut scopes = self
4448 .scopes
4449 .write()
4450 .unwrap_or_else(|poisoned| poisoned.into_inner());
4451 let admitted = scopes
4452 .admit(&principal, &target_module_id, selector, owner_configured)
4453 .and_then(|admission| {
4454 crate::scopes::check_target_flow_support(
4455 &admission.stamp,
4456 &target_module_id,
4457 registration.manifest.capabilities.as_ref(),
4458 )?;
4459 crate::scopes::check_target_agent_run_support(
4460 &admission.stamp,
4461 &target_module_id,
4462 registration.manifest.capabilities.as_ref(),
4463 )?;
4464 let bound = scopes.reserve_bind(
4465 &admission,
4466 &principal,
4467 ctx.connection_id,
4468 selector.hold_presence,
4469 );
4470 Ok((admission, bound))
4471 });
4472 match admitted {
4473 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4474 Err(refusal) => {
4475 return Ok(vec![self.route_open_refusal_frame(
4476 ctx,
4477 &frame,
4478 &target_module_id,
4479 refusal.code,
4480 refusal.code,
4481 refusal.message,
4482 )?]);
4483 }
4484 }
4485 }
4486 };
4487
4488 timings.enter(RouteOpenPhase::ProjectRoot);
4510 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4511 Ok(project_root) => project_root,
4512 Err(err) => {
4513 return Ok(vec![control_error_frame(
4514 &frame,
4515 "invalid_project_root",
4516 err.to_string(),
4517 )?])
4518 }
4519 };
4520 timings.enter(RouteOpenPhase::Admission);
4521 identity.project_root = project_root.as_path().to_path_buf();
4522
4523 let _concurrency_guard = match self
4534 .route_bind_concurrency
4535 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4536 {
4537 Ok(guard) => guard,
4538 Err(in_flight) => {
4539 return Ok(vec![self.route_open_target_capacity_refusal(
4540 ctx,
4541 &frame,
4542 &target_module_id,
4543 in_flight,
4544 )?]);
4545 }
4546 };
4547
4548 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4551 RouteBindAdmission::Admitted { guard, probe } => {
4552 if probe {
4553 info!(
4554 module_id = %target_module_id,
4555 connection_id = ctx.connection_id.get(),
4556 "route.bind breaker half-open: admitting one probe"
4557 );
4558 }
4559 guard
4560 }
4561 RouteBindAdmission::Refused {
4562 consecutive_timeouts,
4563 retry_in,
4564 probe_in_flight,
4565 } => {
4566 return Ok(vec![self.route_open_breaker_refusal_frame(
4567 ctx,
4568 &frame,
4569 &target_module_id,
4570 consecutive_timeouts,
4571 retry_in,
4572 probe_in_flight,
4573 )?]);
4574 }
4575 };
4576
4577 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4582 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4583 timings.enter(RouteOpenPhase::EgressReservation);
4584 let pending = match self
4585 .forwarding
4586 .begin_route_bind_relay_for(
4587 ctx.connection_id,
4588 ctx.egress.clone(),
4589 response_version(&frame),
4590 frame.header.corr,
4591 &target_module_id,
4592 principal.clone(),
4593 bound_scope,
4594 Some(project_root),
4595 relay_deadline,
4596 )
4597 .await
4598 {
4599 Ok(pending) => pending,
4600 Err(err) => {
4601 return Ok(vec![self.route_open_refusal_frame(
4602 ctx,
4603 &frame,
4604 &target_module_id,
4605 "relay_reservation_failed",
4606 forwarding_error_code(&err),
4607 err.to_string(),
4608 )?])
4609 }
4610 };
4611 timings.enter(RouteOpenPhase::Admission);
4612 let crate::forwarding::PendingRouteBindRelay {
4613 endpoint,
4614 module_sink,
4615 negotiated_ver,
4616 client_channel,
4617 client_epoch,
4618 module_channel,
4619 module_epoch,
4620 corr: relay_corr,
4621 receiver,
4622 } = pending;
4623 let mut reservation =
4624 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4625
4626 let relay_registration = self
4631 .registry
4632 .get_module_by_connection(endpoint.connection_id)
4633 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4634 let relay_capabilities = relay_registration
4635 .as_ref()
4636 .and_then(|registration| registration.manifest.capabilities.as_ref());
4637 if let Some(selector) = &scope {
4638 if let Err(refusal) = self
4641 .scopes
4642 .read()
4643 .unwrap_or_else(|poisoned| poisoned.into_inner())
4644 .check_bind_proof(&principal, selector)
4645 {
4646 reservation.release_and_disarm();
4647 return Ok(vec![self.route_open_refusal_frame(
4648 ctx,
4649 &frame,
4650 &target_module_id,
4651 refusal.code,
4652 refusal.code,
4653 refusal.message,
4654 )?]);
4655 }
4656 }
4657 if let Some(stamp) = scope_stamp
4658 .as_ref()
4659 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4660 {
4661 if let Err(refusal) = crate::scopes::check_target_flow_support(
4662 stamp,
4663 &target_module_id,
4664 relay_capabilities,
4665 )
4666 .and_then(|()| {
4667 crate::scopes::check_target_agent_run_support(
4668 stamp,
4669 &target_module_id,
4670 relay_capabilities,
4671 )
4672 }) {
4673 reservation.release_and_disarm();
4674 return Ok(vec![self.route_open_refusal_frame(
4675 ctx,
4676 &frame,
4677 &target_module_id,
4678 refusal.code,
4679 refusal.code,
4680 refusal.message,
4681 )?]);
4682 }
4683 }
4684
4685 let call_mode = if relay_capabilities
4688 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4689 {
4690 Some(derive_call_mode(scope_stamp.as_ref()))
4691 } else {
4692 if let Some(stamp) = scope_stamp.as_mut() {
4693 stamp.expires_at_ms = None;
4694 }
4695 None
4696 };
4697
4698 debug!(
4699 connection_id = ctx.connection_id.get(),
4700 client_channel,
4701 client_epoch,
4702 module_channel,
4703 module_epoch,
4704 "reserved route handle pair"
4705 );
4706 let principal_label = principal_label(&principal);
4709 let relay = ModuleControlRequest::RouteBind {
4710 route_channel: module_channel,
4711 epoch: module_epoch,
4712 target,
4713 identity,
4714 principal: Some(principal),
4715 consumer_capabilities,
4716 role_versions,
4717 admission_facts,
4718 scope: scope_stamp,
4719 call_mode,
4720 };
4721 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4722 RouterError::backend(
4723 0,
4724 frame.header.corr,
4725 format!("failed to encode route.bind request: {err}"),
4726 )
4727 })?;
4728 let relay_frame = Frame::build_with_version(
4729 negotiated_ver,
4730 FrameType::Request,
4731 control_flags(),
4732 0,
4733 0,
4734 relay_corr,
4735 relay_body,
4736 )
4737 .map_err(RouterError::FrameBuild)?;
4738
4739 timings.enter(RouteOpenPhase::BindSend);
4740 let sent = module_sink.send(relay_frame).await;
4741 timings.enter(RouteOpenPhase::Admission);
4742 if let Err(err) = sent {
4743 reservation.release_and_disarm();
4744 return Ok(vec![self.route_open_refusal_frame(
4745 ctx,
4746 &frame,
4747 &target_module_id,
4748 "relay_send_failed",
4749 "target_unavailable",
4750 err.to_string(),
4751 )?]);
4752 }
4753
4754 #[cfg(test)]
4755 if let Some(hold) = &self.route_bind_relay_sent_hold {
4756 hold.wait().await;
4760 hold.wait().await;
4761 }
4762
4763 if self
4768 .forwarding
4769 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4770 .map_err(RouterError::Forwarding)?
4771 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4772 {
4773 self.send_abandoned_route_bind_goodbye(
4774 &module_sink,
4775 negotiated_ver,
4776 module_channel,
4777 module_epoch,
4778 );
4779 }
4780
4781 timings.enter(RouteOpenPhase::BindAckWait);
4782 let outcome = timeout_at(relay_deadline, receiver).await;
4783 timings.enter(RouteOpenPhase::Admission);
4784 match outcome {
4785 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4786 reservation.disarm();
4787 if breaker.record_accepted() {
4788 info!(
4789 module_id = %target_module_id,
4790 "route.bind breaker closed: the probe was accepted"
4791 );
4792 }
4793 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
4794 Ok(Vec::new())
4795 }
4796 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4797 reservation.release_and_disarm();
4798 breaker.record_inconclusive();
4802 let scope_code = match body.code.as_str() {
4806 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4807 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4808 _ => None,
4809 };
4810 if let Some(code) = scope_code {
4811 self.observe_route_open_refusal(
4812 ctx,
4813 &target_module_id,
4814 "scope_changed_before_commit",
4815 code,
4816 );
4817 return Ok(vec![control_error_body_frame(&frame, body)?]);
4818 }
4819 self.counters
4820 .increment_route_open_refused("module_rejected");
4821 info!(
4822 target: "control",
4823 code = "module_rejected",
4824 module_code = ?body.code,
4825 module_id = ?target_module_id,
4826 connection_id = ctx.connection_id.get(),
4827 "route.open refused"
4828 );
4829 Ok(vec![control_error_body_frame(&frame, body)?])
4830 }
4831 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
4832 reservation.release_and_disarm();
4833 breaker.record_inconclusive();
4834 tracing::warn!(
4841 module_id = %target_module_id,
4842 "route.bind relay abandoned: {message}"
4843 );
4844 Ok(vec![self.route_open_refusal_frame(
4845 ctx,
4846 &frame,
4847 &target_module_id,
4848 "relay_abandoned",
4849 "target_unavailable",
4850 message,
4851 )?])
4852 }
4853 Ok(Err(_)) => {
4854 reservation.release_and_disarm();
4855 breaker.record_inconclusive();
4856 Ok(vec![self.route_open_refusal_frame(
4857 ctx,
4858 &frame,
4859 &target_module_id,
4860 "relay_waiter_canceled",
4861 "target_unavailable",
4862 "route.bind relay waiter was canceled before the module responded",
4863 )?])
4864 }
4865 Err(_) => {
4866 reservation.release_and_disarm();
4867 if let Some(opened) = breaker.record_timeout(
4871 self.route_bind_breaker_threshold,
4872 self.route_bind_breaker_cooldown,
4873 ) {
4874 warn!(
4875 module_id = %target_module_id,
4876 consecutive_timeouts = opened.consecutive_timeouts,
4877 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
4878 reopened_after_probe = opened.reopened_after_probe,
4879 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
4880 );
4881 }
4882 tracing::warn!(
4890 module_id = %target_module_id,
4891 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
4892 "route.bind relay timed out: module did not ack within budget"
4893 );
4894 Ok(vec![self.route_open_refusal_frame(
4895 ctx,
4896 &frame,
4897 &target_module_id,
4898 "relay_timed_out",
4899 "module_timeout",
4900 format!(
4901 "module_id '{target_module_id}' did not answer route.bind within {:?}",
4902 route_bind_relay_timeout
4903 ),
4904 )?])
4905 }
4906 }
4907 }
4908
4909 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4910 let response = ClientControlResponse::SupervisorSpawnSnapshot {
4911 snapshot: self.supervisor.spawn_snapshot(),
4912 };
4913 Ok(vec![control_response_body_frame(
4914 &frame,
4915 &response,
4916 "ClientControlResponse::SupervisorSpawnSnapshot",
4917 )?])
4918 }
4919
4920 fn handle_supervisor_spawn_subscribe(
4921 &self,
4922 ctx: &RouteCtx,
4923 frame: Frame,
4924 since: Option<SpawnCursor>,
4925 ) -> Result<Vec<Frame>, RouterError> {
4926 match self.supervisor.subscribe_spawns(
4927 ctx.connection_id,
4928 frame.header.corr,
4929 response_version(&frame),
4930 since,
4931 ctx.egress.clone(),
4932 ) {
4933 Ok(()) => Ok(Vec::new()),
4934 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
4935 Ok(vec![control_error_body_frame(
4936 &frame,
4937 ErrorBody {
4938 code: "spawn_cursor_incarnation_mismatch".to_string(),
4939 message: "spawn cursor belongs to a different daemon incarnation"
4940 .to_string(),
4941 detail: Some(serde_json::json!({
4942 "current_daemon_incarnation": current
4943 })),
4944 },
4945 )?])
4946 }
4947 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
4948 &frame,
4949 ErrorBody {
4950 code: "spawn_cursor_too_old".to_string(),
4951 message: "spawn cursor predates the retained event ring".to_string(),
4952 detail: Some(serde_json::json!({
4953 "oldest_retained_cursor": oldest
4954 })),
4955 },
4956 )?]),
4957 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
4958 0,
4959 frame.header.corr,
4960 format!("failed to open supervisor spawn subscription: {error}"),
4961 )),
4962 }
4963 }
4964
4965 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4966 let generation = self
4967 .registry
4968 .generation()
4969 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4970 let mut modules = Vec::new();
4971 for module in self.supervisor.list() {
4972 let status = module.status_for_control("list").map_err(|err| {
4973 RouterError::backend(
4974 0,
4975 frame.header.corr,
4976 format!("failed to read supervisor status: {err}"),
4977 )
4978 })?;
4979 let (configured, _) = module.configuration().map_err(|err| {
4980 RouterError::backend(
4981 0,
4982 frame.header.corr,
4983 format!("failed to read module configuration: {err}"),
4984 )
4985 })?;
4986 let image = module.running_image_agreement().await;
4988 let resources = Some(module.child_resource_usage());
4991 let pending_reload = Some(reload_verdict(
4992 &configured.program,
4993 status.spawned_from.as_deref(),
4994 image,
4995 ));
4996 modules.push(
4999 SupervisorEntry::new(
5000 status.module_id,
5001 status.state.to_string(),
5002 status.enabled,
5003 status.live,
5004 status.health.status,
5005 )
5006 .with_launch_nonce_env(Some(!cfg!(unix)))
5007 .with_launch_nonce_source(status.launch_nonce_source)
5008 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5009 .with_protocol(status.protocol)
5010 .with_pending_reload(pending_reload)
5011 .with_last_probe_ms(status.health.last_probe_ms)
5012 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5013 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5014 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5015 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5016 .with_restart_count(Some(status.restart_count))
5017 .with_max_restarts(Some(status.max_restarts))
5018 .with_lifetime_restarts(Some(status.lifetime_restarts))
5019 .with_spawn_generation(Some(status.spawn_generation))
5020 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5021 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5022 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5023 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5024 .with_resources(resources),
5025 );
5026 }
5027 let response = ClientControlResponse::SupervisorList {
5028 generation,
5029 modules,
5030 };
5031 Ok(vec![control_response_body_frame(
5032 &frame,
5033 &response,
5034 "ClientControlResponse::SupervisorList",
5035 )?])
5036 }
5037
5038 fn handle_supervisor_stderr_tail(
5039 &self,
5040 frame: Frame,
5041 module_id: String,
5042 max_lines: Option<u32>,
5043 max_bytes: Option<u32>,
5044 ) -> Result<Vec<Frame>, RouterError> {
5045 let Some(module) = self.supervisor.get(&module_id) else {
5046 return Ok(vec![control_error_frame(
5047 &frame,
5048 "unknown_module",
5049 format!("module_id '{module_id}' is not supervised"),
5050 )?]);
5051 };
5052
5053 let snapshot = module.stderr_tail(
5054 max_lines.map(|value| value as usize),
5055 max_bytes.map(|value| value as usize),
5056 );
5057
5058 let response = ClientControlResponse::SupervisorStderrTail {
5059 module_id,
5060 tail: StderrTail {
5061 capture: match snapshot.capture {
5062 CaptureState::Captured => StderrCaptureState::Captured,
5063 CaptureState::Incomplete { reason } => {
5064 StderrCaptureState::Incomplete { reason }
5065 }
5066 CaptureState::NotCaptured { reason } => {
5067 StderrCaptureState::NotCaptured { reason }
5068 }
5069 },
5070 entries: snapshot
5071 .entries
5072 .into_iter()
5073 .map(|entry| match entry {
5074 TailEntry::Line {
5075 text,
5076 truncated,
5077 at_ms,
5078 } => StderrTailEntry::Line {
5079 text,
5080 truncated,
5081 at_ms,
5082 },
5083 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5084 })
5085 .collect(),
5086 dropped_lines: snapshot.dropped_lines,
5087 },
5088 };
5089 Ok(vec![control_response_body_frame(
5090 &frame,
5091 &response,
5092 "ClientControlResponse::SupervisorStderrTail",
5093 )?])
5094 }
5095
5096 async fn handle_supervisor_terminals(
5097 &self,
5098 frame: Frame,
5099 module_id: String,
5100 ) -> Result<Vec<Frame>, RouterError> {
5101 let Some(module) = self.supervisor.get(&module_id) else {
5102 return Ok(vec![control_error_frame(
5103 &frame,
5104 "unknown_module",
5105 format!("module_id '{module_id}' is not supervised"),
5106 )?]);
5107 };
5108
5109 let terminals = module
5112 .read_durable_terminal_history()
5113 .await
5114 .map_err(|error| {
5115 RouterError::backend(
5116 0,
5117 frame.header.corr,
5118 format!("failed to read terminal history: {error}"),
5119 )
5120 })?;
5121 let response = ClientControlResponse::SupervisorTerminals {
5122 module_id,
5123 terminals,
5124 };
5125 Ok(vec![control_response_body_frame(
5126 &frame,
5127 &response,
5128 "ClientControlResponse::SupervisorTerminals",
5129 )?])
5130 }
5131
5132 fn handle_supervisor_routes(
5133 &self,
5134 frame: Frame,
5135 module_id: Option<String>,
5136 ) -> Result<Vec<Frame>, RouterError> {
5137 let modules = self
5138 .forwarding
5139 .route_census(module_id.as_deref())
5140 .map_err(RouterError::Forwarding)?
5141 .into_iter()
5142 .map(|(module_id, routes)| SupervisorRouteModule {
5143 module_id,
5144 routes: routes
5145 .into_iter()
5146 .map(|route| SupervisorRoute {
5147 consumer: match route.principal {
5148 Principal::Reserved { module_id } => {
5149 SupervisorRouteConsumer::Reserved { module_id }
5150 }
5151 Principal::Direct | Principal::Unverified => {
5152 SupervisorRouteConsumer::Direct {
5153 connection_id: route.goodbye_target.connection_id.get(),
5154 }
5155 }
5156 },
5157 age_ms: Instant::now()
5158 .saturating_duration_since(route.bound_at)
5159 .as_millis()
5160 .try_into()
5161 .unwrap_or(u64::MAX),
5162 draining: route.draining,
5163 drain_reason: route.drain_reason,
5164 })
5165 .collect(),
5166 })
5167 .collect();
5168 let response = ClientControlResponse::SupervisorRoutes { modules };
5169 Ok(vec![control_response_body_frame(
5170 &frame,
5171 &response,
5172 "ClientControlResponse::SupervisorRoutes",
5173 )?])
5174 }
5175
5176 async fn handle_supervisor_provenance(
5177 &self,
5178 frame: Frame,
5179 module_id: Option<String>,
5180 ) -> Result<Vec<Frame>, RouterError> {
5181 let mut selected = if let Some(module_id) = module_id {
5182 let Some(module) = self.supervisor.get(&module_id) else {
5183 return Ok(vec![control_error_frame(
5184 &frame,
5185 "unknown_module",
5186 format!("module_id '{module_id}' is not supervised"),
5187 )?]);
5188 };
5189 vec![module]
5190 } else {
5191 self.supervisor.list()
5192 };
5193
5194 let mut modules = Vec::with_capacity(selected.len());
5195 for module in selected.drain(..) {
5196 let (status, observed_image) = module
5197 .status_and_running_image_agreement()
5198 .await
5199 .map_err(|err| {
5200 RouterError::backend(
5201 0,
5202 frame.header.corr,
5203 format!("failed to read supervisor status: {err}"),
5204 )
5205 })?;
5206 let module_declared = self
5207 .registry
5208 .get_module(&status.module_id)
5209 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5210 .and_then(|registration| registration.manifest.provenance)
5211 .map(|build| ModuleDeclaredProvenance::Reported { build })
5212 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5213 #[cfg(test)]
5214 let running_image = match &self.provenance_probe_override {
5215 Some(result) => result.clone(),
5216 None => observed_image,
5217 };
5218 #[cfg(not(test))]
5219 let running_image = observed_image;
5220 modules.push(SupervisorModuleProvenance {
5221 module_id: status.module_id,
5222 module_declared,
5223 daemon_observed: SupervisorObservedProcess {
5224 launch_nonce_source: status.launch_nonce_source,
5225 pid: status.pid,
5226 spawned_at_ms: status.spawned_at_ms,
5227 spawned_from: status.spawned_from,
5228 running_image,
5229 },
5230 });
5231 }
5232 let daemon = SupervisorDaemonProvenance {
5233 daemon_build: self.daemon_provenance.build.clone(),
5234 daemon_observed: DaemonObservedProcess {
5235 pid: self.daemon_provenance.pid,
5236 started_at_ms: self
5237 .daemon_provenance
5238 .start_clock
5239 .map(|clock| clock.started_at_ms())
5240 .or(self.daemon_provenance.started_at_ms),
5241 running_image: self
5242 .daemon_provenance
5243 .probe
5244 .observe(
5245 self.daemon_provenance.pid,
5246 self.daemon_provenance.executable_path.as_deref(),
5247 self.daemon_provenance.executable_identity.clone(),
5248 self.daemon_provenance.process_start_time,
5249 )
5250 .await,
5251 },
5252 };
5253 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5254 Ok(vec![control_response_body_frame(
5255 &frame,
5256 &response,
5257 "ClientControlResponse::SupervisorProvenance",
5258 )?])
5259 }
5260
5261 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5262 self.refresh_capability_requirements();
5263 let generation = self
5264 .registry
5265 .generation()
5266 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5267 let modules = self
5268 .supervisor
5269 .list()
5270 .into_iter()
5271 .map(|module| {
5272 let status = module.status_for_control("health").map_err(|err| {
5273 RouterError::backend(
5274 0,
5275 frame.header.corr,
5276 format!("failed to read supervisor health: {err}"),
5277 )
5278 })?;
5279 let module_id = status.module_id;
5280 let capability_detail = self
5281 .capability_evaluator
5282 .required_problem_detail(&module_id);
5283 Ok(SupervisorHealthEntry {
5284 module_id,
5285 status: status.health.status,
5286 detail: append_capability_problem_detail(
5287 status.health.detail,
5288 capability_detail,
5289 ),
5290 metrics: status.health.metrics,
5291 consecutive_failures: status.health.consecutive_failures,
5292 late_answer_count: status.health.late_answer_count,
5293 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5294 last_action: status.health.last_action,
5295 last_action_ms: status.health.last_action_ms,
5296 last_probe_ms: status.health.last_probe_ms,
5297 })
5298 })
5299 .collect::<Result<Vec<_>, RouterError>>()?;
5300 let response = ClientControlResponse::SupervisorHealth {
5301 generation,
5302 modules,
5303 };
5304 Ok(vec![control_response_body_frame(
5305 &frame,
5306 &response,
5307 "ClientControlResponse::SupervisorHealth",
5308 )?])
5309 }
5310
5311 async fn handle_supervisor_restart(
5312 &self,
5313 frame: Frame,
5314 module_id: String,
5315 drain_timeout_ms: Option<u64>,
5316 requester: Principal,
5317 ) -> Result<Vec<Frame>, RouterError> {
5318 let operation_lock = self.supervisor.operation_lock();
5319 let _operation_guard = operation_lock.lock().await;
5320 let Some(module) = self.supervisor.get(&module_id) else {
5321 return Ok(vec![control_error_frame(
5322 &frame,
5323 "unknown_module",
5324 format!("module_id '{module_id}' is not supervised"),
5325 )?]);
5326 };
5327
5328 self.route_outages.mark_operator_action(&module_id);
5329 if let Err(err) = module
5330 .restart_requested(drain_timeout_ms, Some(requester))
5331 .await
5332 {
5333 self.route_outages
5334 .operator_action_ended_unrefused(&module_id);
5335 let (code, message) = match err {
5336 crate::supervise::SuperviseError::Disabled { .. } => {
5337 ("module_disabled", err.to_string())
5338 }
5339 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5340 ("swap_in_progress", err.to_string())
5341 }
5342 _ => (
5343 "target_unavailable",
5344 format!("failed to restart module_id '{module_id}': {err}"),
5345 ),
5346 };
5347 return Ok(vec![control_error_frame(&frame, code, message)?]);
5348 }
5349
5350 let response = ClientControlResponse::SupervisorAck {
5351 module_id,
5352 applied: true,
5353 };
5354 Ok(vec![control_response_body_frame(
5355 &frame,
5356 &response,
5357 "ClientControlResponse::SupervisorAck",
5358 )?])
5359 }
5360
5361 async fn handle_supervisor_swap(
5365 &self,
5366 frame: Frame,
5367 module_id: String,
5368 ready_timeout_ms: Option<u64>,
5369 requester: Principal,
5370 ) -> Result<Vec<Frame>, RouterError> {
5371 let module = {
5378 let operation_lock = self.supervisor.operation_lock();
5379 let _operation_guard = operation_lock.lock().await;
5380 self.supervisor.get(&module_id)
5381 };
5382 let Some(module) = module else {
5383 return Ok(vec![control_error_frame(
5384 &frame,
5385 "unknown_module",
5386 format!("module_id '{module_id}' is not supervised"),
5387 )?]);
5388 };
5389
5390 self.route_outages.mark_operator_action(&module_id);
5391 if let Err(err) = module
5392 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5393 .await
5394 {
5395 self.route_outages
5396 .operator_action_ended_unrefused(&module_id);
5397 use crate::supervise::SuperviseError;
5398 let message = err.to_string();
5399 let error = match err {
5400 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5401 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5402 code: "swap_refused".to_string(),
5403 message,
5404 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5405 },
5406 SuperviseError::SwapFailed {
5407 arm,
5408 candidate_exit,
5409 ..
5410 } => ErrorBody {
5411 code: "swap_failed".to_string(),
5412 message,
5413 detail: Some(serde_json::json!({
5414 "arm": arm.as_str(),
5415 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5416 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5417 })),
5418 },
5419 _ => ErrorBody::new(
5420 "target_unavailable",
5421 format!("failed to swap module_id '{module_id}': {message}"),
5422 ),
5423 };
5424 return Ok(vec![control_error_body_frame(&frame, error)?]);
5425 }
5426 self.route_outages
5430 .operator_action_ended_unrefused(&module_id);
5431
5432 let response = ClientControlResponse::SupervisorAck {
5433 module_id,
5434 applied: true,
5435 };
5436 Ok(vec![control_response_body_frame(
5437 &frame,
5438 &response,
5439 "ClientControlResponse::SupervisorAck",
5440 )?])
5441 }
5442
5443 async fn handle_supervisor_reload(
5444 &self,
5445 frame: Frame,
5446 module_id: String,
5447 requester: Principal,
5448 ) -> Result<Vec<Frame>, RouterError> {
5449 let operation_lock = self.supervisor.operation_lock();
5450 let _operation_guard = operation_lock.lock().await;
5451 let Some(module) = self.supervisor.get(&module_id) else {
5452 return Ok(vec![control_error_frame(
5453 &frame,
5454 "unknown_module",
5455 format!("module_id '{module_id}' is not supervised"),
5456 )?]);
5457 };
5458
5459 self.route_outages.mark_operator_action(&module_id);
5460 if let Err(err) = module.reload_requested(Some(requester)).await {
5461 self.route_outages
5462 .operator_action_ended_unrefused(&module_id);
5463 let (code, message) = match err {
5464 crate::supervise::SuperviseError::Disabled { .. } => {
5465 ("module_disabled", err.to_string())
5466 }
5467 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5468 ("swap_in_progress", err.to_string())
5469 }
5470 _ => (
5471 "reload_failed",
5472 format!("failed to reload module_id '{module_id}': {err}"),
5473 ),
5474 };
5475 return Ok(vec![control_error_frame(&frame, code, message)?]);
5476 }
5477
5478 let response = ClientControlResponse::SupervisorAck {
5479 module_id,
5480 applied: true,
5481 };
5482 Ok(vec![control_response_body_frame(
5483 &frame,
5484 &response,
5485 "ClientControlResponse::SupervisorAck",
5486 )?])
5487 }
5488
5489 async fn handle_supervisor_rescan(
5490 &self,
5491 frame: Frame,
5492 preview: bool,
5493 requester: Principal,
5494 ) -> Result<Vec<Frame>, RouterError> {
5495 let Some(context) = self.rescan.clone() else {
5496 return Ok(vec![control_error_frame(
5497 &frame,
5498 "rescan_unavailable",
5499 "the daemon was not started with a reloadable config path".to_string(),
5500 )?]);
5501 };
5502
5503 let operation_lock = self.supervisor.operation_lock();
5504 let _operation_guard = operation_lock.lock().await;
5505 let loaded = match crate::daemon_config::load_with_activity(&context.config_path) {
5506 Ok(config) => config,
5507 Err(err) => {
5508 return Ok(vec![control_error_frame(
5509 &frame,
5510 "invalid_daemon_config",
5511 format!("supervisor rescan rejected daemon config: {err}"),
5512 )?])
5513 }
5514 };
5515 let Some((config, activity_config)) = loaded else {
5523 return Ok(vec![control_error_frame(
5524 &frame,
5525 "invalid_daemon_config",
5526 format!(
5527 "daemon config not found at {}; refusing to rescan (an absent config would \
5528 retire every supervised module)",
5529 context.config_path.display()
5530 ),
5531 )?]);
5532 };
5533 let (
5534 configured_port,
5535 storage_config,
5536 admission_facts_carrier_module_id,
5537 admission_facts_targets,
5538 scope_authority_owners,
5539 privileged_operator_prompt,
5540 modules,
5541 reserved_capabilities,
5542 ) = (
5543 config.port,
5544 config.storage,
5545 config.admission_facts_carrier_module_id,
5546 config.admission_facts_targets,
5547 config.scope_authority_owners,
5548 config.privileged_operator_prompt,
5549 config.modules,
5550 config.reserved_capabilities,
5551 );
5552
5553 let mut restart_required = Vec::new();
5562 for section in RestartRequiredSection::ALL {
5563 let changed = match section {
5564 RestartRequiredSection::Port => configured_port != context.configured_port,
5565 RestartRequiredSection::Storage => storage_config != context.storage_config,
5566 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5567 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5568 }
5569 RestartRequiredSection::AdmissionFactsTargets => {
5570 admission_facts_targets != context.admission_facts_targets
5571 }
5572 RestartRequiredSection::ScopeAuthorityOwners => {
5573 scope_authority_owners != context.scope_authority_owners
5574 }
5575 RestartRequiredSection::PrivilegedOperatorPrompt => {
5576 privileged_operator_prompt != context.privileged_operator_prompt
5577 }
5578 };
5579 if changed {
5580 restart_required.push(section.label().to_string());
5581 }
5582 }
5583 if !restart_required.is_empty() {
5584 warn!(
5585 config_path = %context.config_path.display(),
5586 sections = %restart_required.join(", "),
5587 "daemon config changed outside the modules section; restart the daemon to apply those changes"
5588 );
5589 }
5590
5591 for configured in &modules {
5592 if let Err(err) = validate_spec(&configured.module_spec()) {
5593 return Ok(vec![control_error_frame(
5594 &frame,
5595 "invalid_daemon_config",
5596 format!("supervisor rescan rejected daemon config: {err}"),
5597 )?]);
5598 }
5599 }
5600
5601 let configured_capabilities = modules
5602 .iter()
5603 .map(|module| (module.module_id.clone(), module.enabled))
5604 .collect::<Vec<_>>();
5605 let preview_capability_warnings = if preview {
5606 let (_, registrations) = self.runtime_capability_snapshot()?;
5607 let current_modules = self
5608 .supervisor
5609 .list()
5610 .into_iter()
5611 .map(|module| module.module_id().to_string())
5612 .collect::<BTreeSet<_>>();
5613 let resulting_modules = configured_capabilities.clone();
5614 let removed = current_modules
5615 .into_iter()
5616 .filter(|module_id| {
5617 !resulting_modules
5618 .iter()
5619 .any(|(configured_id, _)| configured_id == module_id)
5620 })
5621 .collect::<Vec<_>>();
5622 self.capability_evaluator.preview_removal_warnings(
5623 resulting_modules,
5624 &removed,
5625 ®istrations,
5626 )
5627 } else {
5628 Vec::new()
5629 };
5630 let result = match self
5631 .reconcile_supervised_modules(&context.supervisor, modules, preview, requester.clone())
5632 .await
5633 {
5634 Ok(result) => result,
5635 Err(message) => {
5636 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5637 }
5638 };
5639 if !preview {
5640 if let Some(ledger) = &self.activity {
5641 ledger.module_rescanned(&result.added, &result.removed, requester);
5642 }
5643 if let Err(code) = self.configure_activity(activity_config).await {
5644 warn!(
5645 code,
5646 "activity retention could not be applied during rescan"
5647 );
5648 }
5649 self.capability_evaluator
5650 .configure(configured_capabilities, reserved_capabilities);
5651 self.capability_evaluator.wake_deadline_loop();
5652 self.refresh_capability_requirements();
5653 }
5654 let mut result = result;
5655 result.restart_required = restart_required;
5656 result.capability_warnings = preview_capability_warnings;
5657 let response = ClientControlResponse::SupervisorRescan { result };
5658 Ok(vec![control_response_body_frame(
5659 &frame,
5660 &response,
5661 "ClientControlResponse::SupervisorRescan",
5662 )?])
5663 }
5664
5665 async fn handle_supervisor_release_reserved(
5666 &self,
5667 frame: Frame,
5668 module_id: String,
5669 ) -> Result<Vec<Frame>, RouterError> {
5670 let Some(context) = self.rescan.clone() else {
5671 return Ok(vec![control_error_frame(
5672 &frame,
5673 "release_unavailable",
5674 "reserved-id release requires a daemon started with a reloadable config path",
5675 )?]);
5676 };
5677 let operation_lock = self.supervisor.operation_lock();
5678 let _operation_guard = operation_lock.lock().await;
5679 let loaded = match crate::daemon_config::load(&context.config_path) {
5680 Ok(Some(config)) => config,
5681 Ok(None) => {
5682 return Ok(vec![control_error_frame(
5683 &frame,
5684 "invalid_daemon_config",
5685 format!(
5686 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5687 context.config_path.display()
5688 ),
5689 )?])
5690 }
5691 Err(err) => {
5692 return Ok(vec![control_error_frame(
5693 &frame,
5694 "invalid_daemon_config",
5695 format!("unable to verify reserved-id release against daemon config: {err}"),
5696 )?])
5697 }
5698 };
5699 if loaded
5700 .modules
5701 .iter()
5702 .any(|configured| configured.module_id == module_id)
5703 {
5704 return Ok(vec![control_error_frame(
5705 &frame,
5706 "reserved_module_configured",
5707 format!(
5708 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5709 ),
5710 )?]);
5711 }
5712 if !self.supervisor.release_retained_reserved_gate(&module_id) {
5713 return Ok(vec![control_error_frame(
5714 &frame,
5715 "reserved_gate_not_retained",
5716 format!(
5717 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
5718 ),
5719 )?]);
5720 }
5721
5722 let response = ClientControlResponse::SupervisorAck {
5723 module_id,
5724 applied: true,
5725 };
5726 Ok(vec![control_response_body_frame(
5727 &frame,
5728 &response,
5729 "ClientControlResponse::SupervisorAck",
5730 )?])
5731 }
5732
5733 async fn reconcile_supervised_modules(
5742 &self,
5743 supervisor: &Supervisor,
5744 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
5745 preview: bool,
5746 requester: Principal,
5747 ) -> Result<SupervisorRescanResult, String> {
5748 let mut current = BTreeMap::new();
5749 for module in self.supervisor.list() {
5750 let (spec, health) = module.configuration().map_err(|err| {
5751 format!(
5752 "failed to read configuration for module_id '{}': {err}",
5753 module.module_id()
5754 )
5755 })?;
5756 let enabled = module
5757 .status()
5758 .map_err(|err| {
5759 format!(
5760 "failed to read status for module_id '{}': {err}",
5761 module.module_id()
5762 )
5763 })?
5764 .enabled;
5765 current.insert(
5766 module.module_id().to_string(),
5767 (module, spec, health, enabled),
5768 );
5769 }
5770 let configured = configured_modules
5771 .into_iter()
5772 .map(|module| (module.module_id.clone(), module))
5773 .collect::<BTreeMap<_, _>>();
5774
5775 let added = configured
5776 .keys()
5777 .filter(|module_id| !current.contains_key(*module_id))
5778 .cloned()
5779 .collect::<Vec<_>>();
5780 let removed = current
5781 .keys()
5782 .filter(|module_id| !configured.contains_key(*module_id))
5783 .cloned()
5784 .collect::<Vec<_>>();
5785 let mut changed_pending_reload = Vec::new();
5786 let mut configuration_changes = BTreeSet::new();
5787 let mut enabled_changes = BTreeSet::new();
5788 let mut unchanged = 0_u32;
5789
5790 for (module_id, configured_module) in &configured {
5791 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5792 else {
5793 continue;
5794 };
5795 let launch_changed = *current_spec != configured_module.module_spec();
5799 let configuration_changed =
5800 launch_changed || *current_health != configured_module.health;
5801 let enabled_changed = *current_enabled != configured_module.enabled;
5802 if configuration_changed {
5803 configuration_changes.insert(module_id.clone());
5804 }
5805 if launch_changed {
5806 changed_pending_reload.push(module_id.clone());
5807 }
5808 if enabled_changed {
5809 enabled_changes.insert(module_id.clone());
5810 }
5811 if !configuration_changed && !enabled_changed {
5812 unchanged = unchanged.saturating_add(1);
5813 }
5814 }
5815
5816 if preview {
5822 return Ok(SupervisorRescanResult {
5823 added,
5824 removed,
5825 changed_pending_reload,
5826 enabled_changes: enabled_changes.iter().cloned().collect(),
5827 unchanged,
5828 preview: true,
5829 restart_required: Vec::new(),
5833 capability_warnings: Vec::new(),
5834 });
5835 }
5836
5837 for module_id in &removed {
5838 let module = ¤t
5839 .get(module_id)
5840 .expect("removed module came from current supervisor state")
5841 .0;
5842 module
5843 .retire_requested(Some(requester.clone()))
5844 .await
5845 .map_err(|err| {
5846 format!("failed to retire module_id '{module_id}' during rescan: {err}")
5847 })?;
5848 self.supervisor.record_rescan_removal(module_id);
5882 self.supervisor.retire(module_id);
5883 self.route_outages.forget(module_id);
5884 }
5885
5886 for module_id in configured.keys() {
5887 let Some((module, _, _, _)) = current.get(module_id) else {
5888 continue;
5889 };
5890 let configured_module = configured
5891 .get(module_id)
5892 .expect("configured module id came from configured map");
5893 if configuration_changes.contains(module_id) {
5894 module
5895 .update_configuration(
5896 configured_module.module_spec(),
5897 configured_module.health.clone(),
5898 configured_module.drain_timeout_ms,
5899 )
5900 .await
5901 .map_err(|err| {
5902 format!(
5903 "failed to update module_id '{module_id}' configuration during rescan: {err}"
5904 )
5905 })?;
5906 }
5907 if enabled_changes.contains(module_id) {
5908 self.route_outages.mark_operator_action(module_id);
5911 module
5912 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
5913 .await
5914 .map_err(|err| {
5915 self.route_outages.operator_action_ended_unrefused(module_id);
5916 format!(
5917 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
5918 configured_module.enabled
5919 )
5920 })?;
5921 }
5922 }
5923
5924 for module_id in &added {
5925 let configured_module = configured
5926 .get(module_id)
5927 .expect("added module id came from configured map");
5928 supervisor
5929 .supervise_configured_requested(
5930 configured_module.module_spec(),
5931 configured_module.enabled,
5932 configured_module.health.clone(),
5933 configured_module.drain_timeout_ms,
5934 configured_module.restart,
5935 Some(requester.clone()),
5936 )
5937 .map_err(|err| {
5938 format!("failed to add module_id '{module_id}' during rescan: {err}")
5939 })?;
5940 }
5941
5942 Ok(SupervisorRescanResult {
5943 added,
5944 removed,
5945 changed_pending_reload,
5946 enabled_changes: enabled_changes.iter().cloned().collect(),
5947 unchanged,
5948 preview: false,
5949 restart_required: Vec::new(),
5952 capability_warnings: Vec::new(),
5953 })
5954 }
5955
5956 async fn handle_supervisor_set_enabled(
5957 &self,
5958 frame: Frame,
5959 module_id: String,
5960 enabled: bool,
5961 requester: Principal,
5962 ) -> Result<Vec<Frame>, RouterError> {
5963 let operation_lock = self.supervisor.operation_lock();
5964 let _operation_guard = operation_lock.lock().await;
5965 let Some(module) = self.supervisor.get(&module_id) else {
5966 return Ok(vec![control_error_frame(
5967 &frame,
5968 "unknown_module",
5969 format!("module_id '{module_id}' is not supervised"),
5970 )?]);
5971 };
5972
5973 self.route_outages.mark_operator_action(&module_id);
5976 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
5977 Ok(applied) => applied,
5978 Err(err) => {
5979 self.route_outages
5980 .operator_action_ended_unrefused(&module_id);
5981 return Ok(vec![control_error_frame(
5982 &frame,
5983 "target_unavailable",
5984 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
5985 )?]);
5986 }
5987 };
5988 if !applied {
5989 self.route_outages
5992 .operator_action_ended_unrefused(&module_id);
5993 }
5994
5995 self.capability_evaluator.wake_deadline_loop();
5996 self.refresh_capability_requirements();
5997 let response = ClientControlResponse::SupervisorAck { module_id, applied };
5998 Ok(vec![control_response_body_frame(
5999 &frame,
6000 &response,
6001 "ClientControlResponse::SupervisorAck",
6002 )?])
6003 }
6004
6005 async fn handle_supervisor_health_probe(
6006 &self,
6007 frame: Frame,
6008 module_id: String,
6009 ) -> Result<Vec<Frame>, RouterError> {
6010 self.refresh_capability_requirements();
6011 let Some(registration) = self
6012 .registry
6013 .get_module(&module_id)
6014 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6015 else {
6016 return Ok(vec![control_error_frame(
6017 &frame,
6018 "unknown_module",
6019 format!("module_id '{module_id}' is not registered"),
6020 )?]);
6021 };
6022
6023 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6036 {
6037 return Ok(vec![control_error_frame(
6038 &frame,
6039 "health_not_advertised",
6040 format!("module_id '{module_id}' did not advertise health.check"),
6041 )?]);
6042 }
6043
6044 let deadline = Instant::now() + self.health_probe_timeout;
6045 let pending = match self.forwarding.begin_module_control_rpc_for(
6046 &module_id,
6047 MODULE_CONTROL_OP_HEALTH_CHECK,
6048 deadline,
6049 ) {
6050 Ok(pending) => pending,
6051 Err(err) => {
6052 return Ok(vec![control_error_frame(
6053 &frame,
6054 forwarding_error_code(&err),
6055 err.to_string(),
6056 )?])
6057 }
6058 };
6059
6060 let PendingModuleControlRpc {
6061 endpoint,
6062 module_sink,
6063 negotiated_ver,
6064 corr: probe_corr,
6065 receiver,
6066 } = pending;
6067 let mut guard =
6068 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6069 let probe_body =
6070 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6071 RouterError::backend(
6072 0,
6073 frame.header.corr,
6074 format!("failed to encode health.check request: {err}"),
6075 )
6076 })?;
6077 let probe_frame = Frame::build_with_version(
6078 negotiated_ver,
6079 FrameType::Request,
6080 control_flags(),
6081 0,
6082 0,
6083 probe_corr,
6084 probe_body,
6085 )
6086 .map_err(RouterError::FrameBuild)?;
6087
6088 match timeout_at(deadline, module_sink.send(probe_frame)).await {
6091 Ok(Ok(())) => {}
6092 Ok(Err(err)) => {
6093 return Ok(vec![control_error_frame(
6094 &frame,
6095 "target_unavailable",
6096 err.to_string(),
6097 )?]);
6098 }
6099 Err(_) => {
6100 return Ok(vec![control_error_frame(
6101 &frame,
6102 "module_timeout",
6103 format!(
6104 "module_id '{module_id}' did not answer health.check within {:?}",
6105 self.health_probe_timeout
6106 ),
6107 )?]);
6108 }
6109 }
6110
6111 match timeout_at(deadline, receiver).await {
6112 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6113 guard.disarm();
6114 let Some(report) = response.health_report() else {
6115 return Ok(vec![control_error_frame(
6116 &frame,
6117 "invalid_control_body",
6118 "health.check RPC returned a non-health response",
6119 )?]);
6120 };
6121 let HealthReport {
6126 status,
6127 detail,
6128 metrics,
6129 } = report;
6130 let capability_detail = self
6131 .capability_evaluator
6132 .required_problem_detail(&module_id);
6133 let response = ClientControlResponse::SupervisorHealthProbe {
6134 module_id,
6135 status,
6136 detail: append_capability_problem_detail(detail, capability_detail),
6137 metrics,
6138 };
6139 Ok(vec![control_response_body_frame(
6140 &frame,
6141 &response,
6142 "ClientControlResponse::SupervisorHealthProbe",
6143 )?])
6144 }
6145 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6146 guard.disarm();
6147 Ok(vec![control_error_body_frame(&frame, body)?])
6148 }
6149 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6150 guard.disarm();
6151 Ok(vec![control_error_frame(
6152 &frame,
6153 "target_unavailable",
6154 message,
6155 )?])
6156 }
6157 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6158 guard.disarm();
6159 Ok(vec![control_error_frame(
6160 &frame,
6161 "invalid_control_body",
6162 message,
6163 )?])
6164 }
6165 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6166 guard.disarm();
6167 Ok(vec![control_error_frame(
6168 &frame,
6169 "invalid_control_body",
6170 format!("expected module-control op '{expected}', got '{actual}'"),
6171 )?])
6172 }
6173 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6174 guard.disarm();
6175 Ok(vec![control_error_frame(
6176 &frame,
6177 "module_timeout",
6178 format!(
6179 "module_id '{module_id}' answered health.check after {:?}",
6180 self.health_probe_timeout
6181 ),
6182 )?])
6183 }
6184 Ok(Err(_)) => Ok(vec![control_error_frame(
6185 &frame,
6186 "target_unavailable",
6187 "health.check waiter was canceled before the module responded",
6188 )?]),
6189 Err(_) => Ok(vec![control_error_frame(
6190 &frame,
6191 "module_timeout",
6192 format!(
6193 "module_id '{module_id}' did not answer health.check within {:?}",
6194 self.health_probe_timeout
6195 ),
6196 )?]),
6197 }
6198 }
6199
6200 fn supervisor_status(
6201 &self,
6202 module_id: &str,
6203 corr: u64,
6204 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6205 self.supervisor
6206 .get(module_id)
6207 .map(|module| {
6208 let warming = module.is_warming_for_control("status").map_err(|err| {
6209 RouterError::backend(
6210 0,
6211 corr,
6212 format!(
6213 "failed to read supervisor warming state for module_id '{module_id}': {err}"
6214 ),
6215 )
6216 })?;
6217 module.status_for_control("status").map_err(|err| {
6218 RouterError::backend(
6219 0,
6220 corr,
6221 format!(
6222 "failed to read supervisor status for module_id '{module_id}': {err}"
6223 ),
6224 )
6225 }).map(|status| (status, warming))
6226 })
6227 .transpose()
6228 }
6229
6230 fn guard_module_control_op(
6231 &self,
6232 frame: &Frame,
6233 module_id: &str,
6234 op: &str,
6235 ) -> Result<Option<Frame>, RouterError> {
6236 if self.module_grants_op(module_id, op, frame.header.corr)? {
6237 return Ok(None);
6238 }
6239
6240 Ok(Some(control_error_frame(
6241 frame,
6242 "op_not_allowed",
6243 format!("module_id '{module_id}' did not grant control op '{op}'"),
6244 )?))
6245 }
6246
6247 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6248 let Some(registration) = self
6249 .registry
6250 .get_module(module_id)
6251 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6252 else {
6253 return Ok(false);
6254 };
6255 Ok(module_registration_grants_op(®istration.control_ops, op))
6256 }
6257
6258 fn handle_status_update(
6259 &self,
6260 endpoint: ModuleEndpointId,
6261 frame: Frame,
6262 ) -> Result<Vec<Frame>, RouterError> {
6263 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6264 Ok(update) => update,
6265 Err(err) => {
6266 if is_known_module_push_op(&frame.body) {
6271 return Ok(vec![control_error_frame(
6272 &frame,
6273 "invalid_control_body",
6274 format!("malformed module control push body: {err}"),
6275 )?]);
6276 }
6277 return Ok(Vec::new());
6278 }
6279 };
6280
6281 match update {
6282 ModuleControlPush::RouteStatus {
6283 route_channel,
6284 route_epoch,
6285 status,
6286 } => {
6287 self.forwarding
6288 .cache_status(endpoint, route_channel, route_epoch, status)
6289 .map_err(RouterError::Forwarding)?;
6290 }
6291 }
6292 Ok(Vec::new())
6293 }
6294
6295 fn handle_route_poll(
6296 &self,
6297 ctx: &RouteCtx,
6298 frame: Frame,
6299 route_channel: u16,
6300 route_epoch: u32,
6301 kind: PollKind,
6302 ) -> Result<Vec<Frame>, RouterError> {
6303 let snapshot = self
6304 .forwarding
6305 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6306 .map_err(RouterError::Forwarding)?;
6307 let response = match (kind, snapshot) {
6308 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6309 ClientControlResponse::RoutePoll {
6310 route_channel,
6311 route_epoch,
6312 status,
6313 live: None,
6314 }
6315 }
6316 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6317 route_channel,
6318 route_epoch,
6319 status: None,
6320 live: None,
6321 },
6322 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6323 let live = self
6349 .process_liveness
6350 .as_ref()
6351 .and_then(|source| source.process_live(&module_id))
6352 .unwrap_or(true);
6353 ClientControlResponse::RoutePoll {
6354 route_channel,
6355 route_epoch,
6356 status: None,
6357 live: Some(live),
6358 }
6359 }
6360 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6361 route_channel,
6362 route_epoch,
6363 status: None,
6364 live: Some(false),
6365 },
6366 };
6367
6368 Ok(vec![control_response_body_frame(
6369 &frame,
6370 &response,
6371 "ClientControlResponse::RoutePoll",
6372 )?])
6373 }
6374
6375 pub(crate) fn observe_module_control_completion(
6376 &self,
6377 completion: ModuleControlRpcCompletion,
6378 ) -> bool {
6379 match completion {
6380 ModuleControlRpcCompletion::Unknown => false,
6381 ModuleControlRpcCompletion::Settled => true,
6382 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6383 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6384 info!(
6385 module_id = %module_id,
6386 latency_ms,
6387 "late health.check answer proves the module is alive"
6388 );
6389 match self
6390 .supervisor
6391 .record_late_health_answer(&module_id, latency_ms)
6392 {
6393 Ok(true) => {}
6394 Ok(false) => debug!(
6395 module_id = %module_id,
6396 latency_ms,
6397 "late health.check answer has no active supervisor snapshot"
6398 ),
6399 Err(err) => warn!(
6400 module_id = %module_id,
6401 latency_ms,
6402 error = %err,
6403 "failed to record late health.check answer"
6404 ),
6405 }
6406 true
6407 }
6408 }
6409 }
6410
6411 fn refuse_to_end_module_connection_for_a_client(
6432 &self,
6433 module_connection_id: ConnectionId,
6434 corr: u64,
6435 err: ForwardingError,
6436 ) -> Result<(), RouterError> {
6437 if let ForwardingError::ConnectionClosing { connection_id } = err {
6438 if connection_id != module_connection_id {
6439 warn!(
6440 module_connection_id = module_connection_id.get(),
6441 client_connection_id = connection_id.get(),
6442 corr,
6443 "dropping a route.bind response for a closing client; the module connection keeps serving"
6444 );
6445 return Ok(());
6446 }
6447 }
6448 Err(RouterError::Forwarding(err))
6449 }
6450
6451 fn handle_module_relay_response(
6452 &self,
6453 connection_id: ConnectionId,
6454 frame: Frame,
6455 ) -> Result<Vec<Frame>, RouterError> {
6456 let mut secondary_error = None;
6457 let outcome = match frame.header.ty {
6458 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6459 Ok(probe) if probe.op == "route.bind" => {
6460 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6461 Ok(ModuleControlResponse::RouteBindAck {}) => {
6462 RouteBindRelayOutcome::Accepted
6463 }
6464 Ok(other) => {
6465 let message =
6466 format!("route.bind response carried unexpected body: {other:?}");
6467 secondary_error = Some(control_error_frame(
6468 &frame,
6469 "invalid_control_body",
6470 message.clone(),
6471 )?);
6472 RouteBindRelayOutcome::ModuleGone(message)
6473 }
6474 Err(err) => {
6475 let message = format!("malformed route.bind response body: {err}");
6476 secondary_error = Some(control_error_frame(
6477 &frame,
6478 "invalid_control_body",
6479 message.clone(),
6480 )?);
6481 RouteBindRelayOutcome::ModuleGone(message)
6482 }
6483 }
6484 }
6485 Ok(probe) => {
6486 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6487 {
6488 Ok(response) => ModuleControlRpcOutcome::Response(response),
6489 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6490 "malformed {} response body: {err}",
6491 probe.op
6492 )),
6493 };
6494 let completion = self
6495 .forwarding
6496 .complete_module_control_rpc(
6497 connection_id,
6498 frame.header.corr,
6499 Some(&probe.op),
6500 outcome,
6501 )
6502 .map_err(RouterError::Forwarding)?;
6503 if !self.observe_module_control_completion(completion) {
6504 debug!(
6505 connection_id = connection_id.get(),
6506 corr = frame.header.corr,
6507 op = %probe.op,
6508 "dropping late or unknown module-control RPC response"
6509 );
6510 }
6511 return Ok(Vec::new());
6512 }
6513 Err(err) => {
6514 if let Some(expected_op) = self
6515 .forwarding
6516 .pending_module_control_op(connection_id, frame.header.corr)
6517 .map_err(RouterError::Forwarding)?
6518 {
6519 let completion = self
6520 .forwarding
6521 .complete_module_control_rpc(
6522 connection_id,
6523 frame.header.corr,
6524 None,
6525 ModuleControlRpcOutcome::MalformedResponse(format!(
6526 "malformed {expected_op} response body: {err}"
6527 )),
6528 )
6529 .map_err(RouterError::Forwarding)?;
6530 if !self.observe_module_control_completion(completion) {
6531 debug!(
6532 connection_id = connection_id.get(),
6533 corr = frame.header.corr,
6534 "dropping late malformed module-control RPC response"
6535 );
6536 }
6537 return Ok(Vec::new());
6538 }
6539 let message = format!("malformed route.bind response body: {err}");
6540 secondary_error = Some(control_error_frame(
6541 &frame,
6542 "invalid_control_body",
6543 message.clone(),
6544 )?);
6545 RouteBindRelayOutcome::ModuleGone(message)
6546 }
6547 },
6548 FrameType::Error => {
6549 if self
6550 .forwarding
6551 .pending_module_control_op(connection_id, frame.header.corr)
6552 .map_err(RouterError::Forwarding)?
6553 .is_some()
6554 {
6555 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6556 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
6557 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6558 "malformed module-control ERROR body: {err}"
6559 )),
6560 };
6561 let completion = self
6562 .forwarding
6563 .complete_module_control_rpc(
6564 connection_id,
6565 frame.header.corr,
6566 None,
6567 outcome,
6568 )
6569 .map_err(RouterError::Forwarding)?;
6570 if !self.observe_module_control_completion(completion) {
6571 debug!(
6572 connection_id = connection_id.get(),
6573 corr = frame.header.corr,
6574 "dropping late or unknown module-control RPC error"
6575 );
6576 }
6577 return Ok(Vec::new());
6578 }
6579 match serde_json::from_slice::<ErrorBody>(&frame.body) {
6580 Ok(body) => RouteBindRelayOutcome::Rejected(body),
6581 Err(err) => {
6582 let message = format!("malformed route.bind ERROR body: {err}");
6583 secondary_error = Some(control_error_frame(
6584 &frame,
6585 "invalid_control_body",
6586 message.clone(),
6587 )?);
6588 RouteBindRelayOutcome::ModuleGone(message)
6589 }
6590 }
6591 }
6592 ty => {
6593 return Ok(vec![control_error_frame(
6594 &frame,
6595 "unsupported_control_frame",
6596 format!("unsupported module channel-0 frame {ty:?}"),
6597 )?])
6598 }
6599 };
6600
6601 let settled =
6602 self.forwarding
6603 .complete_pending_relay(connection_id, frame.header.corr, outcome);
6604 let completion = match settled {
6605 Ok(completion) => completion,
6606 Err(err) => {
6607 self.refuse_to_end_module_connection_for_a_client(
6608 connection_id,
6609 frame.header.corr,
6610 err,
6611 )?;
6612 return Ok(secondary_error.into_iter().collect());
6613 }
6614 };
6615 if let Some(target) = completion.abandoned.as_ref() {
6616 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6617 }
6618 if !completion.settled {
6619 debug!(
6620 connection_id = connection_id.get(),
6621 corr = frame.header.corr,
6622 frame_type = ?frame.header.ty,
6623 "dropping late or unknown route.bind relay response"
6624 );
6625 }
6626 Ok(secondary_error.into_iter().collect())
6627 }
6628
6629 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6630 debug!(connection_id = connection_id.get(), "handling GOODBYE");
6631 self.cleanup_connection_with_end_reason(
6635 connection_id,
6636 RegistrationEndReason::ExplicitGoodbye,
6637 )
6638 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6639 Ok(Vec::new())
6640 }
6641}
6642
6643impl Default for ControlHandler {
6644 fn default() -> Self {
6645 Self::new(Arc::new(Registry::default()))
6646 }
6647}
6648
6649impl crate::supervise::SwapPromotionObserver for ControlHandler {
6650 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6651 self.apply_registration_capabilities(registration);
6652 }
6653}
6654
6655fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6656 CapabilityRequirementStatus {
6657 consumer: status.consumer,
6658 capability: status.capability,
6659 need: match status.need {
6660 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6661 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6662 },
6663 verdict: status.verdict.as_str().to_string(),
6664 episode_seq: status.episode_seq,
6665 config_satisfiable: status.config_satisfiable,
6666 runtime_available: status.runtime_available,
6667 detail: status.detail,
6668 }
6669}
6670
6671fn append_capability_problem_detail(
6672 detail: Option<String>,
6673 capability_detail: Option<String>,
6674) -> Option<String> {
6675 match (detail, capability_detail) {
6676 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
6677 (Some(detail), None) => Some(detail),
6678 (None, Some(capability_detail)) => Some(capability_detail),
6679 (None, None) => None,
6680 }
6681}
6682
6683fn subc_ops() -> Vec<String> {
6684 SUBC_CONTROL_OPS
6685 .iter()
6686 .map(|op| (*op).to_string())
6687 .collect()
6688}
6689
6690fn module_subc_ops() -> Vec<String> {
6691 SUBC_CONTROL_OPS
6692 .iter()
6693 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
6694 .map(|op| (*op).to_string())
6695 .collect()
6696}
6697
6698#[cfg(test)]
6699fn module_baseline_control_ops() -> Vec<String> {
6700 MODULE_BASELINE_CONTROL_OPS
6701 .iter()
6702 .map(|op| (*op).to_string())
6703 .collect()
6704}
6705
6706fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
6707 let mut seen = HashSet::new();
6708 let mut effective = Vec::new();
6709 for op in MODULE_BASELINE_CONTROL_OPS {
6710 if seen.insert((*op).to_string()) {
6711 effective.push((*op).to_string());
6712 }
6713 }
6714 for op in declared.unwrap_or_default() {
6715 if seen.insert(op.clone()) {
6716 effective.push(op);
6717 }
6718 }
6719 effective
6720}
6721
6722fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
6723 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
6724}
6725
6726fn target_module_id(target: &RouteTarget) -> &str {
6727 match target {
6728 RouteTarget::ToolProvider { module_id }
6729 | RouteTarget::ManagementSurface { module_id }
6730 | RouteTarget::InternalService { module_id, .. } => module_id,
6731 }
6732}
6733
6734fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
6735 roles.iter().any(|role| match (target, role) {
6736 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
6737 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
6738 (
6739 RouteTarget::InternalService { service_id, .. },
6740 ProviderRole::InternalService {
6741 service_id: provided,
6742 ..
6743 },
6744 ) => service_id == provided,
6745 _ => false,
6746 })
6747}
6748
6749fn is_routable_role(role: &ProviderRole) -> bool {
6750 matches!(
6751 role,
6752 ProviderRole::ToolProvider { .. }
6753 | ProviderRole::ManagementSurface { .. }
6754 | ProviderRole::InternalService { .. }
6755 )
6756}
6757
6758#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6759enum ControlRequestBodyError {
6760 UnknownOp,
6761 InvalidBody,
6762}
6763
6764#[derive(Debug, Deserialize)]
6765struct ControlOpProbe {
6766 op: String,
6767}
6768
6769const MODULE_PUSH_OPS: &[&str] = &["route.status"];
6772
6773fn is_known_module_push_op(body: &[u8]) -> bool {
6774 serde_json::from_slice::<ControlOpProbe>(body)
6775 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
6776 .unwrap_or(false)
6777}
6778
6779fn is_known_module_request_op(body: &[u8]) -> bool {
6780 serde_json::from_slice::<ControlOpProbe>(body)
6781 .map(|probe| is_module_to_subc_op(&probe.op))
6782 .unwrap_or(false)
6783}
6784
6785fn is_module_to_subc_op(op: &str) -> bool {
6786 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
6787}
6788
6789fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
6790 debug!(
6791 op = %op,
6792 connection_id = connection_id.get(),
6793 corr,
6794 "control dispatch"
6795 );
6796}
6797
6798fn log_slow_control_dispatch(
6799 dispatch_started_at: Option<StdInstant>,
6800 op: &'static str,
6801 connection_id: ConnectionId,
6802 corr: u64,
6803 route_open_timings: Option<&RouteOpenTimings>,
6804) {
6805 let Some(dispatch_started_at) = dispatch_started_at else {
6806 return;
6807 };
6808 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
6810 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
6811 if let Some(timings) = route_open_timings {
6812 warn!(
6813 op = %op,
6814 connection_id = connection_id.get(),
6815 corr,
6816 elapsed_ms = elapsed.as_millis() as u64,
6817 module_id = ?timings.module_id,
6818 principal = %timings.principal,
6819 admission_ms = timings.admission.as_millis() as u64,
6820 project_root_ms = timings.project_root.as_millis() as u64,
6821 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
6822 bind_send_ms = timings.bind_send.as_millis() as u64,
6823 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
6824 "slow control dispatch"
6825 );
6826 return;
6827 }
6828 warn!(
6829 op = %op,
6830 connection_id = connection_id.get(),
6831 corr,
6832 elapsed_ms = elapsed.as_millis() as u64,
6833 "slow control dispatch"
6834 );
6835 }
6836}
6837
6838#[cfg(test)]
6840type TestCallerLookup =
6841 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
6842
6843fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
6845 match request {
6846 ClientControlRequest::SupervisorRestart { module_id, .. } => {
6847 Some(PrivilegedOperation::Restart { module_id })
6848 }
6849 ClientControlRequest::SupervisorSwap { module_id, .. } => {
6850 Some(PrivilegedOperation::Swap { module_id })
6851 }
6852 ClientControlRequest::SupervisorReload { module_id } => {
6853 Some(PrivilegedOperation::Reload { module_id })
6854 }
6855 ClientControlRequest::SupervisorRescan { preview } => {
6856 if *preview {
6857 None
6858 } else {
6859 Some(PrivilegedOperation::Rescan)
6860 }
6861 }
6862 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
6863 Some(PrivilegedOperation::SetEnabled {
6864 module_id,
6865 enabled: *enabled,
6866 })
6867 }
6868 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
6869 Some(PrivilegedOperation::ReleaseReserved { module_id })
6870 }
6871 ClientControlRequest::ServerDescribe {}
6872 | ClientControlRequest::CatalogList { .. }
6873 | ClientControlRequest::RouteOpen { .. }
6874 | ClientControlRequest::RoutePoll { .. }
6875 | ClientControlRequest::SupervisorList {}
6876 | ClientControlRequest::SupervisorHealth {}
6877 | ClientControlRequest::SupervisorHealthProbe { .. }
6878 | ClientControlRequest::SupervisorRoutes { .. }
6879 | ClientControlRequest::SupervisorProvenance { .. }
6880 | ClientControlRequest::SupervisorStderrTail { .. }
6881 | ClientControlRequest::SupervisorTerminals { .. }
6882 | ClientControlRequest::SupervisorSpawnSnapshot {}
6883 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
6884 }
6885}
6886
6887fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
6888 match request {
6889 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
6890 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
6891 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
6892 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
6893 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
6894 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
6895 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
6896 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
6897 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
6898 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
6899 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
6900 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
6901 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
6902 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
6903 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
6904 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
6905 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
6906 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
6907 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
6908 }
6909}
6910
6911fn principal_label(principal: &Principal) -> String {
6912 match principal {
6913 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
6914 Principal::Direct => "direct".to_string(),
6915 other => format!("{other:?}"),
6916 }
6917}
6918
6919fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
6920 match request {
6921 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
6922 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
6923 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
6924 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
6925 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
6926 }
6927}
6928
6929fn parse_client_control_request(
6930 body: &[u8],
6931) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
6932 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
6933 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6934 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
6935 ControlRequestBodyError::InvalidBody
6936 }
6937 Ok(_) => ControlRequestBodyError::UnknownOp,
6938 Err(_) => ControlRequestBodyError::InvalidBody,
6939 };
6940 (err, classification)
6941 })
6942}
6943
6944fn parse_module_control_request_from_module(
6945 body: &[u8],
6946) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
6947 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
6948 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6949 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
6950 Ok(_) => ControlRequestBodyError::UnknownOp,
6951 Err(_) => ControlRequestBodyError::InvalidBody,
6952 };
6953 (err, classification)
6954 })
6955}
6956
6957#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
6958enum ProviderRoleKind {
6959 ToolProvider,
6960 PipelineStage,
6961 ManagementSurface,
6962 InternalService,
6963}
6964
6965fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
6966 match role {
6967 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
6968 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
6969 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
6970 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
6971 }
6972}
6973
6974fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
6975 roles.iter().map(provider_role_kind).collect()
6976}
6977
6978const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
6981
6982#[derive(Debug, Default, PartialEq, Eq)]
6984struct ScopeOutcomeCounts {
6985 created: usize,
6986 replaced: usize,
6987 updated: usize,
6988 unchanged: usize,
6989 refused: usize,
6990}
6991
6992impl ScopeOutcomeCounts {
6993 fn of(results: &[ScopeRecordResult]) -> Self {
6994 let mut counts = Self::default();
6995 for result in results {
6996 let slot = match result.outcome {
6997 ScopeRecordOutcome::Created => &mut counts.created,
6998 ScopeRecordOutcome::Replaced => &mut counts.replaced,
6999 ScopeRecordOutcome::Updated => &mut counts.updated,
7000 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7001 ScopeRecordOutcome::Refused => &mut counts.refused,
7002 };
7003 *slot += 1;
7004 }
7005 counts
7006 }
7007}
7008
7009#[cfg(test)]
7010mod scope_outcome_count_tests {
7011 use super::*;
7012
7013 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7014 ScopeRecordResult {
7015 scope_ref: "r".to_string(),
7016 scope_epoch: 1,
7017 outcome,
7018 code: None,
7019 message: None,
7020 version: None,
7021 parent_state: None,
7022 }
7023 }
7024
7025 #[test]
7028 fn every_outcome_is_counted_in_its_own_field() {
7029 let results = [
7030 result(ScopeRecordOutcome::Created),
7031 result(ScopeRecordOutcome::Created),
7032 result(ScopeRecordOutcome::Replaced),
7033 result(ScopeRecordOutcome::Updated),
7034 result(ScopeRecordOutcome::Unchanged),
7035 result(ScopeRecordOutcome::Refused),
7036 result(ScopeRecordOutcome::Refused),
7037 result(ScopeRecordOutcome::Refused),
7038 ];
7039 assert_eq!(
7040 ScopeOutcomeCounts::of(&results),
7041 ScopeOutcomeCounts {
7042 created: 2,
7043 replaced: 1,
7044 updated: 1,
7045 unchanged: 1,
7046 refused: 3,
7047 }
7048 );
7049 }
7050}
7051
7052fn capability_census_trigger(
7056 old: Option<&CapabilityDeclarations>,
7057 new: Option<&CapabilityDeclarations>,
7058) -> bool {
7059 let old_provides = old
7060 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7061 .unwrap_or_default();
7062 let old_denies = old
7063 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7064 .unwrap_or_default();
7065 let new = new.cloned().unwrap_or(CapabilityDeclarations {
7066 provides: Vec::new(),
7067 requires: Vec::new(),
7068 must_never_reach: Vec::new(),
7069 });
7070
7071 new.provides
7072 .iter()
7073 .any(|capability| !old_provides.contains(capability))
7074 || new
7075 .must_never_reach
7076 .iter()
7077 .any(|capability| !old_denies.contains(capability))
7078}
7079
7080fn denied_capability<'a>(
7083 opening_manifest: &'a ModuleManifest,
7084 target_manifest: &ModuleManifest,
7085) -> Option<&'a str> {
7086 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7087 let target_capabilities = target_manifest.capabilities.as_ref()?;
7088 opening_capabilities
7089 .must_never_reach
7090 .iter()
7091 .find(|denied| {
7092 target_capabilities
7093 .provides
7094 .iter()
7095 .any(|provided| provided == *denied)
7096 })
7097 .map(String::as_str)
7098}
7099
7100fn catalog_update_frozen_field_message(
7101 registered: &ModuleManifest,
7102 provides: &[ProviderRole],
7103) -> Option<String> {
7104 let old_has_provides = !registered.provides.is_empty();
7105 let new_has_provides = !provides.is_empty();
7106 if old_has_provides != new_has_provides {
7107 return Some(format!(
7108 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7109 registered.module_id
7110 ));
7111 }
7112
7113 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
7114 return Some(format!(
7115 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7116 registered.module_id
7117 ));
7118 }
7119
7120 let registered_concurrency = manifest_concurrency(registered);
7121 let mut candidate = registered.clone();
7122 candidate.provides = provides.to_vec();
7123 let candidate_concurrency = manifest_concurrency(&candidate);
7124 if candidate_concurrency != registered_concurrency {
7125 return Some(format!(
7126 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7127 registered.module_id, registered_concurrency, candidate_concurrency
7128 ));
7129 }
7130
7131 None
7134}
7135
7136fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7137 manifest.provides.iter().any(is_routable_role)
7138}
7139
7140fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7146 manifest
7147 .provides
7148 .iter()
7149 .find_map(|provider| match provider {
7150 ProviderRole::ToolProvider { concurrency, .. }
7151 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7152 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7153 })
7154 .unwrap_or(Concurrency::ModuleManaged)
7155}
7156
7157fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7164 let has_management_surface = manifest
7165 .provides
7166 .iter()
7167 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7168 if !has_management_surface {
7169 return false;
7170 }
7171 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7172 return false;
7173 };
7174 let Some(provides) = raw
7175 .get("manifest")
7176 .and_then(|manifest| manifest.get("provides"))
7177 .and_then(serde_json::Value::as_array)
7178 else {
7179 return false;
7180 };
7181 provides.iter().any(|role| {
7186 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7187 && role.get("concurrency").is_none()
7188 })
7189}
7190
7191fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7192 if peer_version != PROTOCOL_VERSION {
7193 return Err(format!(
7194 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7195 ));
7196 }
7197 Ok(PROTOCOL_VERSION)
7198}
7199
7200fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7201 Frame::build_with_version(
7202 response_version(frame),
7203 FrameType::Pong,
7204 frame.header.flags,
7205 0,
7206 0,
7207 frame.header.corr,
7208 Vec::new(),
7209 )
7210 .map_err(RouterError::FrameBuild)
7211}
7212
7213pub(crate) fn control_error_frame(
7214 frame: &Frame,
7215 code: &'static str,
7216 message: impl Into<String>,
7217) -> Result<Frame, RouterError> {
7218 control_error_body_frame(
7219 frame,
7220 ErrorBody {
7221 code: code.to_string(),
7222 message: message.into(),
7223 detail: None,
7224 },
7225 )
7226}
7227
7228fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7229 let body = serde_json::to_vec(&error).map_err(|err| {
7230 RouterError::backend(
7231 0,
7232 frame.header.corr,
7233 format!("failed to encode control ERROR: {err}"),
7234 )
7235 })?;
7236
7237 Frame::build_with_version(
7238 response_version(frame),
7239 FrameType::Error,
7240 control_flags(),
7241 0,
7242 0,
7243 frame.header.corr,
7244 body,
7245 )
7246 .map_err(RouterError::FrameBuild)
7247}
7248
7249fn control_response_body_frame<T: Serialize>(
7250 frame: &Frame,
7251 reply: &T,
7252 label: &'static str,
7253) -> Result<Frame, RouterError> {
7254 let body = serde_json::to_vec(reply).map_err(|err| {
7255 RouterError::backend(
7256 0,
7257 frame.header.corr,
7258 format!("failed to encode {label}: {err}"),
7259 )
7260 })?;
7261
7262 Frame::build_with_version(
7263 response_version(frame),
7264 FrameType::Response,
7265 control_flags(),
7266 0,
7267 0,
7268 frame.header.corr,
7269 body,
7270 )
7271 .map_err(RouterError::FrameBuild)
7272}
7273
7274fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7304 match err {
7305 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7306 ForwardingError::NoModuleConnection => "target_unavailable",
7307 ForwardingError::ModuleReloading { .. } => "module_reloading",
7308 ForwardingError::ClientRouteChannelExhausted { .. }
7309 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7310 ForwardingError::StaleModuleEndpoint
7311 | ForwardingError::UnknownReservation { .. }
7312 | ForwardingError::ConnectionClosing { .. }
7313 | ForwardingError::ClientEgressClosed { .. }
7314 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7315 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7318 ForwardingError::RelayCorrelationExhausted
7319 | ForwardingError::RouteOpenBuild(_)
7320 | ForwardingError::Poisoned => "forwarding_error",
7321 }
7322}
7323
7324fn response_version(frame: &Frame) -> u8 {
7325 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7326 frame.header.ver
7327 } else {
7328 PROTOCOL_VERSION
7329 }
7330}
7331
7332fn control_flags() -> Flags {
7333 Flags::new(false, Priority::Passive, false)
7334}
7335
7336fn send_goodbye_target_best_effort(
7341 counters: &DaemonCounters,
7342 target: &GoodbyeTarget,
7343 context: &'static str,
7344) {
7345 let Ok(frame) = Frame::build_with_version(
7346 target.negotiated_ver,
7347 FrameType::Goodbye,
7348 control_flags(),
7349 target.channel,
7350 target.epoch,
7351 0,
7352 Vec::new(),
7353 ) else {
7354 return;
7355 };
7356 crate::forwarding::send_module_route_goodbye(
7357 counters,
7358 &target.sink,
7359 frame,
7360 target.module_id.as_deref(),
7361 context,
7362 );
7363}
7364
7365pub(crate) fn send_route_control_pushes(
7366 forwarding: &ForwardingTable,
7367 routes: Vec<EndpointRoute>,
7368 push: ClientControlPush,
7369) {
7370 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7371 for route in routes {
7372 let target = route.goodbye_target;
7373 if let Some((existing, channels)) = targets
7374 .iter_mut()
7375 .find(|(existing, _)| existing.connection_id == target.connection_id)
7376 {
7377 debug_assert_eq!(
7378 existing.negotiated_ver, target.negotiated_ver,
7379 "one connection cannot negotiate multiple frame versions"
7380 );
7381 if !channels.contains(&target.channel) {
7382 channels.push(target.channel);
7383 }
7384 continue;
7385 }
7386 let channel = target.channel;
7387 targets.push((target, vec![channel]));
7388 }
7389 for (target, mut channels) in targets {
7390 channels.sort_unstable();
7391 let mut push = push.clone();
7392 match &mut push {
7393 ClientControlPush::RouteClosing {
7394 channels: covered, ..
7395 }
7396 | ClientControlPush::RouteClosed {
7397 channels: covered, ..
7398 } => *covered = channels,
7399 }
7400 let body = match serde_json::to_vec(&push) {
7401 Ok(body) => body,
7402 Err(err) => {
7403 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7404 continue;
7405 }
7406 };
7407 let frame = match Frame::build_with_version(
7408 target.negotiated_ver,
7409 FrameType::Push,
7410 control_flags(),
7411 0,
7412 0,
7413 0,
7414 body.clone(),
7415 ) {
7416 Ok(frame) => frame,
7417 Err(err) => {
7418 warn!(
7419 route_channel = target.channel,
7420 error = %err,
7421 "failed to build route lifecycle control PUSH frame"
7422 );
7423 continue;
7424 }
7425 };
7426 if let Err(err) = target.sink.try_send(frame) {
7427 if target.close_on_delivery_failure() {
7428 warn!(
7429 target_connection_id = target.connection_id.get(),
7430 route_channel = target.channel,
7431 error = %err,
7432 "route lifecycle control PUSH was not delivered to client; closing target connection"
7433 );
7434 let _ = forwarding.escalate_client_delivery_failure(
7435 target.connection_id,
7436 target.channel,
7437 target.epoch,
7438 CloseReason::new(
7439 "route_lifecycle_push_delivery_failed",
7440 format!(
7441 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7442 target.channel
7443 ),
7444 ),
7445 crate::forwarding::UndeliveredFrame {
7446 module_id: target.module_id.as_deref(),
7447 sink: &target.sink,
7448 },
7449 );
7450 }
7451 }
7452 }
7453}
7454
7455#[cfg(test)]
7456mod tests {
7457 mod activity_read {
7458 include!("activity_read_tests.rs");
7459 }
7460
7461 mod lifecycle_activity {
7462 include!("control_lifecycle_tests.rs");
7463 }
7464 #[cfg(unix)]
7465 #[tokio::test]
7466 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
7467 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
7468 let path = dir.join("subc.jsonc");
7469 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
7470 "program":"/bin/sleep","args":["60"],"protocol":"none",
7471 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
7472 }}}).to_string()).unwrap();
7473 let mut configured = crate::daemon_config::load(&path)
7474 .unwrap()
7475 .unwrap()
7476 .modules
7477 .pop()
7478 .unwrap();
7479 let registry = std::sync::Arc::new(crate::Registry::default());
7480 let handle = crate::SupervisorHandle::new();
7481 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
7482 .with_handle(handle.clone());
7483 let module = supervisor
7484 .supervise_configured_with_health(
7485 configured.module_spec(),
7486 true,
7487 configured.health.clone(),
7488 None,
7489 configured.restart,
7490 )
7491 .unwrap();
7492 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
7493 let before = module.status().unwrap().pid;
7494 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
7495 configured.health.cadence = std::time::Duration::from_secs(3600);
7496 let health_only = handler
7497 .reconcile_supervised_modules(
7498 &supervisor,
7499 vec![configured.clone()],
7500 false,
7501 Principal::Direct,
7502 )
7503 .await
7504 .unwrap();
7505 assert!(
7506 health_only.changed_pending_reload.is_empty(),
7507 "health policy is already applied live"
7508 );
7509 assert_eq!(module.status().unwrap().pid, before);
7510 assert_eq!(
7511 module.configuration().unwrap().1.http,
7512 configured.health.http
7513 );
7514 configured.args = vec!["61".into()];
7515 let launch = handler
7516 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
7517 .await
7518 .unwrap();
7519 assert_eq!(launch.changed_pending_reload, ["stock"]);
7520 assert_eq!(
7521 module.status().unwrap().pid,
7522 before,
7523 "a launch edit is stored until reload"
7524 );
7525 module.drain().await.unwrap();
7526 }
7527 use cortexkit_test_support::ScratchDir;
7528 use std::{
7529 collections::BTreeMap,
7530 fmt,
7531 path::PathBuf,
7532 sync::{Arc, Mutex},
7533 time::Duration,
7534 };
7535
7536 use serde_json::{json, Value};
7537 use subc_protocol::{
7538 manifest::{
7539 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
7540 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
7541 },
7542 session::HealthStatus,
7543 FrameType,
7544 };
7545
7546 use super::*;
7547 use crate::{
7548 forwarding::{DataRoute, DataRouteState},
7549 registry::ChannelState,
7550 router::FrameSink,
7551 stderr_tail::DEFAULT_MAX_LINE_BYTES,
7552 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
7553 RouteCtx, Router,
7554 };
7555 use tokio::{
7556 sync::mpsc,
7557 time::{sleep, Instant},
7558 };
7559 use tracing::{
7560 field::{Field, Visit},
7561 Event, Subscriber,
7562 };
7563 use tracing_subscriber::{layer::Context, prelude::*, Layer};
7564
7565 fn fake_aft_stub_path() -> PathBuf {
7584 let mut path = std::env::current_exe().expect("current_exe available in tests");
7585 path.pop(); path.pop(); path.push(if cfg!(windows) {
7588 "fake-aft-stub.exe"
7589 } else {
7590 "fake-aft-stub"
7591 });
7592 assert!(
7593 path.exists(),
7594 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
7595 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
7596 path.display()
7597 );
7598 path
7599 }
7600
7601 fn client_retries(code: &str) -> bool {
7606 subc_protocol::error_codes::is_retryable_route_open(code)
7607 }
7608
7609 #[test]
7620 fn retryability_of_forwarding_codes_matches_the_failure() {
7621 let transient = [
7624 ForwardingError::NoModuleConnection,
7625 ForwardingError::ModuleReloading {
7626 module_id: "m".into(),
7627 },
7628 ForwardingError::StaleModuleEndpoint,
7629 ForwardingError::UnknownReservation {
7630 client_channel: 1,
7631 module_channel: 1,
7632 },
7633 ForwardingError::ConnectionClosing {
7634 connection_id: ConnectionId::new(1),
7635 },
7636 ForwardingError::ClientEgressClosed {
7637 connection_id: ConnectionId::new(1),
7638 },
7639 ForwardingError::ModuleEgressUnavailable {
7640 connection_id: ConnectionId::new(1),
7641 },
7642 ];
7643 for err in transient {
7644 let code = forwarding_error_code(&err);
7645 assert!(
7646 client_retries(code),
7647 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
7648 );
7649 }
7650
7651 let permanent = [
7656 ForwardingError::ConnectionRoleConflict {
7657 connection_id: ConnectionId::new(1),
7658 },
7659 ForwardingError::ClientRouteChannelExhausted {
7660 connection_id: ConnectionId::new(1),
7661 },
7662 ForwardingError::ModuleRouteChannelExhausted {
7663 endpoint: ModuleEndpointId {
7664 connection_id: ConnectionId::new(1),
7665 generation: 1,
7666 },
7667 },
7668 ForwardingError::RelayCorrelationExhausted,
7669 ForwardingError::RouteOpenBuild("x".into()),
7670 ForwardingError::Poisoned,
7671 ];
7672 for err in permanent {
7673 let code = forwarding_error_code(&err);
7674 assert!(
7675 !client_retries(code),
7676 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
7677 );
7678 }
7679 }
7680
7681 #[tokio::test]
7709 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
7710 let handler = ControlHandler::default();
7711 let frame =
7712 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
7713
7714 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
7718 assert_eq!(
7719 stamped,
7720 Principal::Direct,
7721 "a caller that proved nothing must not be stamped as a supervised module"
7722 );
7723
7724 let forged = handler
7729 .route_open_principal(
7730 &frame,
7731 Some(ConsumerIdentity {
7732 module_id: "aft".to_string(),
7733 launch_nonce: "not-a-real-nonce".to_string(),
7734 }),
7735 )
7736 .unwrap();
7737 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
7738 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
7739 }
7740
7741 #[test]
7758 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
7759 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
7760 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
7761 let ClientControlRequest::RouteOpen {
7762 consumer_identity, ..
7763 } = parsed
7764 else {
7765 panic!("route.open body must parse as RouteOpen");
7766 };
7767 assert_eq!(
7768 consumer_identity,
7769 Some(ConsumerIdentity {
7770 module_id: "aft".to_string(),
7771 launch_nonce: "n".to_string(),
7772 }),
7773 "the wire field name must reach the value route_open_principal reads"
7774 );
7775 }
7776
7777 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
7778 ModuleManifest::builder(module_id, "0.1.0")
7779 .protocol_ver(protocol_ver)
7780 .provides(vec![ProviderRole::ToolProvider {
7781 tools: vec![Tool {
7782 name: "read".to_string(),
7783 description: None,
7784 execution_mode: ExecutionMode::Pure,
7785 schema: json!({"type": "object"}),
7786 }],
7787 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
7788 concurrency: Concurrency::ModuleManaged,
7789 emits_push: true,
7790 sub_supervises: true,
7791 }])
7792 .build()
7793 }
7794
7795 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
7796 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
7797 }
7798
7799 fn hello_frame_with_control_ops(
7800 module_id: &str,
7801 protocol_ver: u8,
7802 corr: u64,
7803 control_ops: Option<Vec<String>>,
7804 ) -> Frame {
7805 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
7806 }
7807
7808 fn hello_frame_with_nonce(
7809 module_id: &str,
7810 protocol_ver: u8,
7811 corr: u64,
7812 launch_nonce: Option<&str>,
7813 ) -> Frame {
7814 hello_frame_full(
7815 module_id,
7816 protocol_ver,
7817 corr,
7818 None,
7819 launch_nonce.map(ToOwned::to_owned),
7820 )
7821 }
7822
7823 fn hello_frame_full(
7824 module_id: &str,
7825 protocol_ver: u8,
7826 corr: u64,
7827 control_ops: Option<Vec<String>>,
7828 launch_nonce: Option<String>,
7829 ) -> Frame {
7830 let body = serde_json::to_vec(&ModuleHelloBody {
7831 manifest: manifest(module_id, protocol_ver),
7832 protocol_ver,
7833 control_ops,
7834 launch_nonce,
7835 })
7836 .unwrap();
7837 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7838 }
7839
7840 fn non_routable_hello_frame_with_control_ops(
7841 module_id: &str,
7842 corr: u64,
7843 control_ops: Option<Vec<String>>,
7844 ) -> Frame {
7845 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
7846 manifest.provides.clear();
7847 let body = serde_json::to_vec(&ModuleHelloBody {
7848 manifest,
7849 protocol_ver: PROTOCOL_VERSION,
7850 control_ops,
7851 launch_nonce: None,
7852 })
7853 .unwrap();
7854 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7855 }
7856
7857 fn capability_grammar_hello_frame(
7858 capabilities: Value,
7859 runtime_computed: Option<Value>,
7860 corr: u64,
7861 ) -> Frame {
7862 let mut body = serde_json::to_value(ModuleHelloBody {
7863 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
7864 protocol_ver: PROTOCOL_VERSION,
7865 control_ops: None,
7866 launch_nonce: None,
7867 })
7868 .expect("HELLO body serializes");
7869 body["manifest"]["capabilities"] = capabilities;
7870 if let Some(runtime_computed) = runtime_computed {
7871 body["runtime_computed"] = runtime_computed;
7872 }
7873 Frame::build(
7874 FrameType::Hello,
7875 control_flags(),
7876 0,
7877 0,
7878 corr,
7879 serde_json::to_vec(&body).expect("HELLO body reserializes"),
7880 )
7881 .expect("HELLO frame builds")
7882 }
7883
7884 fn channel_request(channel: u16, corr: u64) -> Frame {
7885 Frame::build(
7886 FrameType::Request,
7887 Flags::new(true, Priority::Interactive, false),
7888 channel,
7889 0,
7890 corr,
7891 b"opaque".to_vec(),
7892 )
7893 .unwrap()
7894 }
7895
7896 fn route_ctx(
7897 connection_id: ConnectionId,
7898 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
7899 let (tx, rx) = mpsc::channel(8);
7900 (
7901 RouteCtx {
7902 connection_id,
7903 egress: FrameSink::new(tx),
7904 },
7905 rx,
7906 )
7907 }
7908
7909 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
7910 serde_json::from_slice(&frame.body).unwrap()
7911 }
7912
7913 async fn hello_via_sink(
7918 handler: &ControlHandler,
7919 ctx: &RouteCtx,
7920 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7921 hello: Frame,
7922 ) -> Frame {
7923 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
7924 assert!(
7925 replies.is_empty(),
7926 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
7927 );
7928 let ack = rx
7929 .try_recv()
7930 .expect("HELLO_ACK is queued on the module sink")
7931 .frame;
7932 assert_eq!(ack.header.ty, FrameType::HelloAck);
7933 ack
7934 }
7935
7936 fn parse_error(frame: &Frame) -> Value {
7937 serde_json::from_slice(&frame.body).unwrap()
7938 }
7939
7940 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
7941 serde_json::from_slice(&frame.body).unwrap()
7942 }
7943
7944 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
7945 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
7946 route_channel,
7947 route_epoch: 0,
7948 kind,
7949 })
7950 .unwrap();
7951 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7952 }
7953
7954 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
7955 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
7956 module_id: module_id.to_string(),
7957 })
7958 .unwrap();
7959 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7960 }
7961
7962 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
7963 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
7964 }
7965
7966 fn route_open_frame_with_consumer_capabilities(
7967 corr: u64,
7968 module_id: &str,
7969 project_root: ScratchDir,
7970 consumer_capabilities: Option<Vec<String>>,
7971 ) -> Frame {
7972 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7973 target: RouteTarget::ToolProvider {
7974 module_id: module_id.to_string(),
7975 },
7976 identity: BindIdentity::new(
7977 project_root.path().to_path_buf(),
7978 "unit".to_string(),
7979 "session".to_string(),
7980 ),
7981 consumer_identity: None,
7982 consumer_capabilities,
7983 role_versions: None,
7984 admission_facts: None,
7985 scope: None,
7986 })
7987 .unwrap();
7988 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7989 }
7990
7991 fn route_open_frame_with_role_versions(
7992 corr: u64,
7993 module_id: &str,
7994 project_root: ScratchDir,
7995 role_versions: Option<BTreeMap<String, String>>,
7996 ) -> Frame {
7997 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7998 target: RouteTarget::ToolProvider {
7999 module_id: module_id.to_string(),
8000 },
8001 identity: BindIdentity::new(
8002 project_root.path().to_path_buf(),
8003 "unit".to_string(),
8004 format!("session-{corr}"),
8005 ),
8006 consumer_identity: None,
8007 consumer_capabilities: None,
8008 role_versions,
8009 admission_facts: None,
8010 scope: None,
8011 })
8012 .unwrap();
8013 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8014 }
8015
8016 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8017 entries
8018 .iter()
8019 .map(|(role, version)| (role.to_string(), version.to_string()))
8020 .collect()
8021 }
8022
8023 fn route_open_frame_with_admission_facts(
8024 corr: u64,
8025 module_id: &str,
8026 project_root: ScratchDir,
8027 consumer_identity: Option<subc_control::ConsumerIdentity>,
8028 facts: Option<Value>,
8029 ) -> Frame {
8030 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8031 target: RouteTarget::ToolProvider {
8032 module_id: module_id.to_string(),
8033 },
8034 identity: BindIdentity::new(
8035 project_root.path().to_path_buf(),
8036 "unit".to_string(),
8037 format!("session-{corr}"),
8038 ),
8039 consumer_identity,
8040 consumer_capabilities: None,
8041 role_versions: None,
8042 admission_facts: facts,
8043 scope: None,
8044 })
8045 .unwrap();
8046 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8047 }
8048
8049 #[derive(Clone, Default)]
8050 struct EventCapture {
8051 events: Arc<Mutex<Vec<CapturedEvent>>>,
8052 }
8053
8054 #[derive(Clone, Debug)]
8055 struct CapturedEvent {
8056 target: String,
8057 level: tracing::Level,
8058 fields: BTreeMap<String, String>,
8059 }
8060
8061 impl EventCapture {
8062 fn events(&self) -> Vec<CapturedEvent> {
8063 self.events.lock().unwrap().clone()
8064 }
8065 }
8066
8067 impl<S> Layer<S> for EventCapture
8068 where
8069 S: Subscriber,
8070 {
8071 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8072 let mut visitor = EventFieldVisitor::default();
8073 event.record(&mut visitor);
8074 self.events.lock().unwrap().push(CapturedEvent {
8075 target: event.metadata().target().to_string(),
8076 level: *event.metadata().level(),
8077 fields: visitor.fields,
8078 });
8079 }
8080 }
8081
8082 #[derive(Default)]
8083 struct EventFieldVisitor {
8084 fields: BTreeMap<String, String>,
8085 }
8086
8087 impl Visit for EventFieldVisitor {
8088 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8089 self.fields
8090 .insert(field.name().to_string(), format!("{value:?}"));
8091 }
8092 }
8093
8094 fn health_response(corr: u64, status: HealthStatus) -> Frame {
8095 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8096 status,
8097 detail: Some("warming".to_string()),
8098 metrics: Some(json!({"queue_depth": 3})),
8099 })
8100 .unwrap();
8101 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8102 }
8103
8104 fn route_bind_ack(corr: u64) -> Frame {
8105 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8106 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8107 }
8108
8109 fn unique_project_root(label: &str) -> ScratchDir {
8110 ScratchDir::new(label)
8111 }
8112
8113 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8114 match parse_route_poll(frame) {
8115 ClientControlResponse::RoutePoll {
8116 status: None,
8117 live: Some(live),
8118 ..
8119 } => assert_eq!(live, expected_live),
8120 other => panic!("unexpected route.poll response: {other:?}"),
8121 }
8122 }
8123
8124 fn bind_liveness_route(
8125 registry: &Registry,
8126 forwarding: &ForwardingTable,
8127 module_id: &str,
8128 ) -> (RouteCtx, u16, u32) {
8129 let module_connection = ConnectionId::new(101);
8130 let client_connection = ConnectionId::new(202);
8131 let registration = registry
8132 .register_with_control_ops(
8133 manifest(module_id, PROTOCOL_VERSION),
8134 PROTOCOL_VERSION,
8135 module_connection,
8136 module_baseline_control_ops(),
8137 )
8138 .unwrap();
8139 let (module_tx, _module_rx) = mpsc::channel(8);
8140 let endpoint = forwarding
8141 .register_module_connection(
8142 module_connection,
8143 module_id.to_string(),
8144 PROTOCOL_VERSION,
8145 manifest_concurrency(®istration.manifest),
8146 FrameSink::new(module_tx),
8147 )
8148 .unwrap();
8149 let (client_ctx, _client_rx) = route_ctx(client_connection);
8150 let pending = forwarding
8151 .begin_route_bind_relay_for_test(
8152 client_connection,
8153 client_ctx.egress.clone(),
8154 1,
8155 module_id,
8156 )
8157 .unwrap();
8158 assert_eq!(pending.endpoint, endpoint);
8159 let route_channel = pending.client_channel;
8160 let route_epoch = pending.client_epoch;
8161 forwarding
8162 .complete_pending_relay(
8163 module_connection,
8164 pending.corr,
8165 RouteBindRelayOutcome::Accepted,
8166 )
8167 .unwrap();
8168 (client_ctx, route_channel, route_epoch)
8169 }
8170
8171 struct FakeProcessLiveness {
8172 live: Option<bool>,
8173 }
8174
8175 impl ModuleProcessLiveness for FakeProcessLiveness {
8176 fn process_live(&self, _module_id: &str) -> Option<bool> {
8177 self.live
8178 }
8179 }
8180
8181 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8182 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8183 {
8184 let registry = Arc::new(Registry::default());
8185 let supervisor_handle = SupervisorHandle::new();
8186 let supervisor = Supervisor::new_for_test(
8187 Arc::clone(®istry),
8188 RestartPolicy::new(1, Duration::from_millis(10)),
8189 )
8190 .with_handle(supervisor_handle.clone());
8191 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8192 let module = supervisor
8193 .spawn(ModuleSpec {
8194 module_id: "stderr-tail-wire".to_string(),
8195 program: fake_aft_stub_path(),
8196 args: Vec::new(),
8197 env: vec![
8198 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8199 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8200 ],
8201 reserved: false,
8202 reserved_prefixes: Vec::new(),
8203 protocol: ModuleProtocol::Subc,
8204 overlap: Default::default(),
8205 })
8206 .unwrap();
8207
8208 let deadline = Instant::now() + Duration::from_secs(5);
8209 loop {
8210 let tail = module.stderr_tail(None, None);
8211 if tail
8212 .entries
8213 .iter()
8214 .any(|entry| matches!(entry, TailEntry::ProcessStart))
8215 && tail.entries.iter().any(|entry| {
8216 matches!(
8217 entry,
8218 TailEntry::Line {
8219 truncated: true,
8220 ..
8221 }
8222 )
8223 })
8224 {
8225 break;
8226 }
8227 assert!(
8228 Instant::now() < deadline,
8229 "module did not produce a truncated line and restart boundary: {tail:?}"
8230 );
8231 sleep(Duration::from_millis(10)).await;
8232 }
8233
8234 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8235 let request = ClientControlRequest::SupervisorStderrTail {
8236 module_id: "stderr-tail-wire".to_string(),
8237 max_lines: None,
8238 max_bytes: None,
8239 };
8240 let frame = Frame::build(
8241 FrameType::Request,
8242 control_flags(),
8243 0,
8244 0,
8245 1,
8246 serde_json::to_vec(&request).unwrap(),
8247 )
8248 .unwrap();
8249 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8250 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8251 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8252 serde_json::from_slice(&responses[0].body).unwrap()
8253 else {
8254 panic!("expected supervisor.stderr_tail response");
8255 };
8256
8257 assert!(
8258 tail.entries
8259 .iter()
8260 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8261 "the control response lost the restart boundary"
8262 );
8263 let Some(StderrTailEntry::Line {
8264 text,
8265 truncated,
8266 at_ms,
8267 }) = tail.entries.iter().find(|entry| {
8268 matches!(
8269 entry,
8270 StderrTailEntry::Line {
8271 truncated: true,
8272 ..
8273 }
8274 )
8275 })
8276 else {
8277 panic!("the control response lost the truncated line");
8278 };
8279 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8280 assert!(*truncated);
8281 assert!(
8282 at_ms.is_some(),
8283 "the control response lost the line's capture time"
8284 );
8285 }
8286
8287 #[tokio::test(flavor = "current_thread")]
8292 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8293 let dir = ScratchDir::new("terminals-off-worker");
8294 let journal_path = dir.join("terminals.jsonl");
8295 let registry = Arc::new(Registry::default());
8296 let supervisor_handle = SupervisorHandle::new();
8297 let supervisor =
8298 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8299 .with_handle(supervisor_handle.clone())
8300 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8301 let module = supervisor
8302 .spawn(ModuleSpec {
8303 module_id: "terminal-off-worker".to_string(),
8304 program: fake_aft_stub_path(),
8305 args: Vec::new(),
8306 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8307 reserved: false,
8308 reserved_prefixes: Vec::new(),
8309 protocol: ModuleProtocol::Subc,
8310 overlap: Default::default(),
8311 })
8312 .unwrap();
8313 let deadline = Instant::now() + Duration::from_secs(5);
8314 while module.terminal_history().entries.len() != 2 {
8315 assert!(Instant::now() < deadline, "module did not record two exits");
8316 sleep(Duration::from_millis(10)).await;
8317 }
8318
8319 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8320 let handler =
8321 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
8322 let frame = Frame::build(
8323 FrameType::Request,
8324 control_flags(),
8325 0,
8326 0,
8327 1,
8328 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8329 module_id: "terminal-off-worker".to_string(),
8330 })
8331 .unwrap(),
8332 )
8333 .unwrap();
8334 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8335 let spawned_at = std::time::Instant::now();
8336 let read = tokio::spawn({
8337 let handler = Arc::clone(&handler);
8338 async move { handler.handle_control_frame(&ctx, frame).await }
8339 });
8340 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8343 .await
8344 .unwrap()
8345 .expect("the history read reached its pause");
8346 let elapsed = spawned_at.elapsed();
8347 assert!(
8348 elapsed < Duration::from_secs(2) && !read.is_finished(),
8349 "this task could not run while the history read was paused \
8350 (resumed after {elapsed:?}, read finished: {})",
8351 read.is_finished()
8352 );
8353
8354 drop(release);
8355 let responses = read.await.unwrap().unwrap();
8356 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8357 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8358 panic!("expected supervisor.terminals response");
8359 };
8360 assert_eq!(terminals.entries.len(), 2);
8361 assert_eq!(terminals.journal_skipped_lines, 0);
8362 assert_eq!(terminals.journal_read_errors, 0);
8363 }
8364
8365 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8366 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8367 let registry = Arc::new(Registry::default());
8368 let supervisor_handle = SupervisorHandle::new();
8369 let supervisor =
8370 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8371 .with_handle(supervisor_handle.clone());
8372 let module = supervisor
8373 .spawn(ModuleSpec {
8374 module_id: "terminal-golden".to_string(),
8375 program: fake_aft_stub_path(),
8376 args: Vec::new(),
8377 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8378 reserved: false,
8379 reserved_prefixes: Vec::new(),
8380 protocol: ModuleProtocol::Subc,
8381 overlap: Default::default(),
8382 })
8383 .unwrap();
8384
8385 let deadline = Instant::now() + Duration::from_secs(5);
8386 while module.terminal_history().entries.len() != 2 {
8387 assert!(
8388 Instant::now() < deadline,
8389 "module did not retain two terminal exits: {:?}",
8390 module.terminal_history()
8391 );
8392 sleep(Duration::from_millis(10)).await;
8393 }
8394
8395 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8396 let request = ClientControlRequest::SupervisorTerminals {
8397 module_id: "terminal-golden".to_string(),
8398 };
8399 let frame = Frame::build(
8400 FrameType::Request,
8401 control_flags(),
8402 0,
8403 0,
8404 1,
8405 serde_json::to_vec(&request).unwrap(),
8406 )
8407 .unwrap();
8408 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8409 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8410 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8411 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8412 panic!("expected supervisor.terminals response");
8413 };
8414 assert_eq!(terminals.entries.len(), 2);
8415 assert_eq!(terminals.dropped, 0);
8416
8417 let mut rendered = serde_json::to_value(response).unwrap();
8418 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
8421 for (index, entry) in rendered["entries"]
8422 .as_array_mut()
8423 .expect("terminal response entries array")
8424 .iter_mut()
8425 .enumerate()
8426 {
8427 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
8428 }
8429
8430 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8431 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
8432 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
8433 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8434 std::fs::write(&golden_path, &serialized).unwrap();
8435 }
8436 let expected: Value =
8437 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
8438 assert_eq!(rendered, expected);
8439 }
8440
8441 #[test]
8442 fn hello_registers_manifest_and_returns_ack() {
8443 let registry = Arc::new(Registry::default());
8444 let handler = ControlHandler::new(Arc::clone(®istry));
8445 let conn = ConnectionId::new(1);
8446
8447 let responses = handler
8448 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
8449 .unwrap();
8450
8451 assert_eq!(responses.len(), 1);
8452 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8453 assert_eq!(responses[0].header.channel, 0);
8454 assert_eq!(responses[0].header.corr, 7);
8455 let ack = parse_ack(&responses[0]);
8456 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8457 assert!(ack
8458 .subc_capabilities
8459 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
8460 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
8461 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
8462 assert!(ack
8463 .subc_ops
8464 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
8465 assert!(ack
8466 .subc_ops
8467 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
8468
8469 let registration = registry.get_module("aft").unwrap().unwrap();
8470 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
8471 assert_eq!(registration.state, ChannelState::Active);
8472 assert_eq!(registration.connection_id, conn);
8473 assert_eq!(registration.control_ops, module_baseline_control_ops());
8474 }
8475
8476 #[test]
8477 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
8478 let invalid_identifiers = [
8479 ("case_change", "credentials-Provider/v1"),
8480 ("leading_zero", "credentials-provider/v01"),
8481 ("trailing_hyphen", "credentials-provider-/v1"),
8482 ("consecutive_hyphens", "credentials--provider/v1"),
8483 ("uppercase", "Credentials-provider/v1"),
8484 ("missing_v", "credentials-provider/1"),
8485 ("whitespace", "credentials provider/v1"),
8486 ("zero_version", "credentials-provider/v0"),
8487 ("out_of_range_version", "credentials-provider/v4294967296"),
8488 (
8489 "overlength_name",
8490 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
8491 ),
8492 ];
8493 let mut cases = invalid_identifiers
8494 .into_iter()
8495 .map(|(name, identifier)| {
8496 (
8497 format!("identifier_{name}"),
8498 "capabilities.provides[0]".to_string(),
8499 identifier.to_string(),
8500 json!({ "provides": [identifier] }),
8501 None,
8502 )
8503 })
8504 .collect::<Vec<_>>();
8505 cases.extend([
8506 (
8507 "unknown_need".to_string(),
8508 "capabilities.requires[0].need".to_string(),
8509 "deferred".to_string(),
8510 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
8511 None,
8512 ),
8513 (
8514 "duplicate_provides".to_string(),
8515 "capabilities.provides[1]".to_string(),
8516 "credentials-provider/v1".to_string(),
8517 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
8518 None,
8519 ),
8520 (
8521 "duplicate_must_never_reach".to_string(),
8522 "capabilities.must_never_reach[1]".to_string(),
8523 "credentials-provider/v1".to_string(),
8524 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
8525 None,
8526 ),
8527 (
8528 "duplicate_requires_same_need".to_string(),
8529 "capabilities.requires[1]".to_string(),
8530 "credentials-provider/v1".to_string(),
8531 json!({ "requires": [
8532 { "capability": "credentials-provider/v1", "need": "required" },
8533 { "capability": "credentials-provider/v1", "need": "required" }
8534 ] }),
8535 None,
8536 ),
8537 (
8538 "duplicate_requires_conflicting_need".to_string(),
8539 "capabilities.requires[1]".to_string(),
8540 "credentials-provider/v1".to_string(),
8541 json!({ "requires": [
8542 { "capability": "credentials-provider/v1", "need": "required" },
8543 { "capability": "credentials-provider/v1", "need": "optional" }
8544 ] }),
8545 None,
8546 ),
8547 (
8548 "capabilities_root_pointer".to_string(),
8549 "runtime_computed[0]".to_string(),
8550 "/capabilities".to_string(),
8551 json!({}),
8552 Some(json!(["/capabilities"])),
8553 ),
8554 (
8555 "capabilities_descendant_pointer".to_string(),
8556 "runtime_computed[0]".to_string(),
8557 "/capabilities/provides".to_string(),
8558 json!({}),
8559 Some(json!(["/capabilities/provides"])),
8560 ),
8561 (
8562 "malformed_pointer_without_leading_slash".to_string(),
8563 "runtime_computed[0]".to_string(),
8564 "capabilities".to_string(),
8565 json!({}),
8566 Some(json!(["capabilities"])),
8567 ),
8568 (
8569 "malformed_pointer_escape".to_string(),
8570 "runtime_computed[0]".to_string(),
8571 "/roles/~2/tools".to_string(),
8572 json!({}),
8573 Some(json!(["/roles/~2/tools"])),
8574 ),
8575 (
8576 "unknown_capabilities_field".to_string(),
8577 "capabilities.future".to_string(),
8578 "<array>".to_string(),
8579 json!({ "future": [] }),
8580 None,
8581 ),
8582 ]);
8583
8584 for (index, (name, field, value, capabilities, runtime_computed)) in
8585 cases.into_iter().enumerate()
8586 {
8587 let registry = Arc::new(Registry::default());
8588 let handler = ControlHandler::new(Arc::clone(®istry));
8589 let response = handler
8590 .handle_control(
8591 ConnectionId::new((index + 1) as u64),
8592 capability_grammar_hello_frame(
8593 capabilities,
8594 runtime_computed,
8595 index as u64 + 1,
8596 ),
8597 )
8598 .expect("invalid HELLO returns a refusal");
8599
8600 assert_eq!(response.len(), 1, "{name} must emit one refusal");
8601 let error = parse_error(&response[0]);
8602 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
8603 let message = error["message"]
8604 .as_str()
8605 .expect("error message is a string");
8606 assert!(
8607 message.contains(&field),
8608 "{name}: field missing from {message}"
8609 );
8610 assert!(
8611 message.contains(&value),
8612 "{name}: value missing from {message}"
8613 );
8614 assert_eq!(
8615 registry
8616 .active_registration_count()
8617 .expect("registry reads"),
8618 0,
8619 "{name}: refused HELLO must not create a catalog entry"
8620 );
8621 }
8622 }
8623
8624 #[test]
8625 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
8626 let registry = Arc::new(Registry::default());
8627 let handler = ControlHandler::new(Arc::clone(®istry));
8628 let capabilities = json!({
8629 "provides": ["credentials-provider/v1"],
8630 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
8631 "must_never_reach": ["federation-transport/v1"]
8632 });
8633 let response = handler
8634 .handle_control(
8635 ConnectionId::new(99),
8636 capability_grammar_hello_frame(
8637 capabilities.clone(),
8638 Some(json!(["/roles/0/tools"])),
8639 99,
8640 ),
8641 )
8642 .expect("valid HELLO registers");
8643 assert_eq!(response[0].header.ty, FrameType::HelloAck);
8644
8645 let request = Frame::build(
8646 FrameType::Request,
8647 control_flags(),
8648 0,
8649 0,
8650 100,
8651 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8652 .expect("catalog request serializes"),
8653 )
8654 .expect("catalog request frame builds");
8655 let response = handler
8656 .handle_catalog_list(request, None)
8657 .expect("catalog list succeeds");
8658 let ClientControlResponse::CatalogList { modules, .. } =
8659 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
8660 else {
8661 panic!("catalog request must return catalog.list");
8662 };
8663 assert_eq!(modules.len(), 1);
8664 assert_eq!(
8665 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
8666 capabilities
8667 );
8668 }
8669
8670 #[test]
8671 fn catalog_list_mirrors_management_operation_description() {
8672 let registry = Arc::new(Registry::default());
8673 let handler = ControlHandler::new(Arc::clone(®istry));
8674 let description = "List managed records and return their identifiers and metadata.";
8675 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
8676 manifest.provides = vec![ProviderRole::ManagementSurface {
8677 operations: vec![ManagementOperation {
8678 name: "records.list".to_string(),
8679 kind: ManagementOperationKind::Query,
8680 description: Some(description.to_string()),
8681 }],
8682 config_schema: json!({"type": "object"}),
8683 observability: vec![ObservabilitySurface {
8684 name: "records.stats".to_string(),
8685 kind: ObservabilityKind::Snapshot,
8686 }],
8687 identity_scope: vec![IdentityScope::Project],
8688 concurrency: Concurrency::ModuleManaged,
8689 }];
8690 registry
8691 .register_with_control_ops(
8692 manifest,
8693 PROTOCOL_VERSION,
8694 ConnectionId::new(99),
8695 Vec::new(),
8696 )
8697 .expect("described management manifest registers");
8698
8699 let request = Frame::build(
8700 FrameType::Request,
8701 control_flags(),
8702 0,
8703 0,
8704 100,
8705 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8706 .expect("catalog request serializes"),
8707 )
8708 .expect("catalog request frame builds");
8709 let response = handler
8710 .handle_catalog_list(request, None)
8711 .expect("catalog list succeeds");
8712 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8713 assert_eq!(
8714 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
8715 "catalog.list must preserve the declared operation description verbatim"
8716 );
8717 }
8718
8719 #[test]
8720 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
8721 let registry = Arc::new(Registry::default());
8722 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8723 [("vault".to_string(), true), ("squatter".to_string(), true)],
8724 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8725 );
8726 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
8727 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8728 provides: vec!["credentials-provider/v1".to_string()],
8729 requires: Vec::new(),
8730 must_never_reach: Vec::new(),
8731 });
8732 let frame = Frame::build(
8733 FrameType::Hello,
8734 control_flags(),
8735 0,
8736 0,
8737 77,
8738 serde_json::to_vec(&ModuleHelloBody {
8739 manifest: squatter,
8740 protocol_ver: PROTOCOL_VERSION,
8741 control_ops: None,
8742 launch_nonce: None,
8743 })
8744 .expect("HELLO serializes"),
8745 )
8746 .expect("HELLO frame builds");
8747 let response = handler
8748 .handle_control(ConnectionId::new(77), frame)
8749 .expect("reserved claim receives a typed refusal");
8750 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8751 assert_eq!(
8752 registry
8753 .active_registration_count()
8754 .expect("registry reads"),
8755 0,
8756 "a reserved capability refusal must not leave a catalog entry"
8757 );
8758 }
8759
8760 #[test]
8761 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
8762 for settlement in ["timeout", "inconclusive", "drop"] {
8763 let breakers = RouteBindBreakers::default();
8764 let RouteBindAdmission::Admitted {
8765 guard: mut old,
8766 probe: false,
8767 } = breakers.admit("prov")
8768 else {
8769 panic!("ordinary relay admitted")
8770 };
8771 let RouteBindAdmission::Admitted {
8772 guard: mut opener, ..
8773 } = breakers.admit("prov")
8774 else {
8775 panic!("second relay admitted")
8776 };
8777 assert!(
8778 !opener
8779 .record_timeout(1, Duration::ZERO)
8780 .unwrap()
8781 .reopened_after_probe
8782 );
8783 let RouteBindAdmission::Admitted {
8784 guard: mut probe,
8785 probe: true,
8786 } = breakers.admit("prov")
8787 else {
8788 panic!("one cooldown probe admitted")
8789 };
8790 match settlement {
8791 "timeout" => assert!(
8792 !old.record_timeout(1, Duration::ZERO)
8793 .unwrap()
8794 .reopened_after_probe
8795 ),
8796 "inconclusive" => old.record_inconclusive(),
8797 "drop" => drop(old),
8798 _ => unreachable!(),
8799 }
8800 assert!(
8801 matches!(
8802 breakers.admit("prov"),
8803 RouteBindAdmission::Refused {
8804 probe_in_flight: true,
8805 ..
8806 }
8807 ),
8808 "{settlement} of a pre-open relay cannot release the real probe"
8809 );
8810 assert!(
8811 probe
8812 .record_timeout(1, Duration::ZERO)
8813 .unwrap()
8814 .reopened_after_probe
8815 );
8816 assert!(matches!(
8817 breakers.admit("prov"),
8818 RouteBindAdmission::Admitted { probe: true, .. }
8819 ));
8820 }
8821 let breakers = RouteBindBreakers::default();
8822 let admit = || match breakers.admit("prov") {
8823 RouteBindAdmission::Admitted { guard, .. } => guard,
8824 _ => panic!("relay admitted"),
8825 };
8826 admit().record_timeout(1, Duration::ZERO);
8827 let mut old_probe = admit();
8828 breakers.reset_for_new_module_connection("prov");
8829 admit().record_timeout(1, Duration::ZERO);
8830 let _new_probe = admit();
8831 old_probe.record_inconclusive();
8832 assert!(matches!(
8833 breakers.admit("prov"),
8834 RouteBindAdmission::Refused {
8835 probe_in_flight: true,
8836 ..
8837 }
8838 ));
8839 }
8840
8841 #[tokio::test]
8842 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
8843 for candidate in [false, true] {
8844 let registry = Arc::new(Registry::default());
8845 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8846 [("vault".to_string(), true), ("squatter".to_string(), true)],
8847 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8848 );
8849 let conn = ConnectionId::new(77);
8850 let (ctx, mut rx) = route_ctx(conn);
8851 let initial = capability_manifest("squatter", &[], &[]);
8852 if candidate {
8853 registry
8854 .register_candidate_with_control_ops(
8855 initial.clone(),
8856 PROTOCOL_VERSION,
8857 conn,
8858 module_baseline_control_ops(),
8859 )
8860 .unwrap();
8861 handler
8862 .forwarding
8863 .register_candidate_module_connection(
8864 conn,
8865 "squatter".to_string(),
8866 PROTOCOL_VERSION,
8867 manifest_concurrency(&initial),
8868 ctx.egress.clone(),
8869 )
8870 .unwrap();
8871 } else {
8872 hello_via_sink(
8873 &handler,
8874 &ctx,
8875 &mut rx,
8876 hello_frame_with_manifest(initial.clone(), 1),
8877 )
8878 .await;
8879 }
8880 let response = handler
8881 .handle_control_frame(
8882 &ctx,
8883 catalog_update_with_capabilities_frame(
8884 2,
8885 capability_manifest("squatter", &["credentials-provider/v1"], &[])
8886 .capabilities
8887 .unwrap(),
8888 ),
8889 )
8890 .await
8891 .unwrap();
8892 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8893 assert_eq!(
8894 registry
8895 .get_module_by_connection(conn)
8896 .unwrap()
8897 .unwrap()
8898 .manifest,
8899 initial
8900 );
8901 }
8902 }
8903
8904 #[test]
8905 fn server_describe_surfaces_required_capability_verdict_fields() {
8906 let registry = Arc::new(Registry::default());
8907 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8908 [
8909 ("consumer".to_string(), true),
8910 ("provider".to_string(), false),
8911 ],
8912 BTreeMap::new(),
8913 );
8914 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
8915 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8916 provides: Vec::new(),
8917 requires: vec![subc_protocol::manifest::CapabilityRequirement {
8918 capability: "credentials-provider/v1".to_string(),
8919 need: subc_protocol::manifest::CapabilityNeed::Required,
8920 }],
8921 must_never_reach: Vec::new(),
8922 });
8923 let hello = Frame::build(
8924 FrameType::Hello,
8925 control_flags(),
8926 0,
8927 0,
8928 78,
8929 serde_json::to_vec(&ModuleHelloBody {
8930 manifest: consumer,
8931 protocol_ver: PROTOCOL_VERSION,
8932 control_ops: None,
8933 launch_nonce: None,
8934 })
8935 .expect("HELLO serializes"),
8936 )
8937 .expect("HELLO frame builds");
8938 handler
8939 .handle_control(ConnectionId::new(78), hello)
8940 .expect("consumer registers");
8941 let describe = Frame::build(
8942 FrameType::Request,
8943 control_flags(),
8944 0,
8945 0,
8946 79,
8947 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
8948 .expect("request serializes"),
8949 )
8950 .expect("describe frame builds");
8951 let response = handler
8952 .handle_server_describe(describe)
8953 .expect("server.describe succeeds");
8954 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
8955 let requirement = &rendered["capability_requirements"][0];
8956 assert_eq!(requirement["consumer"], "consumer");
8957 assert_eq!(requirement["verdict"], "never_provided");
8958 assert_eq!(requirement["episode_seq"], 1);
8959 assert_eq!(requirement["config_satisfiable"], false);
8960 assert_eq!(requirement["runtime_available"], false);
8961 assert!(requirement["detail"]
8962 .as_str()
8963 .expect("detail string")
8964 .contains("credentials-provider/v1"));
8965 }
8966
8967 #[test]
8968 fn catalog_list_omits_capabilities_for_legacy_manifest() {
8969 let registry = Arc::new(Registry::default());
8970 let handler = ControlHandler::new(Arc::clone(®istry));
8971 let hello = handler
8972 .handle_control(
8973 ConnectionId::new(101),
8974 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
8975 )
8976 .expect("legacy HELLO registers");
8977 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
8978
8979 let request = Frame::build(
8980 FrameType::Request,
8981 control_flags(),
8982 0,
8983 0,
8984 102,
8985 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8986 .expect("catalog request serializes"),
8987 )
8988 .expect("catalog request frame builds");
8989 let response = handler
8990 .handle_catalog_list(request, None)
8991 .expect("catalog list succeeds");
8992 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8993 assert!(
8994 body["modules"][0].get("capabilities").is_none(),
8995 "legacy manifest must retain an absent capabilities field on catalog.list"
8996 );
8997 }
8998
8999 #[test]
9000 fn hello_ack_omits_storage_when_no_storage_config() {
9001 let registry = Arc::new(Registry::default());
9002 let handler = ControlHandler::new(Arc::clone(®istry));
9003 let responses = handler
9004 .handle_control(
9005 ConnectionId::new(1),
9006 hello_frame("aft", PROTOCOL_VERSION, 7),
9007 )
9008 .unwrap();
9009 let ack = parse_ack(&responses[0]);
9010 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
9011 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
9012 }
9013
9014 #[tokio::test]
9015 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
9016 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
9017 let registry = Arc::new(Registry::default());
9018 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
9019 let responses = handler
9020 .handle_control(
9021 ConnectionId::new(1),
9022 hello_frame("aft", PROTOCOL_VERSION, 7),
9023 )
9024 .unwrap();
9025 let ack = parse_ack(&responses[0]);
9026 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
9027
9028 let described = handler
9029 .handle_control_frame(
9030 &route_ctx(ConnectionId::new(2)).0,
9031 Frame::build(
9032 FrameType::Request,
9033 control_flags(),
9034 0,
9035 0,
9036 9,
9037 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9038 )
9039 .unwrap(),
9040 )
9041 .await
9042 .unwrap();
9043 let ClientControlResponse::ServerDescribe { machine_id, .. } =
9044 serde_json::from_slice(&described[0].body).unwrap()
9045 else {
9046 panic!("server.describe answered with another shape");
9047 };
9048 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
9049 }
9050
9051 #[test]
9052 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
9053 let registry = Arc::new(Registry::default());
9056 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
9057 crate::daemon_config::StorageConfig::Sqlite {
9058 data_home: std::path::PathBuf::from("/data"),
9059 },
9060 ));
9061
9062 let responses = handler
9063 .handle_control(
9064 ConnectionId::new(1),
9065 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
9066 )
9067 .unwrap();
9068 let ack = parse_ack(&responses[0]);
9069 assert_eq!(
9070 ack.storage,
9071 Some(serde_json::json!({
9072 "module_id": "alfonso-routing",
9073 "storage_namespace": "default",
9074 "isolation": { "kind": "module" },
9075 "backend": {
9076 "backend": "sqlite",
9077 "path": "/data/cortexkit/alfonso-routing/store.db"
9078 }
9079 })),
9080 "the delivered descriptor is the module's own sqlite store path"
9081 );
9082 }
9083
9084 #[test]
9085 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
9086 let registry = Arc::new(Registry::default());
9087 let handler = ControlHandler::new(Arc::clone(®istry));
9088 let conn = ConnectionId::new(1);
9089 let responses = handler
9090 .handle_control(
9091 conn,
9092 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9093 )
9094 .unwrap();
9095 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9096 let registration = registry.get_module("aft").unwrap().unwrap();
9097 assert_eq!(registration.control_ops, module_baseline_control_ops());
9098
9099 let frame =
9100 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
9101 assert!(handler
9102 .guard_module_control_op(&frame, "aft", "route.bind")
9103 .unwrap()
9104 .is_none());
9105 let error = handler
9106 .guard_module_control_op(&frame, "aft", "test.synthetic")
9107 .unwrap()
9108 .expect("synthetic ungranted op should be rejected");
9109 assert_eq!(error.header.ty, FrameType::Error);
9110 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
9111 }
9112
9113 #[test]
9114 fn hello_control_ops_some_adds_optional_grants() {
9115 let registry = Arc::new(Registry::default());
9116 let handler = ControlHandler::new(Arc::clone(®istry));
9117 handler
9118 .handle_control(
9119 ConnectionId::new(1),
9120 hello_frame_with_control_ops(
9121 "aft",
9122 PROTOCOL_VERSION,
9123 7,
9124 Some(vec![
9125 "future.synthetic".to_string(),
9126 "route.bind".to_string(),
9127 ]),
9128 ),
9129 )
9130 .unwrap();
9131 let registration = registry.get_module("aft").unwrap().unwrap();
9132 assert_eq!(
9133 registration.control_ops,
9134 vec![
9135 "route.bind".to_string(),
9136 "route.status".to_string(),
9137 "future.synthetic".to_string(),
9138 ]
9139 );
9140 let frame =
9141 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
9142 assert!(handler
9143 .guard_module_control_op(&frame, "aft", "future.synthetic")
9144 .unwrap()
9145 .is_none());
9146 }
9147
9148 #[tokio::test]
9149 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
9150 let registry = Arc::new(Registry::default());
9151 let forwarding = Arc::new(ForwardingTable::default());
9152 let handler =
9153 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9154 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
9155 hello_via_sink(
9156 &handler,
9157 &module_ctx,
9158 &mut module_rx,
9159 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9160 )
9161 .await;
9162
9163 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
9164 let responses = handler
9165 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
9166 .await
9167 .unwrap();
9168 assert_eq!(responses.len(), 1);
9169 assert_eq!(responses[0].header.ty, FrameType::Error);
9170 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
9171 assert!(module_rx.try_recv().is_err());
9172 }
9173
9174 #[tokio::test]
9175 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
9176 let registry = Arc::new(Registry::default());
9177 let forwarding = Arc::new(ForwardingTable::default());
9178 let handler =
9179 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9180 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
9181 hello_via_sink(
9182 &handler,
9183 &module_ctx,
9184 &mut module_rx,
9185 hello_frame_with_control_ops(
9186 "aft",
9187 PROTOCOL_VERSION,
9188 7,
9189 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9190 ),
9191 )
9192 .await;
9193
9194 let project_root = unique_project_root("demux");
9195 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
9196 let route_handler = handler.clone();
9197 let route_task = tokio::spawn(async move {
9198 route_handler
9199 .handle_control_frame(
9200 &route_client_ctx,
9201 route_open_frame(100, "aft", project_root),
9202 )
9203 .await
9204 .unwrap()
9205 });
9206 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9207 .await
9208 .unwrap()
9209 .unwrap();
9210 assert!(matches!(
9211 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
9212 ModuleControlRequest::RouteBind { .. }
9213 ));
9214
9215 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
9216 let health_handler = handler.clone();
9217 let health_task = tokio::spawn(async move {
9218 health_handler
9219 .handle_control_frame(
9220 &health_client_ctx,
9221 supervisor_health_probe_frame(101, "aft"),
9222 )
9223 .await
9224 .unwrap()
9225 });
9226 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9227 .await
9228 .unwrap()
9229 .unwrap();
9230 assert_eq!(
9231 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9232 ModuleControlRequest::HealthCheck {}
9233 );
9234
9235 handler
9236 .handle_control_frame(
9237 &module_ctx,
9238 health_response(health_frame.header.corr, HealthStatus::Degraded),
9239 )
9240 .await
9241 .unwrap();
9242 let health_response = health_task.await.unwrap();
9243 assert_eq!(health_response.len(), 1);
9244 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9245 ClientControlResponse::SupervisorHealthProbe {
9246 module_id,
9247 status,
9248 detail,
9249 metrics,
9250 } => {
9251 assert_eq!(module_id, "aft");
9252 assert_eq!(status, HealthStatus::Degraded);
9253 assert_eq!(detail.as_deref(), Some("warming"));
9254 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
9255 }
9256 other => panic!("unexpected health response: {other:?}"),
9257 }
9258
9259 handler
9260 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9261 .await
9262 .unwrap();
9263 let route_response = route_task.await.unwrap();
9264 assert!(route_response.is_empty());
9265 let published = route_client_rx.recv().await.unwrap();
9266 assert!(matches!(
9267 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9268 ClientControlResponse::RouteOpen { .. }
9269 ));
9270 }
9271
9272 async fn relay_route_open(
9277 handler: &ControlHandler,
9278 client_connection: ConnectionId,
9279 client_egress: &FrameSink,
9280 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9281 corr: u64,
9282 module_id: &str,
9283 project_root_label: &str,
9284 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
9285 let ctx = RouteCtx {
9286 connection_id: client_connection,
9287 egress: client_egress.clone(),
9288 };
9289 let handler = handler.clone();
9290 let project_root = unique_project_root(project_root_label);
9291 let module_id = module_id.to_string();
9292 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
9293 let task = tokio::spawn(async move {
9294 let _guard = tracing::dispatcher::set_default(&dispatch);
9295 handler
9296 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
9297 .await
9298 .unwrap()
9299 });
9300 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
9301 .await
9302 .expect("module receives the relayed route.bind")
9303 .expect("module egress is open");
9304 (task, bind.frame)
9305 }
9306
9307 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
9308 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
9309 ModuleControlRequest::RouteBind {
9310 route_channel,
9311 epoch,
9312 ..
9313 } => (route_channel, epoch),
9314 other => panic!("expected a route.bind request, got {other:?}"),
9315 }
9316 }
9317
9318 fn published_route(frame: &Frame) -> (u16, u32) {
9319 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
9320 ClientControlResponse::RouteOpen {
9321 route_channel,
9322 route_epoch,
9323 } => (route_channel, route_epoch),
9324 other => panic!("expected a route.open response, got {other:?}"),
9325 }
9326 }
9327
9328 #[tokio::test]
9348 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
9349 let registry = Arc::new(Registry::default());
9350 let forwarding = Arc::new(ForwardingTable::default());
9351 let handler =
9352 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9353
9354 let module_connection = ConnectionId::new(30);
9355 let (module_ctx, mut module_rx) = route_ctx(module_connection);
9356 hello_via_sink(
9357 &handler,
9358 &module_ctx,
9359 &mut module_rx,
9360 hello_frame("aft", PROTOCOL_VERSION, 7),
9361 )
9362 .await;
9363
9364 let dying_client = ConnectionId::new(31);
9365 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
9366
9367 let (first_task, first_bind) = relay_route_open(
9370 &handler,
9371 dying_client,
9372 &dying_ctx.egress,
9373 &mut module_rx,
9374 100,
9375 "aft",
9376 "closing-first",
9377 )
9378 .await;
9379 handler
9380 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
9381 .await
9382 .unwrap();
9383 assert!(first_task.await.unwrap().is_empty());
9384 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
9385
9386 let (second_task, second_bind) = relay_route_open(
9388 &handler,
9389 dying_client,
9390 &dying_ctx.egress,
9391 &mut module_rx,
9392 101,
9393 "aft",
9394 "closing-second",
9395 )
9396 .await;
9397 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
9398
9399 assert!(forwarding
9402 .escalate_client_delivery_failure(
9403 dying_client,
9404 first_channel,
9405 first_epoch,
9406 CloseReason::new(
9407 "module_to_client_delivery_failed",
9408 "client egress refused a module frame",
9409 ),
9410 crate::forwarding::UndeliveredFrame {
9411 module_id: None,
9412 sink: &dying_ctx.egress,
9413 },
9414 )
9415 .unwrap());
9416 assert!(!dying_ctx.egress.is_closed());
9417
9418 let ack = handler
9420 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
9421 .await;
9422 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
9423 if module_loop_error.is_some() {
9424 handler.cleanup_connection(module_connection).unwrap();
9428 }
9429 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9433 .await
9434 .ok()
9435 .flatten();
9436
9437 assert!(
9439 registry
9440 .get_module_by_connection(module_connection)
9441 .unwrap()
9442 .is_some(),
9443 "one client's closing connection ended the shared module connection: \
9444 {module_loop_error:?}"
9445 );
9446 let cotenant = ConnectionId::new(32);
9448 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
9449 let (cotenant_task, cotenant_bind) = relay_route_open(
9450 &handler,
9451 cotenant,
9452 &cotenant_ctx.egress,
9453 &mut module_rx,
9454 102,
9455 "aft",
9456 "closing-cotenant",
9457 )
9458 .await;
9459 handler
9460 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
9461 .await
9462 .unwrap();
9463 assert!(cotenant_task.await.unwrap().is_empty());
9464 let (cotenant_channel, cotenant_epoch) =
9465 published_route(&cotenant_rx.recv().await.unwrap());
9466 assert!(matches!(
9467 forwarding
9468 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
9469 .unwrap(),
9470 DataRoute::Client(DataRouteState::Bound(_))
9471 ));
9472
9473 let goodbye = post_ack_module_frame
9476 .expect("module receives a GOODBYE for the abandoned route channel");
9477 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9478 assert_eq!(goodbye.header.channel, abandoned_channel);
9479 assert_eq!(goodbye.header.epoch, abandoned_epoch);
9480
9481 assert!(dying_rx.try_recv().is_err());
9485 let second_response = second_task.await.unwrap();
9486 assert_eq!(second_response.len(), 1);
9487 assert_eq!(
9488 parse_error(&second_response[0])["code"],
9489 "target_unavailable"
9490 );
9491 }
9492
9493 async fn catalog_snapshot(
9496 handler: &ControlHandler,
9497 client_ctx: &RouteCtx,
9498 corr: u64,
9499 ) -> (u64, Vec<(String, bool)>) {
9500 let body =
9501 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
9502 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
9503 let reply = handler
9504 .handle_control_frame(client_ctx, request)
9505 .await
9506 .unwrap()
9507 .pop()
9508 .unwrap();
9509 let ClientControlResponse::CatalogList {
9510 generation,
9511 modules,
9512 ..
9513 } = serde_json::from_slice(&reply.body).unwrap()
9514 else {
9515 panic!("not a catalog.list reply");
9516 };
9517 (
9518 generation,
9519 modules
9520 .into_iter()
9521 .map(|entry| (entry.module_id, entry.ready))
9522 .collect(),
9523 )
9524 }
9525
9526 fn hello_held_before_forwarding() -> (
9532 ControlHandler,
9533 Arc<std::sync::Barrier>,
9534 std::thread::JoinHandle<Vec<Frame>>,
9535 RouteCtx,
9536 mpsc::Receiver<crate::router::OutboundFrame>,
9537 ) {
9538 let hold = Arc::new(std::sync::Barrier::new(2));
9539 let handler = ControlHandler::with_forwarding(
9540 Arc::new(Registry::default()),
9541 Arc::new(ForwardingTable::default()),
9542 )
9543 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
9544 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9545 let hello = {
9546 let handler = handler.clone();
9547 let module_ctx = module_ctx.clone();
9548 std::thread::spawn(move || {
9549 handler
9550 .handle_hello(
9551 module_ctx.connection_id,
9552 Some(module_ctx.egress.clone()),
9553 hello_frame("aft", PROTOCOL_VERSION, 7),
9554 )
9555 .unwrap()
9556 })
9557 };
9558 (handler, hold, hello, module_ctx, module_rx)
9559 }
9560
9561 #[tokio::test]
9570 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
9571 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
9572 let client = ConnectionId::new(41);
9573 let (client_ctx, mut client_rx) = route_ctx(client);
9574
9575 hold.wait();
9576 assert!(handler.registry.get_module("aft").unwrap().is_some());
9578 assert!(!handler
9579 .forwarding
9580 .has_live_module_connection("aft")
9581 .unwrap());
9582 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
9583 assert!(
9584 listed.is_empty(),
9585 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
9586 );
9587 let refused = handler
9588 .handle_control_frame(
9589 &client_ctx,
9590 route_open_frame(301, "aft", unique_project_root("listed-window")),
9591 )
9592 .await
9593 .unwrap();
9594 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
9595 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
9596 let duplicate = handler
9597 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
9598 .await
9599 .unwrap();
9600 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9601
9602 hold.wait();
9603 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9604 let ack = module_rx.try_recv().unwrap().frame;
9605 assert_eq!(ack.header.ty, FrameType::HelloAck);
9606 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9607 assert_eq!(listed, vec![("aft".to_string(), true)]);
9608
9609 let (open_task, bind) = relay_route_open(
9610 &handler,
9611 client,
9612 &client_ctx.egress,
9613 &mut module_rx,
9614 303,
9615 "aft",
9616 "listed-after",
9617 )
9618 .await;
9619 assert!(handler
9620 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9621 .await
9622 .unwrap()
9623 .is_empty());
9624 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
9625 published_route(&client_rx.try_recv().unwrap().frame);
9626 }
9627
9628 #[tokio::test]
9635 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
9636 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
9637 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9638 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9639
9640 hold.wait();
9641 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
9642 assert!(listed.is_empty());
9643 assert_eq!(
9644 during, before,
9645 "the generation moved while the module was not listed"
9646 );
9647
9648 hold.wait();
9649 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9650 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9651 assert_eq!(listed, vec![("aft".to_string(), true)]);
9652 assert_eq!(
9653 after,
9654 before + 1,
9655 "the module appeared without exactly one generation bump"
9656 );
9657 }
9658
9659 #[tokio::test]
9664 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
9665 let registry = Arc::new(Registry::default());
9666 let handler = ControlHandler::with_forwarding(
9667 Arc::clone(®istry),
9668 Arc::new(ForwardingTable::default()),
9669 );
9670 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9671 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9672 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9673 drop(module_rx);
9674
9675 let replies = handler
9676 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
9677 .await
9678 .unwrap();
9679 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
9680 assert!(registry.get_module("aft").unwrap().is_none());
9681 assert!(registry
9682 .get_module_by_connection(module_ctx.connection_id)
9683 .unwrap()
9684 .is_none());
9685 assert_eq!(
9686 catalog_snapshot(&handler, &client_ctx, 301).await,
9687 (before, Vec::new())
9688 );
9689 }
9690
9691 async fn route_open_held_after_bind_send() -> (
9695 ControlHandler,
9696 Arc<tokio::sync::Barrier>,
9697 RouteCtx,
9698 mpsc::Receiver<crate::router::OutboundFrame>,
9699 ) {
9700 let hold = Arc::new(tokio::sync::Barrier::new(2));
9701 let handler = ControlHandler::with_forwarding(
9702 Arc::new(Registry::default()),
9703 Arc::new(ForwardingTable::default()),
9704 )
9705 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
9706 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
9707 hello_via_sink(
9708 &handler,
9709 &module_ctx,
9710 &mut module_rx,
9711 hello_frame("aft", PROTOCOL_VERSION, 7),
9712 )
9713 .await;
9714 (handler, hold, module_ctx, module_rx)
9715 }
9716
9717 #[tokio::test]
9724 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
9725 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9726 let client = ConnectionId::new(41);
9727 let (client_ctx, mut client_rx) = route_ctx(client);
9728
9729 let (open_task, bind) = relay_route_open(
9730 &handler,
9731 client,
9732 &client_ctx.egress,
9733 &mut module_rx,
9734 200,
9735 "aft",
9736 "acked-before-mark",
9737 )
9738 .await;
9739 let (module_channel, module_epoch) = route_bind_channel(&bind);
9740 hold.wait().await;
9742
9743 assert!(handler
9745 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9746 .await
9747 .unwrap()
9748 .is_empty());
9749 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
9750
9751 hold.wait().await;
9752 assert!(
9753 open_task.await.unwrap().is_empty(),
9754 "route.open answers accepted"
9755 );
9756
9757 match module_rx.try_recv() {
9760 Err(mpsc::error::TryRecvError::Empty) => {}
9761 Ok(outbound) => panic!(
9762 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
9763 outbound.frame.header.ty,
9764 outbound.frame.header.channel,
9765 outbound.frame.header.epoch
9766 ),
9767 Err(err) => panic!("module egress closed: {err}"),
9768 }
9769
9770 let router = Router::with_control_handler(Arc::new(handler.clone()));
9773 let request = Frame::build(
9774 FrameType::Request,
9775 Flags::new(true, Priority::Interactive, false),
9776 client_channel,
9777 client_epoch,
9778 201,
9779 b"first request".to_vec(),
9780 )
9781 .unwrap();
9782 router
9783 .route_for_connection(&client_ctx, request)
9784 .await
9785 .unwrap();
9786 let delivered = module_rx.try_recv().unwrap().frame;
9787 assert_eq!(delivered.header.ty, FrameType::Request);
9788 assert_eq!(
9789 (delivered.header.channel, delivered.header.epoch),
9790 (module_channel, module_epoch)
9791 );
9792 assert_eq!(delivered.body, b"first request");
9793 }
9794
9795 #[tokio::test]
9801 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
9802 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9803 let client = ConnectionId::new(42);
9804 let (client_ctx, _client_rx) = route_ctx(client);
9805
9806 let (open_task, bind) = relay_route_open(
9807 &handler,
9808 client,
9809 &client_ctx.egress,
9810 &mut module_rx,
9811 300,
9812 "aft",
9813 "released-before-mark",
9814 )
9815 .await;
9816 let (module_channel, module_epoch) = route_bind_channel(&bind);
9817 hold.wait().await;
9818
9819 handler.cleanup_connection(client).unwrap();
9820 assert!(
9821 module_rx.try_recv().is_err(),
9822 "client teardown must not GOODBYE a bind route.open is still queueing"
9823 );
9824
9825 hold.wait().await;
9826 let response = open_task.await.unwrap();
9827 assert_eq!(response.len(), 1);
9828 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9829
9830 let goodbye = module_rx
9831 .try_recv()
9832 .expect("module is told to drop the binding the daemon abandoned")
9833 .frame;
9834 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9835 assert_eq!(
9836 (goodbye.header.channel, goodbye.header.epoch),
9837 (module_channel, module_epoch)
9838 );
9839 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9840 }
9841
9842 #[tokio::test]
9847 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
9848 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9849 let client = ConnectionId::new(43);
9850 let (client_ctx, client_rx) = route_ctx(client);
9851
9852 let (open_task, bind) = relay_route_open(
9853 &handler,
9854 client,
9855 &client_ctx.egress,
9856 &mut module_rx,
9857 400,
9858 "aft",
9859 "gone-client-before-mark",
9860 )
9861 .await;
9862 let (module_channel, module_epoch) = route_bind_channel(&bind);
9863 hold.wait().await;
9864
9865 drop(client_rx);
9866 assert!(client_ctx.egress.is_closed());
9867 handler
9868 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9869 .await
9870 .unwrap();
9871
9872 hold.wait().await;
9873 let response = open_task.await.unwrap();
9874 assert_eq!(response.len(), 1);
9875 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9876
9877 let goodbye = module_rx
9878 .try_recv()
9879 .expect("module is told to drop the binding it acked")
9880 .frame;
9881 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9882 assert_eq!(
9883 (goodbye.header.channel, goodbye.header.epoch),
9884 (module_channel, module_epoch)
9885 );
9886 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9887 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
9888 }
9889
9890 #[test]
9897 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
9898 let handler = ControlHandler::default();
9899 let module_connection = ConnectionId::new(30);
9900 let client_connection = ConnectionId::new(31);
9901
9902 handler
9903 .refuse_to_end_module_connection_for_a_client(
9904 module_connection,
9905 77,
9906 ForwardingError::ConnectionClosing {
9907 connection_id: client_connection,
9908 },
9909 )
9910 .expect("a closing client must never end the module connection");
9911
9912 assert!(matches!(
9913 handler.refuse_to_end_module_connection_for_a_client(
9914 module_connection,
9915 78,
9916 ForwardingError::ConnectionClosing {
9917 connection_id: module_connection,
9918 },
9919 ),
9920 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
9921 connection_id
9922 })) if connection_id == module_connection
9923 ));
9924 assert!(matches!(
9925 handler.refuse_to_end_module_connection_for_a_client(
9926 module_connection,
9927 79,
9928 ForwardingError::Poisoned,
9929 ),
9930 Err(RouterError::Forwarding(ForwardingError::Poisoned))
9931 ));
9932 assert!(matches!(
9933 handler.refuse_to_end_module_connection_for_a_client(
9934 module_connection,
9935 80,
9936 ForwardingError::StaleModuleEndpoint,
9937 ),
9938 Err(RouterError::Forwarding(
9939 ForwardingError::StaleModuleEndpoint
9940 ))
9941 ));
9942 }
9943
9944 #[tokio::test]
9957 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
9958 let registry = Arc::new(Registry::default());
9959 let forwarding = Arc::new(ForwardingTable::default());
9960 let supervisor = SupervisorHandle::new();
9961 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9962 let handler =
9963 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9964 .with_supervisor(supervisor);
9965
9966 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9967 hello_via_sink(
9968 &handler,
9969 &target_ctx,
9970 &mut target_rx,
9971 hello_frame("target", PROTOCOL_VERSION, 1),
9972 )
9973 .await;
9974
9975 let wrong_nonce = handler
9979 .handle_control_frame(
9980 &route_ctx(ConnectionId::new(91)).0,
9981 route_open_frame_with_admission_facts(
9982 20,
9983 "target",
9984 unique_project_root("admission-facts"),
9985 Some(subc_control::ConsumerIdentity {
9986 module_id: "fed".to_string(),
9987 launch_nonce: "not-the-real-nonce".to_string(),
9988 }),
9989 None,
9990 ),
9991 )
9992 .await
9993 .unwrap();
9994 assert_eq!(
9995 parse_error(&wrong_nonce[0])["code"],
9996 "bad_consumer_identity",
9997 "a mismatched launch nonce must be refused, not stamped Reserved"
9998 );
9999
10000 let never_spawned = handler
10004 .handle_control_frame(
10005 &route_ctx(ConnectionId::new(92)).0,
10006 route_open_frame_with_admission_facts(
10007 21,
10008 "target",
10009 unique_project_root("admission-facts"),
10010 Some(subc_control::ConsumerIdentity {
10011 module_id: "never-spawned".to_string(),
10012 launch_nonce: "any-nonce".to_string(),
10013 }),
10014 None,
10015 ),
10016 )
10017 .await
10018 .unwrap();
10019 assert_eq!(
10020 parse_error(&never_spawned[0])["code"],
10021 "bad_consumer_identity",
10022 "an unspawned module_id must be refused rather than accepted for lack of a record"
10023 );
10024 }
10025
10026 #[tokio::test]
10049 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
10050 let registry = Arc::new(Registry::default());
10051 let forwarding = Arc::new(ForwardingTable::default());
10052 let supervisor = SupervisorHandle::new();
10053 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10054 let handler =
10055 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10056 .with_supervisor(supervisor);
10057
10058 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
10059 hello_via_sink(
10060 &handler,
10061 &target_ctx,
10062 &mut target_rx,
10063 hello_frame("target", PROTOCOL_VERSION, 1),
10064 )
10065 .await;
10066
10067 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
10068 let route_handler = handler.clone();
10069 let route_task = tokio::spawn(async move {
10070 route_handler
10071 .handle_control_frame(
10072 &client_ctx,
10073 route_open_frame_with_admission_facts(
10074 30,
10075 "target",
10076 unique_project_root("admission-facts"),
10077 Some(subc_control::ConsumerIdentity {
10078 module_id: "fed".to_string(),
10079 launch_nonce: "fed-nonce".to_string(),
10080 }),
10081 None,
10082 ),
10083 )
10084 .await
10085 .unwrap()
10086 });
10087
10088 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10095 .await
10096 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
10097 .expect("module control channel closed before route.bind");
10098 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10099 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10100 panic!("expected route.bind")
10101 };
10102 assert_eq!(
10103 principal,
10104 Some(Principal::Reserved {
10105 module_id: "fed".to_string()
10106 }),
10107 "a correctly attested consumer must be stamped Reserved for its own id"
10108 );
10109
10110 handler
10111 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10112 .await
10113 .unwrap();
10114 assert!(route_task.await.unwrap().is_empty());
10115 assert!(
10116 matches!(
10117 serde_json::from_slice::<ClientControlResponse>(
10118 &client_rx.recv().await.unwrap().body
10119 )
10120 .unwrap(),
10121 ClientControlResponse::RouteOpen { .. }
10122 ),
10123 "the route must actually open, not merely avoid an error"
10124 );
10125 }
10126
10127 #[tokio::test(start_paused = true)]
10128 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
10129 let registry = Arc::new(Registry::default());
10130 let forwarding = Arc::new(ForwardingTable::default());
10131 let supervisor = SupervisorHandle::new();
10132 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10133 let handler =
10134 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10135 .with_supervisor(supervisor);
10136
10137 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
10138 hello_via_sink(
10139 &handler,
10140 &target_ctx,
10141 &mut target_rx,
10142 hello_frame("target", PROTOCOL_VERSION, 1),
10143 )
10144 .await;
10145
10146 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
10147 let direct_handler = handler.clone();
10148 let direct_open = tokio::spawn(async move {
10149 direct_handler
10150 .handle_control_frame(
10151 &direct_ctx,
10152 route_open_frame(2, "target", unique_project_root("route-census-direct")),
10153 )
10154 .await
10155 .unwrap()
10156 });
10157 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10158 .await
10159 .expect("no direct route.bind within 5s")
10160 .expect("target control channel closed before direct route.bind");
10161 handler
10162 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
10163 .await
10164 .unwrap();
10165 assert!(direct_open.await.unwrap().is_empty());
10166 let _ = direct_rx.recv().await.unwrap();
10167
10168 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
10169 let reserved_handler = handler.clone();
10170 let reserved_open = tokio::spawn(async move {
10171 reserved_handler
10172 .handle_control_frame(
10173 &reserved_ctx,
10174 route_open_frame_with_admission_facts(
10175 3,
10176 "target",
10177 unique_project_root("admission-facts"),
10178 Some(ConsumerIdentity {
10179 module_id: "fed".to_string(),
10180 launch_nonce: "fed-nonce".to_string(),
10181 }),
10182 None,
10183 ),
10184 )
10185 .await
10186 .unwrap()
10187 });
10188 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10189 .await
10190 .expect("no reserved route.bind within 5s")
10191 .expect("target control channel closed before reserved route.bind");
10192 handler
10193 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
10194 .await
10195 .unwrap();
10196 assert!(reserved_open.await.unwrap().is_empty());
10197 let _ = reserved_rx.recv().await.unwrap();
10198
10199 forwarding
10200 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
10201 .unwrap();
10202 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
10203 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
10204 module_id: Some("target".to_string()),
10205 })
10206 .unwrap();
10207 let census_frame =
10208 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
10209 let response = handler
10210 .handle_control_frame(&census_ctx, census_frame)
10211 .await
10212 .unwrap()
10213 .pop()
10214 .unwrap();
10215 let actual: Value = serde_json::from_slice(&response.body).unwrap();
10216 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
10217 assert!(matches!(
10218 decoded,
10219 ClientControlResponse::SupervisorRoutes { .. }
10220 ));
10221 let routes = actual["modules"][0]["routes"].as_array().unwrap();
10222 assert_eq!(routes.len(), 2);
10223 assert!(routes.iter().all(|route| route["draining"] == true));
10224 assert!(
10227 routes.iter().all(|route| route["drain_reason"] == "reload"),
10228 "draining routes must name the drain's reason: {routes:?}"
10229 );
10230 assert!(routes.iter().any(|route| {
10231 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
10232 }));
10233 assert!(routes.iter().any(|route| {
10234 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
10235 }));
10236
10237 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10238 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
10239 if std::env::var_os("UPDATE_GOLDEN").is_some() {
10240 std::fs::write(
10241 &golden_path,
10242 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
10243 )
10244 .unwrap();
10245 }
10246 let expected: Value =
10247 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
10248 assert_eq!(actual, expected);
10249 }
10250
10251 async fn query_live_roots(
10252 handler: &ControlHandler,
10253 module_ctx: &RouteCtx,
10254 ) -> ModuleControlResponseToModule {
10255 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
10256 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
10257 let response = handler
10258 .handle_control_frame(module_ctx, frame)
10259 .await
10260 .unwrap()
10261 .pop()
10262 .unwrap();
10263 serde_json::from_slice(&response.body).unwrap()
10264 }
10265
10266 #[tokio::test(start_paused = true)]
10267 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
10268 let registry = Arc::new(Registry::default());
10269 let forwarding = Arc::new(ForwardingTable::default());
10270 let handler = ControlHandler::with_forwarding(registry, forwarding);
10271 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
10272 hello_via_sink(
10273 &handler,
10274 &target_ctx,
10275 &mut target_rx,
10276 hello_frame("target", PROTOCOL_VERSION, 1),
10277 )
10278 .await;
10279 let root = unique_project_root("live-roots-known");
10280 let path = ProjectRootId::from_path_allowing_missing(root.path())
10281 .unwrap()
10282 .as_path()
10283 .to_path_buf();
10284 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
10285 let open_handler = handler.clone();
10286 let opened = tokio::spawn(async move {
10287 open_handler
10288 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
10289 .await
10290 .unwrap()
10291 });
10292 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10293 .await
10294 .unwrap()
10295 .unwrap();
10296 handler
10297 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
10298 .await
10299 .unwrap();
10300 assert!(opened.await.unwrap().is_empty());
10301 let _ = client_rx.recv().await.unwrap();
10302
10303 let root = unique_project_root("live-roots-pending");
10304 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
10305 .unwrap()
10306 .as_path()
10307 .to_path_buf();
10308 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
10309 let open_handler = handler.clone();
10310 let pending = tokio::spawn(async move {
10311 open_handler
10312 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
10313 .await
10314 .unwrap()
10315 });
10316 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10317 .await
10318 .unwrap()
10319 .unwrap();
10320 let actual = query_live_roots(&handler, &target_ctx).await;
10321 let ModuleControlResponseToModule::LiveRoots {
10322 roots,
10323 unknown_root_bindings,
10324 total_bindings,
10325 } = actual
10326 else {
10327 panic!("expected live roots")
10328 };
10329 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
10330 assert_eq!(unknown_root_bindings, 0);
10331 assert_eq!(
10332 roots.len(),
10333 2,
10334 "root-known arm must retain each canonical root"
10335 );
10336 assert_eq!(
10337 total_bindings,
10338 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10339 );
10340 let counts = roots
10341 .iter()
10342 .map(|root| (root.project_root.clone(), root.bound, root.pending))
10343 .collect::<Vec<_>>();
10344 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
10345 expected.sort_by(|a, b| a.0.cmp(&b.0));
10346 assert_eq!(
10347 counts, expected,
10348 "roots must sort by path and count pending separately"
10349 );
10350 handler
10351 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
10352 .await
10353 .unwrap();
10354 assert!(pending.await.unwrap().is_empty());
10355 }
10356
10357 #[tokio::test(start_paused = true)]
10358 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
10359 let forwarding = Arc::new(ForwardingTable::default());
10360 let handler =
10361 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10362 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
10363 hello_via_sink(
10364 &handler,
10365 &target_ctx,
10366 &mut target_rx,
10367 hello_frame("target", PROTOCOL_VERSION, 1),
10368 )
10369 .await;
10370 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
10371 let pending = forwarding
10372 .begin_route_bind_relay_for_test(
10373 client_ctx.connection_id,
10374 client_ctx.egress.clone(),
10375 2,
10376 "target",
10377 )
10378 .unwrap();
10379 forwarding
10380 .complete_pending_relay(
10381 target_ctx.connection_id,
10382 pending.corr,
10383 RouteBindRelayOutcome::Accepted,
10384 )
10385 .unwrap();
10386 let actual = query_live_roots(&handler, &target_ctx).await;
10387 let ModuleControlResponseToModule::LiveRoots {
10388 roots,
10389 unknown_root_bindings,
10390 total_bindings,
10391 } = actual
10392 else {
10393 panic!("expected live roots")
10394 };
10395 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
10396 assert_eq!(
10397 unknown_root_bindings, 1,
10398 "unknown-root arm must not read as no bindings"
10399 );
10400 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
10401 assert_eq!(
10402 total_bindings,
10403 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10404 );
10405 }
10406
10407 #[tokio::test(start_paused = true)]
10415 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
10416 let forwarding = Arc::new(ForwardingTable::default());
10417 let handler =
10418 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10419 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
10420 let replies = handler
10421 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
10422 .await
10423 .unwrap();
10424 let queued_by_hello = module_rx.len();
10425
10426 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
10427 let open_handler = handler.clone();
10428 let open = tokio::spawn(async move {
10429 open_handler
10430 .handle_control_frame(
10431 &client_ctx,
10432 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
10433 )
10434 .await
10435 .unwrap()
10436 });
10437 let mut spins = 0;
10439 while module_rx.len() == queued_by_hello {
10440 spins += 1;
10441 assert!(spins < 10_000, "route.open never queued a route.bind");
10442 tokio::task::yield_now().await;
10443 }
10444
10445 for reply in replies {
10447 module_ctx.egress.send(reply).await.unwrap();
10448 }
10449
10450 let first = module_rx.recv().await.unwrap().frame;
10451 assert_eq!(
10452 first.header.ty,
10453 FrameType::HelloAck,
10454 "the first frame a registering module reads must be its HELLO_ACK"
10455 );
10456 assert_eq!(first.header.corr, 7);
10457 let second = module_rx.recv().await.unwrap().frame;
10458 assert_eq!(second.header.ty, FrameType::Request);
10459 assert!(
10460 matches!(
10461 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
10462 ModuleControlRequest::RouteBind { .. }
10463 ),
10464 "the route.bind follows the ack"
10465 );
10466 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
10467
10468 handler
10469 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
10470 .await
10471 .unwrap();
10472 assert!(open.await.unwrap().is_empty());
10473 let _ = client_rx.recv().await.unwrap();
10474 }
10475
10476 #[tokio::test(start_paused = true)]
10477 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
10478 let handler = ControlHandler::with_forwarding(
10479 Arc::new(Registry::default()),
10480 Arc::new(ForwardingTable::default()),
10481 );
10482 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
10483 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
10484 hello_via_sink(
10485 &handler,
10486 &first_ctx,
10487 &mut first_rx,
10488 hello_frame("first", PROTOCOL_VERSION, 1),
10489 )
10490 .await;
10491 hello_via_sink(
10492 &handler,
10493 &second_ctx,
10494 &mut second_rx,
10495 hello_frame("second", PROTOCOL_VERSION, 2),
10496 )
10497 .await;
10498 let root = unique_project_root("second-only");
10499 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
10500 let cloned = handler.clone();
10501 let open = tokio::spawn(async move {
10502 cloned
10503 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
10504 .await
10505 .unwrap()
10506 });
10507 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
10508 .await
10509 .unwrap()
10510 .unwrap();
10511 let first = query_live_roots(&handler, &first_ctx).await;
10512 let second = query_live_roots(&handler, &second_ctx).await;
10513 assert!(
10514 matches!(
10515 first,
10516 ModuleControlResponseToModule::LiveRoots {
10517 total_bindings: 0,
10518 ..
10519 }
10520 ),
10521 "cross-module scope must not expose another module's roots"
10522 );
10523 assert!(
10524 matches!(
10525 second,
10526 ModuleControlResponseToModule::LiveRoots {
10527 total_bindings: 1,
10528 ..
10529 }
10530 ),
10531 "second module must see its pending route"
10532 );
10533 handler
10534 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
10535 .await
10536 .unwrap();
10537 assert!(open.await.unwrap().is_empty());
10538 }
10539
10540 #[tokio::test(start_paused = true)]
10541 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
10542 let handler = ControlHandler::with_forwarding(
10543 Arc::new(Registry::default()),
10544 Arc::new(ForwardingTable::default()),
10545 );
10546 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
10547 hello_via_sink(
10548 &handler,
10549 &target_ctx,
10550 &mut target_rx,
10551 hello_frame("target", PROTOCOL_VERSION, 1),
10552 )
10553 .await;
10554 let actual = query_live_roots(&handler, &target_ctx).await;
10555 let ModuleControlResponseToModule::LiveRoots {
10556 roots,
10557 unknown_root_bindings,
10558 total_bindings,
10559 } = actual
10560 else {
10561 panic!("expected live roots")
10562 };
10563 assert!(roots.is_empty());
10564 assert_eq!(unknown_root_bindings, 0);
10565 assert_eq!(total_bindings, 0);
10566 assert_eq!(
10567 total_bindings,
10568 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10569 );
10570 }
10571
10572 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
10579 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10580 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
10581 let text = std::fs::read_to_string(&path)
10582 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
10583 let vectors: Vec<(String, Value)> = text
10584 .lines()
10585 .filter(|line| !line.trim().is_empty())
10586 .map(|line| {
10587 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
10588 let id = entry["corpus_id"]
10589 .as_str()
10590 .expect("every vector carries a corpus_id")
10591 .to_string();
10592 (id, entry["package"].clone())
10593 })
10594 .collect();
10595 assert_eq!(
10599 vectors.len(),
10600 3,
10601 "vendored fed corpus changed size; re-sync from subc-federation"
10602 );
10603
10604 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
10619 let richest = vectors
10620 .iter()
10621 .filter_map(|(_, package)| package.as_object())
10622 .map(|object| {
10623 object
10624 .keys()
10625 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
10626 .count()
10627 })
10628 .max()
10629 .unwrap_or(0);
10630 assert!(
10631 richest >= 2,
10632 "vendored corpus no longer carries a package with unmodelled fields, \
10633 so the relay test can no longer distinguish a verbatim relay from a lossy one"
10634 );
10635
10636 vectors
10637 }
10638
10639 #[tokio::test]
10647 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
10648 for (corpus_id, package) in fed_admission_facts_vectors() {
10649 let registry = Arc::new(Registry::default());
10650 let forwarding = Arc::new(ForwardingTable::default());
10651 let supervisor = SupervisorHandle::new();
10652 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10653 let handler =
10654 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10655 .with_supervisor(supervisor)
10656 .with_admission_facts_config(
10657 Some("fed".to_string()),
10658 Some(vec!["target".to_string()]),
10659 );
10660
10661 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10662 hello_via_sink(
10663 &handler,
10664 &target_ctx,
10665 &mut target_rx,
10666 hello_frame("target", PROTOCOL_VERSION, 1),
10667 )
10668 .await;
10669
10670 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
10671 let route_handler = handler.clone();
10672 let expected = package.clone();
10673 let route_task = tokio::spawn(async move {
10674 route_handler
10675 .handle_control_frame(
10676 &client_ctx,
10677 route_open_frame_with_admission_facts(
10678 20,
10679 "target",
10680 unique_project_root("admission-facts"),
10681 Some(subc_control::ConsumerIdentity {
10682 module_id: "fed".to_string(),
10683 launch_nonce: "fed-nonce".to_string(),
10684 }),
10685 Some(package),
10686 ),
10687 )
10688 .await
10689 .unwrap()
10690 });
10691
10692 let bind_frame = target_rx.recv().await.unwrap();
10693 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10694 let ModuleControlRequest::RouteBind {
10695 admission_facts, ..
10696 } = bind
10697 else {
10698 panic!("{corpus_id}: expected route.bind")
10699 };
10700 assert_eq!(
10701 admission_facts,
10702 Some(expected),
10703 "{corpus_id}: relay must not add, drop or reshape any field"
10704 );
10705
10706 handler
10707 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10708 .await
10709 .unwrap();
10710 route_task.await.unwrap();
10711 }
10712 }
10713
10714 #[tokio::test]
10715 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
10716 let registry = Arc::new(Registry::default());
10717 let forwarding = Arc::new(ForwardingTable::default());
10718 let supervisor = SupervisorHandle::new();
10719 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10720 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
10721 let handler =
10722 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10723 .with_supervisor(supervisor)
10724 .with_admission_facts_config(
10725 Some("fed".to_string()),
10726 Some(vec!["target".to_string()]),
10727 );
10728
10729 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
10730 hello_via_sink(
10731 &handler,
10732 &target_ctx,
10733 &mut target_rx,
10734 hello_frame("target", PROTOCOL_VERSION, 1),
10735 )
10736 .await;
10737 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
10738 hello_via_sink(
10739 &handler,
10740 &other_ctx,
10741 &mut other_rx,
10742 hello_frame("other", PROTOCOL_VERSION, 2),
10743 )
10744 .await;
10745
10746 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
10747 let expected_facts = facts.clone();
10748 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
10749 let route_handler = handler.clone();
10750 let route_task = tokio::spawn(async move {
10751 route_handler
10752 .handle_control_frame(
10753 &client_ctx,
10754 route_open_frame_with_admission_facts(
10755 10,
10756 "target",
10757 unique_project_root("admission-facts"),
10758 Some(subc_control::ConsumerIdentity {
10759 module_id: "fed".to_string(),
10760 launch_nonce: "fed-nonce".to_string(),
10761 }),
10762 Some(facts.clone()),
10763 ),
10764 )
10765 .await
10766 .unwrap()
10767 });
10768 let bind_frame = target_rx.recv().await.unwrap();
10769 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10770 let ModuleControlRequest::RouteBind {
10771 admission_facts, ..
10772 } = bind
10773 else {
10774 panic!("expected route.bind")
10775 };
10776 assert_eq!(admission_facts, Some(expected_facts));
10777 handler
10778 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10779 .await
10780 .unwrap();
10781 assert!(route_task.await.unwrap().is_empty());
10782 assert!(matches!(
10783 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
10784 .unwrap(),
10785 ClientControlResponse::RouteOpen { .. }
10786 ));
10787
10788 let direct = handler
10789 .handle_control_frame(
10790 &route_ctx(ConnectionId::new(73)).0,
10791 route_open_frame_with_admission_facts(
10792 11,
10793 "target",
10794 unique_project_root("admission-facts"),
10795 None,
10796 Some(json!({"x": 1})),
10797 ),
10798 )
10799 .await
10800 .unwrap();
10801 assert_eq!(
10802 parse_error(&direct[0])["code"],
10803 "admission_facts_not_permitted"
10804 );
10805
10806 let different_reserved = handler
10807 .handle_control_frame(
10808 &route_ctx(ConnectionId::new(77)).0,
10809 route_open_frame_with_admission_facts(
10810 15,
10811 "target",
10812 unique_project_root("admission-facts"),
10813 Some(subc_control::ConsumerIdentity {
10814 module_id: "other".to_string(),
10815 launch_nonce: "other-nonce".to_string(),
10816 }),
10817 Some(json!({"x": 1})),
10818 ),
10819 )
10820 .await
10821 .unwrap();
10822 assert_eq!(
10823 parse_error(&different_reserved[0])["code"],
10824 "admission_facts_not_permitted"
10825 );
10826
10827 let other_target = handler
10828 .handle_control_frame(
10829 &route_ctx(ConnectionId::new(74)).0,
10830 route_open_frame_with_admission_facts(
10831 12,
10832 "other",
10833 unique_project_root("admission-facts"),
10834 Some(subc_control::ConsumerIdentity {
10835 module_id: "fed".to_string(),
10836 launch_nonce: "fed-nonce".to_string(),
10837 }),
10838 Some(json!({"x": 1})),
10839 ),
10840 )
10841 .await
10842 .unwrap();
10843 assert_eq!(
10844 parse_error(&other_target[0])["code"],
10845 "admission_facts_target_not_allowed"
10846 );
10847
10848 let nonexistent = handler
10849 .handle_control_frame(
10850 &route_ctx(ConnectionId::new(75)).0,
10851 route_open_frame_with_admission_facts(
10852 13,
10853 "missing",
10854 unique_project_root("admission-facts"),
10855 None,
10856 Some(json!({"x": 1})),
10857 ),
10858 )
10859 .await
10860 .unwrap();
10861 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
10862
10863 let described = handler
10864 .handle_control_frame(
10865 &route_ctx(ConnectionId::new(76)).0,
10866 Frame::build(
10867 FrameType::Request,
10868 control_flags(),
10869 0,
10870 0,
10871 14,
10872 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10873 )
10874 .unwrap(),
10875 )
10876 .await
10877 .unwrap();
10878 let ClientControlResponse::ServerDescribe { capabilities, .. } =
10879 serde_json::from_slice(&described[0].body).unwrap()
10880 else {
10881 panic!("expected server.describe response")
10882 };
10883 assert!(capabilities
10884 .iter()
10885 .any(|cap| cap == "admission_facts_relay_v1"));
10886 }
10887
10888 #[tokio::test]
10889 async fn admission_facts_without_configured_carrier_are_rejected() {
10890 let registry = Arc::new(Registry::default());
10891 let forwarding = Arc::new(ForwardingTable::default());
10892 let handler = ControlHandler::with_forwarding(registry, forwarding);
10893 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
10894 hello_via_sink(
10895 &handler,
10896 &target_ctx,
10897 &mut target_rx,
10898 hello_frame("target", PROTOCOL_VERSION, 1),
10899 )
10900 .await;
10901
10902 let responses = handler
10903 .handle_control_frame(
10904 &route_ctx(ConnectionId::new(79)).0,
10905 route_open_frame_with_admission_facts(
10906 16,
10907 "target",
10908 unique_project_root("admission-facts"),
10909 None,
10910 Some(json!({"x": 1})),
10911 ),
10912 )
10913 .await
10914 .unwrap();
10915 assert_eq!(
10916 parse_error(&responses[0])["code"],
10917 "admission_facts_not_permitted"
10918 );
10919 }
10920
10921 #[tokio::test]
10922 async fn route_open_relays_consumer_capabilities_verbatim() {
10923 let registry = Arc::new(Registry::default());
10924 let forwarding = Arc::new(ForwardingTable::default());
10925 let handler =
10926 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10927 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
10928 hello_via_sink(
10929 &handler,
10930 &module_ctx,
10931 &mut module_rx,
10932 hello_frame("aft", PROTOCOL_VERSION, 7),
10933 )
10934 .await;
10935
10936 let expected = vec!["elicitation".to_string(), "roots".to_string()];
10937 let expected_for_request = expected.clone();
10938 let project_root = unique_project_root("consumer-capabilities-present");
10939 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
10940 let route_handler = handler.clone();
10941 let route_task = tokio::spawn(async move {
10942 route_handler
10943 .handle_control_frame(
10944 &client_ctx,
10945 route_open_frame_with_consumer_capabilities(
10946 401,
10947 "aft",
10948 project_root,
10949 Some(expected_for_request),
10950 ),
10951 )
10952 .await
10953 .unwrap()
10954 });
10955 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10956 .await
10957 .unwrap()
10958 .unwrap();
10959 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10960 let ModuleControlRequest::RouteBind {
10961 consumer_capabilities,
10962 ..
10963 } = bind
10964 else {
10965 panic!("expected route.bind request, got {bind:?}");
10966 };
10967 assert_eq!(consumer_capabilities, Some(expected.clone()));
10968
10969 handler
10970 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10971 .await
10972 .unwrap();
10973 let route_response = route_task.await.unwrap();
10974 assert!(route_response.is_empty());
10975 let published = client_rx.recv().await.unwrap();
10976 assert!(matches!(
10977 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10978 ClientControlResponse::RouteOpen { .. }
10979 ));
10980 }
10981
10982 #[tokio::test]
10983 async fn route_open_without_consumer_capabilities_relays_none() {
10984 let registry = Arc::new(Registry::default());
10985 let forwarding = Arc::new(ForwardingTable::default());
10986 let handler =
10987 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10988 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
10989 hello_via_sink(
10990 &handler,
10991 &module_ctx,
10992 &mut module_rx,
10993 hello_frame("aft", PROTOCOL_VERSION, 7),
10994 )
10995 .await;
10996
10997 let project_root = unique_project_root("consumer-capabilities-absent");
10998 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
10999 let route_handler = handler.clone();
11000 let route_task = tokio::spawn(async move {
11001 route_handler
11002 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
11003 .await
11004 .unwrap()
11005 });
11006 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11007 .await
11008 .unwrap()
11009 .unwrap();
11010 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11011 let ModuleControlRequest::RouteBind {
11012 consumer_capabilities,
11013 ..
11014 } = bind
11015 else {
11016 panic!("expected route.bind request, got {bind:?}");
11017 };
11018 assert_eq!(consumer_capabilities, None);
11019
11020 handler
11021 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11022 .await
11023 .unwrap();
11024 let route_response = route_task.await.unwrap();
11025 assert!(route_response.is_empty());
11026 let published = client_rx.recv().await.unwrap();
11027 assert!(matches!(
11028 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11029 ClientControlResponse::RouteOpen { .. }
11030 ));
11031 }
11032
11033 async fn bind_role_versions_for(
11037 sent: Option<BTreeMap<String, String>>,
11038 connection: u64,
11039 ) -> Option<BTreeMap<String, String>> {
11040 let registry = Arc::new(Registry::default());
11041 let forwarding = Arc::new(ForwardingTable::default());
11042 let handler =
11043 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11044 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
11045 hello_via_sink(
11046 &handler,
11047 &module_ctx,
11048 &mut module_rx,
11049 hello_frame("aft", PROTOCOL_VERSION, 7),
11050 )
11051 .await;
11052 let project_root = unique_project_root("role-versions");
11053 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
11054 let route_handler = handler.clone();
11055 let route_task = tokio::spawn(async move {
11056 route_handler
11057 .handle_control_frame(
11058 &client_ctx,
11059 route_open_frame_with_role_versions(403, "aft", project_root, sent),
11060 )
11061 .await
11062 .unwrap()
11063 });
11064 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11065 .await
11066 .expect("a well-formed route.open reaches the module as a bind")
11067 .unwrap();
11068 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11069 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
11070 panic!("expected route.bind request, got {bind:?}");
11071 };
11072 handler
11073 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11074 .await
11075 .unwrap();
11076 assert!(route_task.await.unwrap().is_empty());
11077 let published = client_rx.recv().await.unwrap();
11078 assert!(matches!(
11079 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11080 ClientControlResponse::RouteOpen { .. }
11081 ));
11082 role_versions
11083 }
11084
11085 #[tokio::test]
11086 async fn route_open_relays_role_versions_verbatim() {
11087 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
11088 assert_eq!(
11089 bind_role_versions_for(Some(sent.clone()), 141).await,
11090 Some(sent)
11091 );
11092 }
11093
11094 #[tokio::test]
11097 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
11098 assert_eq!(bind_role_versions_for(None, 143).await, None);
11099 assert_eq!(
11100 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
11101 None
11102 );
11103 }
11104
11105 #[tokio::test]
11106 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
11107 let registry = Arc::new(Registry::default());
11108 let forwarding = Arc::new(ForwardingTable::default());
11109 let handler =
11110 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11111 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
11112 hello_via_sink(
11113 &handler,
11114 &module_ctx,
11115 &mut module_rx,
11116 hello_frame("aft", PROTOCOL_VERSION, 7),
11117 )
11118 .await;
11119
11120 let nine: BTreeMap<String, String> = (0..9)
11121 .map(|index| (format!("role-{index}"), "v1".to_string()))
11122 .collect();
11123 for (label, malformed) in [
11124 (
11125 "invalid role name",
11126 role_versions(&[("Tool_Provider", "v1")]),
11127 ),
11128 ("invalid version", role_versions(&[("tool-provider", "v0")])),
11129 ("nine entries", nine),
11130 ] {
11131 let responses = tokio::time::timeout(
11134 Duration::from_secs(1),
11135 handler.handle_control_frame(
11136 &route_ctx(ConnectionId::new(148)).0,
11137 route_open_frame_with_role_versions(
11138 404,
11139 "aft",
11140 unique_project_root("role-versions-malformed"),
11141 Some(malformed),
11142 ),
11143 ),
11144 )
11145 .await
11146 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
11147 .unwrap();
11148 assert_eq!(responses.len(), 1, "{label}");
11149 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
11150 let error = parse_error(&responses[0]);
11151 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
11152 assert_eq!(
11153 error["detail"]["field"], "role_versions",
11154 "{label}: {error}"
11155 );
11156 assert!(
11157 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
11158 "{label}: a malformed declaration is terminal"
11159 );
11160 assert!(
11161 module_rx.try_recv().is_err(),
11162 "{label}: the module must never see a bind"
11163 );
11164 }
11165 }
11166
11167 #[tokio::test]
11171 async fn route_role_versions_capability_is_advertised() {
11172 let handler = ControlHandler::new(Arc::new(Registry::default()));
11173 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
11174 let ack = hello_via_sink(
11175 &handler,
11176 &ctx,
11177 &mut rx,
11178 hello_frame("m", PROTOCOL_VERSION, 1),
11179 )
11180 .await;
11181 let ack = parse_ack(&ack);
11182 assert!(
11183 ack.subc_capabilities
11184 .iter()
11185 .any(|c| c == "route-role-versions/v1"),
11186 "{:?}",
11187 ack.subc_capabilities
11188 );
11189
11190 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11191 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11192 let reply = handler
11193 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
11194 .await
11195 .unwrap()
11196 .pop()
11197 .unwrap();
11198 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11199 serde_json::from_slice(&reply.body).unwrap()
11200 else {
11201 panic!("not a server.describe reply");
11202 };
11203 assert!(
11204 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
11205 "{capabilities:?}"
11206 );
11207 }
11208
11209 #[tokio::test]
11213 async fn catalog_events_capability_is_advertised() {
11214 let handler = ControlHandler::new(Arc::new(Registry::default()));
11215 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11216 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11217 let reply = handler
11218 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
11219 .await
11220 .unwrap()
11221 .pop()
11222 .unwrap();
11223 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11224 serde_json::from_slice(&reply.body).unwrap()
11225 else {
11226 panic!("not a server.describe reply");
11227 };
11228 assert!(
11229 capabilities
11230 .iter()
11231 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
11232 "{capabilities:?}"
11233 );
11234 }
11235
11236 #[tokio::test]
11237 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
11238 let handler = ControlHandler::new(Arc::new(Registry::default()));
11239 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
11240 let ack = hello_via_sink(
11241 &handler,
11242 &module,
11243 &mut module_rx,
11244 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
11245 )
11246 .await;
11247 let ack = parse_ack(&ack);
11248 assert!(
11249 ack.subc_capabilities
11250 .iter()
11251 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11252 "{:?}",
11253 ack.subc_capabilities
11254 );
11255 assert!(ack
11256 .subc_capabilities
11257 .iter()
11258 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11259
11260 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
11261 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11262 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11263 let reply = handler
11264 .handle_control_frame(&client, frame)
11265 .await
11266 .unwrap()
11267 .pop()
11268 .unwrap();
11269 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11270 serde_json::from_slice(&reply.body).unwrap()
11271 else {
11272 panic!("not a server.describe reply");
11273 };
11274 assert!(
11275 capabilities
11276 .iter()
11277 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11278 "{capabilities:?}"
11279 );
11280 assert!(capabilities
11281 .iter()
11282 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11283 }
11284
11285 #[tokio::test]
11286 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
11287 let handler = ControlHandler::new(Arc::new(Registry::default()));
11288 let (module, mut rx) = route_ctx(ConnectionId::new(154));
11289 let ack = hello_via_sink(
11290 &handler,
11291 &module,
11292 &mut rx,
11293 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
11294 )
11295 .await;
11296 assert!(parse_ack(&ack)
11297 .subc_capabilities
11298 .iter()
11299 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11300 let (client, _rx) = route_ctx(ConnectionId::new(155));
11301 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11302 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11303 let reply = handler
11304 .handle_control_frame(&client, frame)
11305 .await
11306 .unwrap()
11307 .pop()
11308 .unwrap();
11309 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11310 serde_json::from_slice(&reply.body).unwrap()
11311 else {
11312 panic!("not a server.describe reply")
11313 };
11314 assert!(capabilities
11315 .iter()
11316 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11317 }
11318
11319 #[tokio::test]
11320 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
11321 let registry = Arc::new(Registry::default());
11322 let forwarding = Arc::new(ForwardingTable::default());
11323 let handler =
11324 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11325 .with_health_probe_timeout(Duration::from_secs(5));
11326 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
11327 hello_via_sink(
11328 &handler,
11329 &module_ctx,
11330 &mut module_rx,
11331 non_routable_hello_frame_with_control_ops(
11332 "mcp",
11333 300,
11334 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11335 ),
11336 )
11337 .await;
11338 assert!(registry
11339 .get_module("mcp")
11340 .unwrap()
11341 .unwrap()
11342 .manifest
11343 .provides
11344 .is_empty());
11345
11346 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
11347 let route_response = handler
11348 .handle_control_frame(
11349 &route_client_ctx,
11350 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
11351 )
11352 .await
11353 .unwrap();
11354 assert_eq!(route_response[0].header.ty, FrameType::Error);
11355 assert_eq!(
11356 parse_error(&route_response[0])["code"],
11357 "target_unavailable"
11358 );
11359 assert!(parse_error(&route_response[0])["message"]
11360 .as_str()
11361 .unwrap()
11362 .contains("does not provide the requested target"));
11363 assert!(module_rx.try_recv().is_err());
11364
11365 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
11366 let health_handler = handler.clone();
11367 let health_task = tokio::spawn(async move {
11368 health_handler
11369 .handle_control_frame(
11370 &health_client_ctx,
11371 supervisor_health_probe_frame(302, "mcp"),
11372 )
11373 .await
11374 .unwrap()
11375 });
11376 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11377 .await
11378 .unwrap()
11379 .unwrap();
11380 assert_eq!(
11381 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11382 ModuleControlRequest::HealthCheck {}
11383 );
11384 handler
11385 .handle_control_frame(
11386 &module_ctx,
11387 health_response(health_frame.header.corr, HealthStatus::Ok),
11388 )
11389 .await
11390 .unwrap();
11391 let health_response = health_task.await.unwrap();
11392 assert_eq!(health_response[0].header.ty, FrameType::Response);
11393 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11394 ClientControlResponse::SupervisorHealthProbe {
11395 module_id, status, ..
11396 } => {
11397 assert_eq!(module_id, "mcp");
11398 assert_eq!(status, HealthStatus::Ok);
11399 }
11400 other => panic!("unexpected health response: {other:?}"),
11401 }
11402
11403 forwarding
11408 .cleanup_connection(module_ctx.connection_id)
11409 .unwrap();
11410 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
11411 let cleanup_response = tokio::time::timeout(
11412 Duration::from_millis(200),
11413 handler.handle_control_frame(
11414 &cleanup_probe_ctx,
11415 supervisor_health_probe_frame(303, "mcp"),
11416 ),
11417 )
11418 .await
11419 .expect("probe should fail immediately when the control lane is gone")
11420 .unwrap();
11421 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
11422 assert_eq!(
11423 parse_error(&cleanup_response[0])["code"],
11424 "target_unavailable"
11425 );
11426 assert!(parse_error(&cleanup_response[0])["message"]
11427 .as_str()
11428 .unwrap()
11429 .contains("no module connection"));
11430
11431 handler
11432 .cleanup_connection(module_ctx.connection_id)
11433 .unwrap();
11434 }
11435
11436 #[tokio::test]
11437 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
11438 let registry = Arc::new(Registry::default());
11439 let supervisor_handle = SupervisorHandle::new();
11440 let supervisor =
11441 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11442 .with_handle(supervisor_handle.clone())
11443 .with_connection_file_path(
11444 std::env::temp_dir()
11445 .join(format!("subc-route-open-warming-{}", std::process::id())),
11446 );
11447 let module = supervisor
11448 .supervise_configured(
11449 ModuleSpec {
11450 module_id: "warming".to_string(),
11451 program: fake_aft_stub_path(),
11452 args: Vec::new(),
11453 env: Vec::new(),
11454 reserved: false,
11455 reserved_prefixes: Vec::new(),
11456 protocol: ModuleProtocol::Subc,
11457 overlap: Default::default(),
11458 },
11459 true,
11460 )
11461 .unwrap();
11462 assert_eq!(module.state().unwrap(), ModuleState::Running);
11463
11464 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11465 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
11466 let response = handler
11467 .handle_control_frame(
11468 &ctx,
11469 route_open_frame(304, "warming", unique_project_root("warming")),
11470 )
11471 .await
11472 .unwrap();
11473 module.stop().await.unwrap();
11474
11475 assert_eq!(response[0].header.ty, FrameType::Error);
11476 let error = parse_error(&response[0]);
11477 assert_eq!(error["code"], "module_warming");
11478 assert!(error["message"]
11479 .as_str()
11480 .unwrap()
11481 .contains("state=running, enabled=true, live=false"));
11482 }
11483
11484 #[test]
11485 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
11486 let handler = ControlHandler::new(Arc::new(Registry::default()));
11487 let capture = EventCapture::default();
11488 let _subscriber =
11489 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11490 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
11491 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
11492 let pending = (0..limit).collect::<Vec<_>>();
11493 let response = handler
11494 .route_open_capacity_refusal(
11495 &ctx,
11496 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
11497 "busy",
11498 pending.len(),
11499 limit,
11500 )
11501 .unwrap();
11502 assert_eq!(parse_error(&response)["code"], "target_unavailable");
11503 let event = capture
11504 .events()
11505 .into_iter()
11506 .find(|event| {
11507 event.target == "control"
11508 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
11509 })
11510 .expect("connection admission refusal event");
11511 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11512 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11513 }
11514
11515 #[test]
11516 fn route_open_target_cap_logs_admission_reason_and_capacity() {
11517 let handler = ControlHandler::new(Arc::new(Registry::default()));
11518 let capture = EventCapture::default();
11519 let _subscriber =
11520 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11521 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
11522 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
11523 let guards = (0..limit)
11524 .map(|_| {
11525 handler
11526 .route_bind_concurrency
11527 .try_admit("busy", limit)
11528 .unwrap()
11529 })
11530 .collect::<Vec<_>>();
11531 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
11532 Err(in_flight) => in_flight,
11533 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
11534 };
11535 let response = handler
11536 .route_open_target_capacity_refusal(
11537 &ctx,
11538 &route_open_frame(397, "busy", unique_project_root("target-cap")),
11539 "busy",
11540 in_flight,
11541 )
11542 .unwrap();
11543 assert_eq!(parse_error(&response)["code"], "target_unavailable");
11544 let event = capture
11545 .events()
11546 .into_iter()
11547 .find(|event| {
11548 event.target == "control"
11549 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
11550 })
11551 .expect("target admission refusal event");
11552 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11553 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11554 drop(guards);
11555 }
11556
11557 #[tokio::test(flavor = "current_thread")]
11558 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
11559 let handler = ControlHandler::new(Arc::new(Registry::default()));
11560 let capture = EventCapture::default();
11561 let _subscriber =
11562 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11563 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
11564 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11565
11566 let _ = handler
11567 .handle_client_control_request(
11568 &ctx,
11569 frame,
11570 ClientControlRequest::SupervisorSetEnabled {
11571 module_id: "broca".to_string(),
11572 enabled: false,
11573 },
11574 &mut None,
11575 )
11576 .await
11577 .unwrap();
11578
11579 let events = capture
11580 .events()
11581 .into_iter()
11582 .filter(|event| {
11583 event.target == "control"
11584 && event.fields.get("message").map(String::as_str)
11585 == Some("supervisor request received")
11586 })
11587 .collect::<Vec<_>>();
11588 assert_eq!(events.len(), 1, "one supervisor request log line");
11589 let event = &events[0];
11590 assert_eq!(event.level, tracing::Level::INFO);
11591 assert_eq!(
11592 event.fields.get("op"),
11593 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
11594 );
11595 assert_eq!(
11596 event.fields.get("module_id"),
11597 Some(&"\"broca\"".to_string())
11598 );
11599 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
11600 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
11601 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
11602 }
11603
11604 #[tokio::test(flavor = "current_thread")]
11605 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
11606 let registry = Arc::new(Registry::default());
11607 let module_connection = ConnectionId::new(303);
11608 registry
11609 .register_with_control_ops(
11610 manifest("aft", PROTOCOL_VERSION),
11611 PROTOCOL_VERSION,
11612 module_connection,
11613 module_baseline_control_ops(),
11614 )
11615 .unwrap();
11616 let handler = ControlHandler::new(registry);
11617 let capture = EventCapture::default();
11618 let _subscriber =
11619 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11620 let (ctx, _rx) = route_ctx(module_connection);
11621 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11622
11623 let _ = handler
11624 .handle_client_control_request(
11625 &ctx,
11626 frame,
11627 ClientControlRequest::SupervisorSetEnabled {
11628 module_id: "broca".to_string(),
11629 enabled: false,
11630 },
11631 &mut None,
11632 )
11633 .await
11634 .unwrap();
11635
11636 let callers = capture
11637 .events()
11638 .into_iter()
11639 .filter(|event| {
11640 event.fields.get("message").map(String::as_str)
11641 == Some("supervisor request received")
11642 })
11643 .map(|event| event.fields.get("caller").cloned())
11644 .collect::<Vec<_>>();
11645 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
11646 }
11647
11648 #[tokio::test(flavor = "current_thread")]
11649 async fn supervisor_list_does_not_log_request_received() {
11650 let handler = ControlHandler::new(Arc::new(Registry::default()));
11651 let capture = EventCapture::default();
11652 let _subscriber =
11653 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11654 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
11655 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
11656
11657 handler
11658 .handle_client_control_request(
11659 &ctx,
11660 frame,
11661 ClientControlRequest::SupervisorList {},
11662 &mut None,
11663 )
11664 .await
11665 .unwrap();
11666
11667 assert!(capture.events().into_iter().all(|event| {
11668 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11669 }));
11670 }
11671
11672 #[tokio::test(flavor = "current_thread")]
11673 async fn supervisor_rescan_preview_does_not_log_request_received() {
11674 let handler = ControlHandler::new(Arc::new(Registry::default()));
11675 let capture = EventCapture::default();
11676 let _subscriber =
11677 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11678 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
11679 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
11680
11681 handler
11682 .handle_client_control_request(
11683 &ctx,
11684 frame,
11685 ClientControlRequest::SupervisorRescan { preview: true },
11686 &mut None,
11687 )
11688 .await
11689 .unwrap();
11690
11691 assert!(capture.events().into_iter().all(|event| {
11692 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11693 }));
11694 }
11695
11696 struct ApproveOperator;
11697 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
11698 fn prompt(
11699 &self,
11700 _: &str,
11701 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
11702 ) -> crate::operator_confirm::ProviderResult {
11703 crate::operator_confirm::ProviderResult::Approved
11704 }
11705 }
11706
11707 #[tokio::test]
11708 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
11709 assert_eq!(
11710 ControlHandler::default().privileged_operator_prompt,
11711 PrivilegedOperatorPrompt::Log
11712 );
11713 for (initial, changed, running) in [
11714 ("log", "enforce", PrivilegedOperatorPrompt::Log),
11715 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
11716 ] {
11717 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
11718 let path = dir.join("subc.jsonc");
11719 std::fs::write(
11720 &path,
11721 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
11722 )
11723 .unwrap();
11724 let config = crate::daemon_config::load(&path).unwrap().unwrap();
11725 let registry = Arc::new(Registry::default());
11726 let handle = SupervisorHandle::new();
11727 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
11728 .with_handle(handle.clone());
11729 let handler = ControlHandler::new(registry)
11730 .with_supervisor(handle)
11731 .with_privileged_operator_prompt(config.privileged_operator_prompt)
11732 .with_supervisor_rescan(supervisor, &path, config.port);
11733 handler.forwarding.operator_confirms().configure(
11734 Arc::new(ApproveOperator),
11735 crate::operator_confirm::OperatorLimits::default(),
11736 );
11737 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
11738
11739 for (disk_mode, needs_restart) in [
11740 (Some(initial), false),
11741 (Some(changed), true),
11742 (Some(changed), true),
11743 (None, running == PrivilegedOperatorPrompt::Enforce),
11744 (Some(initial), false),
11745 ] {
11746 let mut doc = json!({"version":1});
11747 if let Some(mode) = disk_mode {
11748 doc["privileged_operator_prompt"] = json!(mode);
11749 }
11750 std::fs::write(&path, doc.to_string()).unwrap();
11751 for preview in [true, false] {
11752 let request = ClientControlRequest::SupervisorRescan { preview };
11753 let frame = Frame::build(
11754 FrameType::Request,
11755 control_flags(),
11756 0,
11757 0,
11758 1,
11759 serde_json::to_vec(&request).unwrap(),
11760 )
11761 .unwrap();
11762 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
11763 assert_eq!(frames.len(), 1);
11764 let ClientControlResponse::SupervisorRescan { result } =
11765 serde_json::from_slice(&frames[0].body).unwrap()
11766 else {
11767 panic!("expected a rescan response");
11768 };
11769 let expected = if needs_restart {
11770 vec!["privileged_operator_prompt"]
11771 } else {
11772 Vec::new()
11773 };
11774 assert_eq!(result.restart_required, expected);
11775 assert_eq!(handler.privileged_operator_prompt, running);
11776 assert_eq!(
11777 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
11778 running
11779 );
11780 }
11781 }
11782 }
11783 }
11784
11785 #[tokio::test]
11786 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
11787 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
11788 let path = dir.join("subc.jsonc");
11789 let registry = Arc::new(Registry::default());
11790 let handle = SupervisorHandle::new();
11791 let supervisor =
11792 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
11793 let handler = ControlHandler::new(registry)
11794 .with_supervisor(handle)
11795 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
11796 .with_supervisor_rescan(supervisor, &path, None);
11797 handler.forwarding.operator_confirms().configure(
11798 Arc::new(ApproveOperator),
11799 crate::operator_confirm::OperatorLimits::default(),
11800 );
11801 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
11802 for preview in [true, false] {
11803 std::fs::write(
11804 &path,
11805 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
11806 )
11807 .unwrap();
11808 let request = ClientControlRequest::SupervisorRescan { preview };
11809 let frame = Frame::build(
11810 FrameType::Request,
11811 control_flags(),
11812 0,
11813 0,
11814 1,
11815 serde_json::to_vec(&request).unwrap(),
11816 )
11817 .unwrap();
11818 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
11819 assert_eq!(frames.len(), 1);
11820 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
11821 assert_eq!(
11822 handler.privileged_operator_prompt,
11823 PrivilegedOperatorPrompt::Enforce
11824 );
11825 }
11826 }
11827
11828 #[tokio::test]
11833 async fn route_open_refusal_names_the_check_that_refused() {
11834 let handler = ControlHandler::new(Arc::new(Registry::default()));
11835 let capture = EventCapture::default();
11836 let _subscriber =
11837 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11838 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
11839 let response = handler
11840 .handle_control_frame(
11841 &ctx,
11842 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
11843 )
11844 .await
11845 .unwrap();
11846
11847 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11848 let event = capture
11849 .events()
11850 .into_iter()
11851 .find(|event| {
11852 event.target == "control"
11853 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
11854 })
11855 .expect("route.open refusal event");
11856 assert_eq!(
11857 event.fields.get("reason"),
11858 Some(&"\"not_registered\"".to_string())
11859 );
11860 }
11861
11862 #[tokio::test]
11863 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
11864 let registry = Arc::new(Registry::default());
11865 let supervisor_handle = SupervisorHandle::new();
11866 let supervisor =
11867 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11868 .with_handle(supervisor_handle.clone())
11869 .with_connection_file_path(std::env::temp_dir().join(format!(
11870 "subc-route-open-refusal-info-{}",
11871 std::process::id()
11872 )));
11873 let module = supervisor
11874 .supervise_configured(
11875 ModuleSpec {
11876 module_id: "warming".to_string(),
11877 program: fake_aft_stub_path(),
11878 args: Vec::new(),
11879 env: Vec::new(),
11880 reserved: false,
11881 reserved_prefixes: Vec::new(),
11882 protocol: ModuleProtocol::Subc,
11883 overlap: Default::default(),
11884 },
11885 true,
11886 )
11887 .unwrap();
11888 assert_eq!(module.state().unwrap(), ModuleState::Running);
11889
11890 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11891 assert!(handler
11892 .counters()
11893 .snapshot()
11894 .get("route_open_refused_by_code")
11895 .is_none());
11896 let capture = EventCapture::default();
11897 let _subscriber =
11898 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11899 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
11900 let response = handler
11901 .handle_control_frame(
11902 &ctx,
11903 route_open_frame(394, "warming", unique_project_root("refusal-info")),
11904 )
11905 .await
11906 .unwrap();
11907 module.stop().await.unwrap();
11908
11909 assert_eq!(parse_error(&response[0])["code"], "module_warming");
11910 let event = capture
11911 .events()
11912 .into_iter()
11913 .find(|event| {
11914 event.target == "control"
11915 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
11916 })
11917 .expect("route.open refusal event");
11918 assert_eq!(
11919 event.fields.get("module_id"),
11920 Some(&"\"warming\"".to_string())
11921 );
11922 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
11923 assert_eq!(
11924 event.fields.get("reason"),
11925 Some(&"\"supervised_not_registered\"".to_string())
11926 );
11927 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
11928 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
11929 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
11930 assert_eq!(
11931 handler.counters().snapshot()["route_open_refused_by_code"],
11932 json!({ "module_warming": 1 })
11933 );
11934 }
11935
11936 const OUTAGE_START: &str = "route.open refusing module: not serving";
11937 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
11938
11939 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
11940 capture
11941 .events()
11942 .into_iter()
11943 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
11944 .collect()
11945 }
11946
11947 fn supervise_stub(
11948 registry: &Arc<Registry>,
11949 module_id: &str,
11950 enabled: bool,
11951 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
11952 let supervisor_handle = SupervisorHandle::new();
11953 let supervisor =
11954 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
11955 .with_handle(supervisor_handle.clone())
11956 .with_connection_file_path(std::env::temp_dir().join(format!(
11957 "subc-route-outage-{module_id}-{}",
11958 std::process::id()
11959 )));
11960 let module = supervisor
11961 .supervise_configured(
11962 ModuleSpec {
11963 module_id: module_id.to_string(),
11964 program: fake_aft_stub_path(),
11965 args: Vec::new(),
11966 env: Vec::new(),
11967 reserved: false,
11968 reserved_prefixes: Vec::new(),
11969 protocol: ModuleProtocol::Subc,
11970 overlap: Default::default(),
11971 },
11972 enabled,
11973 )
11974 .unwrap();
11975 (supervisor_handle, module)
11976 }
11977
11978 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
11979 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
11980 module_id: module_id.to_string(),
11981 drain_timeout_ms: Some(50),
11982 })
11983 .unwrap();
11984 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11985 }
11986
11987 #[test]
11991 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
11992 let registry = Arc::new(Registry::default());
11993 let forwarding = Arc::new(ForwardingTable::default());
11994 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11995 let second = ControlHandler::with_forwarding(registry, forwarding);
11996 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
11997 }
11998
11999 #[tokio::test(flavor = "current_thread")]
12003 async fn route_open_unknown_module_refusals_add_no_outage_state() {
12004 let handler = ControlHandler::new(Arc::new(Registry::default()));
12005 let capture = EventCapture::default();
12006 let _subscriber =
12007 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12008 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
12009 for corr in 0..8 {
12010 let response = handler
12011 .handle_control_frame(
12012 &ctx,
12013 route_open_frame(
12014 380 + corr,
12015 &format!("nobody-{corr}"),
12016 unique_project_root("outage-unknown"),
12017 ),
12018 )
12019 .await
12020 .unwrap();
12021 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12022 }
12023
12024 assert_eq!(handler.route_outages.tracked_module_count(), 0);
12025 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
12026 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
12027 }
12028
12029 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12034 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
12035 let registry = Arc::new(Registry::default());
12036 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
12037 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12038 let capture = EventCapture::default();
12039 let _subscriber =
12040 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12041 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
12042 handler.route_outages.record_accepted("outage-restart");
12045
12046 let response = handler
12047 .handle_control_frame(
12048 &ctx,
12049 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
12050 )
12051 .await
12052 .unwrap();
12053 assert_eq!(response[0].header.ty, FrameType::Error);
12054 let starts = outage_lines(&capture, OUTAGE_START);
12055 assert_eq!(starts.len(), 1, "{starts:?}");
12056 assert_eq!(starts[0].level, tracing::Level::WARN);
12057 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
12058 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
12059 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
12060 handler.route_outages.record_accepted("outage-restart");
12061 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
12062
12063 let restart = handler
12064 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
12065 .await
12066 .unwrap();
12067 assert_eq!(
12068 restart[0].header.ty,
12069 FrameType::Response,
12070 "{:?}",
12071 parse_error(&restart[0])
12072 );
12073 handler
12074 .handle_control_frame(
12075 &ctx,
12076 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
12077 )
12078 .await
12079 .unwrap();
12080 module.stop().await.unwrap();
12081
12082 let starts = outage_lines(&capture, OUTAGE_START);
12083 assert_eq!(starts.len(), 2, "{starts:?}");
12084 assert_eq!(starts[1].level, tracing::Level::INFO);
12085 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
12086 }
12087
12088 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12092 async fn failed_operator_restart_leaves_no_operator_mark() {
12093 let registry = Arc::new(Registry::default());
12094 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
12095 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12096 let capture = EventCapture::default();
12097 let _subscriber =
12098 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12099 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
12100 handler.route_outages.record_accepted("outage-disabled");
12101
12102 let restart = handler
12103 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
12104 .await
12105 .unwrap();
12106 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
12107 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
12108
12109 handler
12110 .handle_control_frame(
12111 &ctx,
12112 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
12113 )
12114 .await
12115 .unwrap();
12116 let starts = outage_lines(&capture, OUTAGE_START);
12117 assert_eq!(starts.len(), 1, "{starts:?}");
12118 assert_eq!(starts[0].level, tracing::Level::WARN);
12119 }
12120
12121 #[tokio::test(flavor = "current_thread")]
12122 async fn route_open_unknown_module_escapes_target_module_id() {
12123 let handler = ControlHandler::new(Arc::new(Registry::default()));
12124 let capture = EventCapture::default();
12125 let _subscriber =
12126 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12127 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
12128 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
12129 let response = handler
12130 .handle_control_frame(
12131 &ctx,
12132 route_open_frame(
12133 395,
12134 hostile_module_id,
12135 unique_project_root("hostile-target-module-id"),
12136 ),
12137 )
12138 .await
12139 .unwrap();
12140
12141 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12142 let event = capture
12143 .events()
12144 .into_iter()
12145 .find(|event| {
12146 event.target == "control"
12147 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12148 })
12149 .expect("route.open unknown-module refusal event");
12150 let logged = event.fields.get("module_id").expect("module_id field");
12151 assert!(!logged.bytes().any(|byte| byte < 0x20));
12152 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12153 }
12154
12155 #[tokio::test(flavor = "current_thread")]
12156 async fn route_open_module_rejection_uses_daemon_counter_key() {
12157 let registry = Arc::new(Registry::default());
12158 let forwarding = Arc::new(ForwardingTable::default());
12159 let handler =
12160 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12161 let module_connection = ConnectionId::new(95);
12162 let (module_ctx, mut module_rx) = route_ctx(module_connection);
12163 hello_via_sink(
12164 &handler,
12165 &module_ctx,
12166 &mut module_rx,
12167 hello_frame("aft", PROTOCOL_VERSION, 395),
12168 )
12169 .await;
12170
12171 let client_connection = ConnectionId::new(96);
12172 let (client_ctx, _client_rx) = route_ctx(client_connection);
12173 let capture = EventCapture::default();
12174 let _subscriber =
12175 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12176 let (route_task, bind) = relay_route_open(
12177 &handler,
12178 client_connection,
12179 &client_ctx.egress,
12180 &mut module_rx,
12181 396,
12182 "aft",
12183 "hostile-module-code",
12184 )
12185 .await;
12186 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
12187 let rejection = Frame::build(
12188 FrameType::Error,
12189 control_flags(),
12190 0,
12191 0,
12192 bind.header.corr,
12193 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
12194 )
12195 .unwrap();
12196 handler
12197 .handle_control_frame(&module_ctx, rejection)
12198 .await
12199 .unwrap();
12200
12201 let response = route_task.await.unwrap();
12202 assert_eq!(parse_error(&response[0])["code"], hostile_code);
12203 let counters = handler.counters().snapshot();
12204 assert_eq!(
12205 counters["route_open_refused_by_code"],
12206 json!({ "module_rejected": 1 })
12207 );
12208 assert!(counters["route_open_refused_by_code"]
12209 .get(hostile_code)
12210 .is_none());
12211
12212 let event = capture
12213 .events()
12214 .into_iter()
12215 .find(|event| {
12216 event.target == "control"
12217 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
12218 })
12219 .expect("route.open module-rejection refusal event");
12220 let logged = event.fields.get("module_code").expect("module_code field");
12221 assert!(!logged.bytes().any(|byte| byte < 0x20));
12222 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12223 }
12224
12225 #[tokio::test]
12226 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
12227 let registry = Arc::new(Registry::default());
12228 let supervisor_handle = SupervisorHandle::new();
12229 let missing_program = std::env::temp_dir().join(format!(
12230 "subc-route-open-missing-program-{}",
12231 std::process::id()
12232 ));
12233 let supervisor =
12234 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12235 .with_handle(supervisor_handle.clone());
12236 let module = supervisor
12237 .supervise_configured(
12238 ModuleSpec {
12239 module_id: "failed".to_string(),
12240 program: missing_program,
12241 args: Vec::new(),
12242 env: Vec::new(),
12243 reserved: false,
12244 reserved_prefixes: Vec::new(),
12245 protocol: ModuleProtocol::Subc,
12246 overlap: Default::default(),
12247 },
12248 true,
12249 )
12250 .unwrap();
12251 assert_eq!(module.state().unwrap(), ModuleState::Failed);
12252
12253 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12254 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
12255 let response = handler
12256 .handle_control_frame(
12257 &ctx,
12258 route_open_frame(305, "failed", unique_project_root("failed")),
12259 )
12260 .await
12261 .unwrap();
12262
12263 assert_eq!(response[0].header.ty, FrameType::Error);
12264 let error = parse_error(&response[0]);
12265 assert_eq!(error["code"], "target_unavailable");
12266 assert!(error["message"]
12267 .as_str()
12268 .unwrap()
12269 .contains("state=failed, enabled=true, live=false"));
12270 }
12271
12272 #[tokio::test]
12273 async fn route_open_role_mismatch_remains_target_unavailable() {
12274 let registry = Arc::new(Registry::default());
12275 let handler = ControlHandler::new(Arc::clone(®istry));
12276 handler
12277 .handle_control(
12278 ConnectionId::new(41),
12279 non_routable_hello_frame_with_control_ops("health-only", 306, None),
12280 )
12281 .unwrap();
12282
12283 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
12284 let response = handler
12285 .handle_control_frame(
12286 &ctx,
12287 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
12288 )
12289 .await
12290 .unwrap();
12291
12292 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12293 assert!(parse_error(&response[0])["message"]
12294 .as_str()
12295 .unwrap()
12296 .contains("does not provide the requested target"));
12297 }
12298
12299 #[tokio::test]
12300 async fn route_open_inactive_registration_remains_target_unavailable() {
12301 let registry = Arc::new(Registry::default());
12302 let handler = ControlHandler::new(Arc::clone(®istry));
12303 handler
12304 .handle_control(
12305 ConnectionId::new(43),
12306 hello_frame("inactive", PROTOCOL_VERSION, 308),
12307 )
12308 .unwrap();
12309 assert!(registry
12310 .set_module_state_for_test("inactive", ChannelState::Closed)
12311 .unwrap());
12312
12313 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
12314 let response = handler
12315 .handle_control_frame(
12316 &ctx,
12317 route_open_frame(309, "inactive", unique_project_root("inactive")),
12318 )
12319 .await
12320 .unwrap();
12321
12322 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12323 assert!(parse_error(&response[0])["message"]
12324 .as_str()
12325 .unwrap()
12326 .contains("is not active"));
12327 }
12328
12329 #[tokio::test]
12330 async fn late_health_reply_is_recorded_through_the_module_response_path() {
12331 let registry = Arc::new(Registry::default());
12332 let forwarding = Arc::new(ForwardingTable::default());
12333 let supervisor_handle = SupervisorHandle::new();
12334 let supervisor =
12335 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
12336 .with_forwarding(Arc::clone(&forwarding))
12337 .with_handle(supervisor_handle.clone());
12338 let module = supervisor
12339 .supervise_configured(
12340 crate::ModuleSpec {
12341 module_id: "late-health-response".to_string(),
12342 program: PathBuf::from("disabled-module"),
12343 args: Vec::new(),
12344 env: Vec::new(),
12345 reserved: false,
12346 reserved_prefixes: Vec::new(),
12347 protocol: ModuleProtocol::Subc,
12348 overlap: Default::default(),
12349 },
12350 false,
12351 )
12352 .unwrap();
12353 let handler =
12354 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12355 .with_supervisor(supervisor_handle);
12356 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
12357 handler
12358 .handle_control_frame(
12359 &module_ctx,
12360 hello_frame_with_control_ops(
12361 "late-health-response",
12362 PROTOCOL_VERSION,
12363 7,
12364 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12365 ),
12366 )
12367 .await
12368 .unwrap();
12369 let probe_started_at = Instant::now() - Duration::from_millis(80);
12370 let pending = forwarding
12371 .begin_health_probe_rpc_for(
12372 "late-health-response",
12373 MODULE_CONTROL_OP_HEALTH_CHECK,
12374 probe_started_at,
12375 Instant::now() - Duration::from_millis(1),
12376 )
12377 .unwrap();
12378 assert!(forwarding
12379 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
12380 .unwrap());
12381
12382 let responses = handler
12383 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
12384 .await
12385 .unwrap();
12386
12387 assert!(responses.is_empty());
12388 let health = module.status().unwrap().health;
12389 assert_eq!(health.late_answer_count, 1);
12390 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
12391 }
12392
12393 #[tokio::test]
12394 async fn health_probe_timeout_and_module_death_are_typed() {
12395 let registry = Arc::new(Registry::default());
12396 let forwarding = Arc::new(ForwardingTable::default());
12397 let handler =
12398 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12399 .with_health_probe_timeout(Duration::from_millis(50));
12400 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
12401 hello_via_sink(
12402 &handler,
12403 &module_ctx,
12404 &mut module_rx,
12405 hello_frame_with_control_ops(
12406 "aft",
12407 PROTOCOL_VERSION,
12408 7,
12409 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12410 ),
12411 )
12412 .await;
12413
12414 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12415 let responses = handler
12416 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
12417 .await
12418 .unwrap();
12419 assert_eq!(responses[0].header.ty, FrameType::Error);
12420 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
12421 let _ = module_rx.try_recv();
12422
12423 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
12424 let health_handler = handler.clone();
12425 let death_task = tokio::spawn(async move {
12426 health_handler
12427 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
12428 .await
12429 .unwrap()
12430 });
12431 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12432 .await
12433 .unwrap()
12434 .unwrap();
12435 handler
12436 .cleanup_connection(module_ctx.connection_id)
12437 .unwrap();
12438 let responses = death_task.await.unwrap();
12439 assert_eq!(responses[0].header.ty, FrameType::Error);
12440 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
12441 }
12442
12443 #[test]
12444 fn hello_requires_exact_protocol_version() {
12445 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
12446 let registry = Arc::new(Registry::default());
12447 let handler = ControlHandler::new(Arc::clone(®istry));
12448 let responses = handler
12449 .handle_control(
12450 ConnectionId::new(connection),
12451 hello_frame("aft", offered, 9),
12452 )
12453 .unwrap();
12454
12455 assert_eq!(responses.len(), 1);
12456 assert_eq!(responses[0].header.ty, FrameType::Error);
12457 let error = parse_error(&responses[0]);
12458 assert_eq!(error["code"], "version_unsupported");
12459 assert!(registry.get_module("aft").unwrap().is_none());
12460 assert_eq!(registry.active_registration_count().unwrap(), 0);
12461 }
12462 }
12463
12464 #[test]
12465 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
12466 let registry = Arc::new(Registry::default());
12467 let forwarding = Arc::new(ForwardingTable::default());
12468 let handler =
12469 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12470 let module_connection = ConnectionId::new(301);
12471 let registration = registry
12472 .register_with_control_ops(
12473 manifest("aft-push", PROTOCOL_VERSION),
12474 PROTOCOL_VERSION,
12475 module_connection,
12476 module_baseline_control_ops(),
12477 )
12478 .unwrap();
12479 let (module_tx, _module_rx) = mpsc::channel(8);
12480 let endpoint = forwarding
12481 .register_module_connection(
12482 module_connection,
12483 "aft-push".to_string(),
12484 PROTOCOL_VERSION,
12485 manifest_concurrency(®istration.manifest),
12486 FrameSink::new(module_tx),
12487 )
12488 .unwrap();
12489
12490 let unknown = Frame::build(
12492 FrameType::Push,
12493 control_flags(),
12494 0,
12495 0,
12496 5,
12497 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
12498 )
12499 .unwrap();
12500 let out = handler.handle_status_update(endpoint, unknown).unwrap();
12501 assert!(
12502 out.is_empty(),
12503 "unknown push op must be ignored, got {out:?}"
12504 );
12505
12506 let malformed = Frame::build(
12508 FrameType::Push,
12509 control_flags(),
12510 0,
12511 0,
12512 6,
12513 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
12514 )
12515 .unwrap();
12516 let out = handler.handle_status_update(endpoint, malformed).unwrap();
12517 assert_eq!(out.len(), 1);
12518 assert_eq!(out[0].header.ty, FrameType::Error);
12519 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
12520 }
12521
12522 #[test]
12523 fn hello_rejected_when_connection_already_owns_client_routes() {
12524 let registry = Arc::new(Registry::default());
12525 let forwarding = Arc::new(ForwardingTable::default());
12526 let handler =
12527 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12528 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
12530 let client_connection = ConnectionId::new(202);
12531
12532 let responses = handler
12535 .handle_control(
12536 client_connection,
12537 hello_frame("aft-second", PROTOCOL_VERSION, 9),
12538 )
12539 .unwrap();
12540 assert_eq!(responses[0].header.ty, FrameType::Error);
12541 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
12542 assert!(registry.get_module("aft-second").unwrap().is_none());
12543 }
12544
12545 #[tokio::test]
12546 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
12547 let registry = Arc::new(Registry::default());
12548 let forwarding = Arc::new(ForwardingTable::default());
12549 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
12550 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
12551 hello_via_sink(
12552 &handler,
12553 &module_ctx,
12554 &mut module_rx,
12555 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
12556 )
12557 .await;
12558 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
12559 let pending = forwarding
12560 .begin_route_bind_relay_for_test(
12561 client_ctx.connection_id,
12562 client_ctx.egress.clone(),
12563 2,
12564 "alpha",
12565 )
12566 .unwrap();
12567 forwarding
12568 .complete_pending_relay(
12569 module_ctx.connection_id,
12570 pending.corr,
12571 RouteBindRelayOutcome::Accepted,
12572 )
12573 .unwrap();
12574 client_rx.try_recv().unwrap();
12575 for module_id in ["beta", "alpha"] {
12576 let replies = handler
12577 .handle_control_frame(
12578 &module_ctx,
12579 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
12580 )
12581 .await
12582 .unwrap();
12583 assert_eq!(replies.len(), 1, "second HELLO must be refused");
12584 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
12585 }
12586 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
12587 assert!(registry.get_module("beta").unwrap().is_none());
12588 assert!(matches!(
12589 forwarding
12590 .lookup_data_route(
12591 client_ctx.connection_id,
12592 pending.client_channel,
12593 pending.client_epoch,
12594 )
12595 .unwrap(),
12596 DataRoute::Client(DataRouteState::Bound(_))
12597 ));
12598 assert!(handler
12599 .hello_launch_nonces
12600 .lock()
12601 .unwrap()
12602 .presented(module_ctx.connection_id, Some("alpha-nonce")));
12603 assert!(module_rx.try_recv().is_err());
12604 }
12605
12606 #[test]
12607 fn reserved_module_hello_requires_matching_launch_nonce() {
12608 let registry = Arc::new(Registry::default());
12609 let supervisor = SupervisorHandle::new();
12610 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
12613 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12614
12615 let no_nonce = handler
12617 .handle_control(
12618 ConnectionId::new(1),
12619 hello_frame("vault", PROTOCOL_VERSION, 1),
12620 )
12621 .unwrap();
12622 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
12623 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
12624 assert!(registry.get_module("vault").unwrap().is_none());
12625
12626 let wrong = handler
12628 .handle_control(
12629 ConnectionId::new(2),
12630 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
12631 )
12632 .unwrap();
12633 assert_eq!(wrong[0].header.ty, FrameType::Error);
12634 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
12635 assert!(registry.get_module("vault").unwrap().is_none());
12636
12637 let ok = handler
12639 .handle_control(
12640 ConnectionId::new(3),
12641 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
12642 )
12643 .unwrap();
12644 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
12645 assert!(registry.get_module("vault").unwrap().is_some());
12646 }
12647
12648 #[test]
12649 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
12650 let registry = Arc::new(Registry::default());
12651 let supervisor = SupervisorHandle::new();
12652 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
12653 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
12654 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12655
12656 let squat = handler
12657 .handle_control(
12658 ConnectionId::new(1),
12659 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
12660 )
12661 .unwrap();
12662 assert_eq!(squat[0].header.ty, FrameType::Error);
12663 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
12664 assert!(parse_error(&squat[0])["message"]
12665 .as_str()
12666 .unwrap()
12667 .contains("fed:"));
12668
12669 let accepted_peer = handler
12670 .handle_control(
12671 ConnectionId::new(2),
12672 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
12673 )
12674 .unwrap();
12675 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
12676
12677 let accepted_short = handler
12678 .handle_control(
12679 ConnectionId::new(3),
12680 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
12681 )
12682 .unwrap();
12683 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
12684
12685 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
12686 let response = handler
12687 .handle_control(
12688 ConnectionId::new(conn),
12689 hello_frame(module_id, PROTOCOL_VERSION, conn),
12690 )
12691 .unwrap();
12692 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
12693 }
12694 }
12695
12696 #[test]
12697 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
12698 let registry = Arc::new(Registry::default());
12699 let supervisor = SupervisorHandle::new();
12700 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
12701 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
12702 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
12703 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12704
12705 let owner_nonce = handler
12706 .handle_control(
12707 ConnectionId::new(1),
12708 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
12709 )
12710 .unwrap();
12711 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
12712 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
12713 assert!(registry.get_module("fed:special").unwrap().is_none());
12714
12715 let exact_nonce = handler
12716 .handle_control(
12717 ConnectionId::new(2),
12718 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
12719 )
12720 .unwrap();
12721 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
12722 assert!(registry.get_module("fed:special").unwrap().is_some());
12723 }
12724
12725 #[test]
12726 fn non_reserved_module_ignores_launch_nonce() {
12727 let registry = Arc::new(Registry::default());
12728 let handler = ControlHandler::new(Arc::clone(®istry));
12731 let no_nonce = handler
12732 .handle_control(
12733 ConnectionId::new(1),
12734 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
12735 )
12736 .unwrap();
12737 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
12738 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
12739
12740 let echoed_nonce = handler
12741 .handle_control(
12742 ConnectionId::new(2),
12743 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
12744 )
12745 .unwrap();
12746 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
12747 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
12748 }
12749
12750 #[test]
12751 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
12752 let handler = ControlHandler::default();
12753 let conn = ConnectionId::new(1);
12754 let malformed = Frame::build(
12755 FrameType::Hello,
12756 control_flags(),
12757 0,
12758 0,
12759 3,
12760 b"{not json".to_vec(),
12761 )
12762 .unwrap();
12763
12764 let error = handler.handle_control(conn, malformed).unwrap();
12765 assert_eq!(error[0].header.ty, FrameType::Error);
12766 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
12767
12768 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
12769 let pong = handler.handle_control(conn, ping).unwrap();
12770 assert_eq!(pong[0].header.ty, FrameType::Pong);
12771 assert_eq!(pong[0].header.corr, 4);
12772 }
12773
12774 #[test]
12775 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
12776 let registry = Arc::new(Registry::default());
12777 let handler = ControlHandler::new(Arc::clone(®istry));
12778
12779 handler
12780 .handle_control(
12781 ConnectionId::new(1),
12782 hello_frame("aft", PROTOCOL_VERSION, 1),
12783 )
12784 .unwrap();
12785 let duplicate = handler
12786 .handle_control(
12787 ConnectionId::new(2),
12788 hello_frame("aft", PROTOCOL_VERSION, 2),
12789 )
12790 .unwrap();
12791
12792 assert_eq!(duplicate[0].header.ty, FrameType::Error);
12793 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
12794 let registration = registry.get_module("aft").unwrap().unwrap();
12795 assert_eq!(registration.connection_id, ConnectionId::new(1));
12796 }
12797
12798 #[test]
12799 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
12800 let registry = Arc::new(Registry::default());
12801 let forwarding = Arc::new(ForwardingTable::default());
12802 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
12803 let handler =
12804 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12805 .with_process_liveness(process_liveness);
12806 let (ctx, route_channel, route_epoch) =
12807 bind_liveness_route(®istry, &forwarding, "aft-dead");
12808 let responses = handler
12809 .handle_route_poll(
12810 &ctx,
12811 route_poll_frame(41, PollKind::Liveness, route_channel),
12812 route_channel,
12813 route_epoch,
12814 PollKind::Liveness,
12815 )
12816 .unwrap();
12817
12818 assert_eq!(responses.len(), 1);
12819 assert_eq!(responses[0].header.ty, FrameType::Response);
12820 assert_route_poll_liveness(&responses[0], false);
12821 }
12822
12823 #[test]
12824 fn liveness_poll_without_process_source_uses_bound_route() {
12825 let registry = Arc::new(Registry::default());
12826 let forwarding = Arc::new(ForwardingTable::default());
12827 let handler =
12828 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12829 let (ctx, route_channel, route_epoch) =
12830 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
12831 let responses = handler
12832 .handle_route_poll(
12833 &ctx,
12834 route_poll_frame(42, PollKind::Liveness, route_channel),
12835 route_channel,
12836 route_epoch,
12837 PollKind::Liveness,
12838 )
12839 .unwrap();
12840
12841 assert_route_poll_liveness(&responses[0], true);
12842 }
12843
12844 #[test]
12845 fn liveness_poll_untracked_process_source_uses_bound_route() {
12846 let registry = Arc::new(Registry::default());
12847 let forwarding = Arc::new(ForwardingTable::default());
12848 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
12849 let handler =
12850 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12851 .with_process_liveness(process_liveness);
12852 let (ctx, route_channel, route_epoch) =
12853 bind_liveness_route(®istry, &forwarding, "aft-untracked");
12854 let responses = handler
12855 .handle_route_poll(
12856 &ctx,
12857 route_poll_frame(43, PollKind::Liveness, route_channel),
12858 route_channel,
12859 route_epoch,
12860 PollKind::Liveness,
12861 )
12862 .unwrap();
12863
12864 assert_route_poll_liveness(&responses[0], true);
12865 }
12866
12867 #[tokio::test]
12868 async fn unknown_op_returns_unknown_control_op() {
12869 let handler = ControlHandler::default();
12870 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
12871 let request = Frame::build(
12872 FrameType::Request,
12873 control_flags(),
12874 0,
12875 0,
12876 55,
12877 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
12878 )
12879 .unwrap();
12880
12881 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12882
12883 assert_eq!(response.len(), 1);
12884 assert_eq!(response[0].header.ty, FrameType::Error);
12885 assert_eq!(response[0].header.corr, 55);
12886 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
12887 }
12888
12889 #[tokio::test]
12890 async fn supervisor_provenance_rejects_unknown_exact_module() {
12891 let handler = ControlHandler::default();
12892 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
12893 let request = Frame::build(
12894 FrameType::Request,
12895 control_flags(),
12896 0,
12897 0,
12898 57,
12899 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
12900 )
12901 .unwrap();
12902
12903 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12904
12905 assert_eq!(response.len(), 1);
12906 assert_eq!(response[0].header.ty, FrameType::Error);
12907 assert_eq!(response[0].header.corr, 57);
12908 let error = parse_error(&response[0]);
12909 assert_eq!(error["code"], "unknown_module");
12910 assert_eq!(error["message"], "module_id 'missing' is not supervised");
12911 }
12912
12913 #[test]
12914 fn provenance_probe_override_keeps_handler_tests_deterministic() {
12915 let expected = subc_control::RunningImageAgreement::Unavailable {
12916 reason: subc_control::RunningImageUnavailableReason::HashFailed,
12917 };
12918 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
12919 assert_eq!(handler.provenance_probe_override, Some(expected));
12920 }
12921
12922 #[test]
12923 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
12924 let verdict = reload_verdict(
12925 std::path::Path::new("/bin/new"),
12926 Some(std::path::Path::new("/bin/old")),
12927 subc_control::RunningImageAgreement::Unavailable {
12928 reason: subc_control::RunningImageUnavailableReason::HashFailed,
12929 },
12930 );
12931 assert!(matches!(
12932 verdict.path,
12933 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
12934 if configured == std::path::Path::new("/bin/new")
12935 && spawned_from == std::path::Path::new("/bin/old")
12936 ));
12937 }
12938
12939 #[cfg(windows)]
12940 #[test]
12941 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
12942 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
12943 let configured = dir.join("module.exe");
12944 std::fs::write(&configured, b"executable path fixture").unwrap();
12945 let spawned = std::fs::canonicalize(&configured).unwrap();
12946 assert_ne!(
12947 configured, spawned,
12948 "fixture must exercise different path spellings"
12949 );
12950 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
12951 let lower_case = configured.to_string_lossy().to_lowercase();
12952 for spelling in [
12953 &configured,
12954 Path::new(&forward_slashes),
12955 Path::new(&lower_case),
12956 ] {
12957 let verdict = reload_verdict(
12958 spelling,
12959 Some(&spawned),
12960 subc_control::RunningImageAgreement::Unavailable {
12961 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12962 },
12963 );
12964 assert_eq!(
12965 verdict.path,
12966 ReloadPathAgreement::Match,
12967 "configured {spelling:?}, spawned {spawned:?}"
12968 );
12969 }
12970 }
12971
12972 #[cfg(windows)]
12973 #[test]
12974 fn windows_reload_path_keeps_different_existing_program_pending() {
12975 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
12976 let old = dir.join("old.exe");
12977 let new = dir.join("new.exe");
12978 std::fs::write(&old, b"same executable content").unwrap();
12979 std::fs::copy(&old, &new).unwrap();
12980 let spawned = std::fs::canonicalize(&old).unwrap();
12981 let image = subc_control::RunningImageAgreement::Unavailable {
12982 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12983 };
12984 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
12985 assert!(
12986 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
12987 if configured == new && spawned_from == spawned)
12988 );
12989 let missing = dir.join("missing.exe");
12990 assert!(matches!(
12991 reload_verdict(&missing, Some(&spawned), image).path,
12992 ReloadPathAgreement::Mismatch { .. }
12993 ));
12994 }
12995
12996 #[test]
12997 fn reload_verdict_detects_replaced_image_at_same_path() {
12998 let image = subc_control::RunningImageAgreement::Mismatch {
12999 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
13000 digest: "old".into(),
13001 },
13002 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
13003 digest: "new".into(),
13004 },
13005 };
13006 let verdict = reload_verdict(
13007 std::path::Path::new("/bin/same"),
13008 Some(std::path::Path::new("/bin/same")),
13009 image.clone(),
13010 );
13011 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13012 assert_eq!(verdict.image, image);
13013 }
13014
13015 #[test]
13016 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
13017 let image = subc_control::RunningImageAgreement::Unavailable {
13018 reason: subc_control::RunningImageUnavailableReason::NotRunning,
13019 };
13020 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
13021 assert_eq!(
13022 verdict.path,
13023 subc_control::ReloadPathAgreement::Unavailable {
13024 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
13025 }
13026 );
13027 assert_eq!(verdict.image, image);
13028
13029 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
13030 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13031 };
13032 let verdict = reload_verdict(
13033 std::path::Path::new("/bin/same"),
13034 Some(std::path::Path::new("/bin/same")),
13035 unconfirmed.clone(),
13036 );
13037 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13038 assert_eq!(verdict.image, unconfirmed);
13039 }
13040
13041 #[test]
13042 fn reload_verdict_preserves_each_image_unavailability_reason() {
13043 use subc_control::RunningImageUnavailableReason as Reason;
13044
13045 for reason in [
13046 Reason::NotRunning,
13047 Reason::UnsupportedPlatform,
13048 Reason::RunningExecutableUnreadable,
13049 Reason::SpawnedPathUnreadable,
13050 Reason::HashFailed,
13051 Reason::ProcessIdentityUnconfirmed,
13052 Reason::Unknown("future_probe_reason".to_string()),
13053 ] {
13054 let image = subc_control::RunningImageAgreement::Unavailable {
13055 reason: reason.clone(),
13056 };
13057 let verdict = reload_verdict(
13058 std::path::Path::new("/bin/same"),
13059 Some(std::path::Path::new("/bin/same")),
13060 image.clone(),
13061 );
13062 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13063 assert_eq!(verdict.image, image, "{reason:?}");
13064 }
13065 }
13066
13067 #[tokio::test]
13068 async fn malformed_control_bodies_return_invalid_control_body() {
13069 let handler = ControlHandler::default();
13070 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
13071
13072 for (corr, body) in [
13073 (56, br#"{"route_channel":1}"#.as_slice()),
13074 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
13075 (
13076 58,
13077 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
13078 ),
13079 ] {
13080 let request = Frame::build(
13081 FrameType::Request,
13082 control_flags(),
13083 0,
13084 0,
13085 corr,
13086 body.to_vec(),
13087 )
13088 .unwrap();
13089 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13090
13091 assert_eq!(response.len(), 1);
13092 assert_eq!(response[0].header.ty, FrameType::Error);
13093 assert_eq!(response[0].header.corr, corr);
13094 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
13095 }
13096 }
13097
13098 #[tokio::test]
13099 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
13100 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13101 let registry = Arc::new(Registry::default());
13102 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
13103 let router = Router::with_control_handler(Arc::clone(&control));
13104 let connection = router.begin_connection();
13105 let (ctx, mut rx) = route_ctx(connection.id());
13106
13107 router
13108 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
13109 .await
13110 .unwrap();
13111 let response = rx.recv().await.unwrap();
13112 let ack = parse_ack(&response);
13113 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13114 let channel = 1;
13115
13116 let goodbye =
13117 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
13118 router.route_for_connection(&ctx, goodbye).await.unwrap();
13119 assert!(rx.try_recv().is_err());
13120 assert!(registry.get_module("aft").unwrap().is_none());
13121
13122 router
13123 .route_for_connection(&ctx, channel_request(channel, 13))
13124 .await
13125 .unwrap();
13126 let error_frame = rx.recv().await.unwrap();
13127 assert_eq!(error_frame.header.ty, FrameType::Error);
13128 assert_eq!(error_frame.header.channel, channel);
13129 let captured = crate::router::test_log::captured_logs(&logs);
13130 assert_eq!(
13131 captured
13132 .lines()
13133 .filter(|line| {
13134 line.contains("module_id=aft")
13135 && line.contains("reason=explicit_goodbye")
13136 && line.contains("module registration ended")
13137 })
13138 .count(),
13139 1,
13140 "unexpected GOODBYE registry log: {captured}"
13141 );
13142 }
13143
13144 #[tokio::test]
13145 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
13146 let registry = Arc::new(Registry::default());
13147 let handler = ControlHandler::new(registry).with_capability_config(
13148 [("prov".to_string(), true), ("cons".to_string(), true)],
13149 BTreeMap::new(),
13150 );
13151 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
13152 register_capability_manifest(
13153 &handler,
13154 &provider_ctx,
13155 &mut provider_rx,
13156 capability_manifest("prov", &["thing/v1"], &[]),
13157 1,
13158 )
13159 .await;
13160 let mut consumer = capability_manifest("cons", &[], &[]);
13161 consumer.capabilities.as_mut().unwrap().requires.push(
13162 subc_protocol::manifest::CapabilityRequirement {
13163 capability: "thing/v1".to_string(),
13164 need: subc_protocol::manifest::CapabilityNeed::Required,
13165 },
13166 );
13167 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
13168 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
13169 assert_eq!(
13170 handler.capability_evaluator.verdict("cons", "thing/v1"),
13171 Some(CapabilityVerdict::Provided)
13172 );
13173 let (mut client_rx, _) = open_route_for_capability_test(
13174 &handler,
13175 &provider_ctx,
13176 &mut provider_rx,
13177 703,
13178 3,
13179 "prov",
13180 None,
13181 )
13182 .await;
13183 let goodbye =
13184 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13185 handler
13186 .handle_control_frame(&provider_ctx, goodbye)
13187 .await
13188 .unwrap();
13189 assert_eq!(
13190 handler.capability_evaluator.verdict("cons", "thing/v1"),
13191 Some(CapabilityVerdict::NeverProvided)
13192 );
13193 let closed = client_rx
13194 .try_recv()
13195 .expect("GOODBYE pushes route.closed before route GOODBYE");
13196 assert!(
13197 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
13198 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
13199 );
13200 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
13201 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
13202 }
13203
13204 struct RouteClosedFixture {
13205 handler: ControlHandler,
13206 #[cfg(unix)]
13207 supervisor: Supervisor,
13208 module_ctx: RouteCtx,
13209 #[cfg(unix)]
13210 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13211 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13212 route: Arc<crate::forwarding::RouteBinding>,
13213 }
13214
13215 async fn route_closed_fixture() -> RouteClosedFixture {
13216 let registry = Arc::new(Registry::default());
13217 let handler = ControlHandler::new(Arc::clone(®istry));
13218 #[cfg(unix)]
13219 let supervisor = Supervisor::new(registry, RestartPolicy::default())
13220 .with_forwarding(Arc::clone(&handler.forwarding));
13221 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
13222 register_capability_manifest(
13223 &handler,
13224 &module_ctx,
13225 &mut module_rx,
13226 capability_manifest("prov", &[], &[]),
13227 1,
13228 )
13229 .await;
13230 let (client_rx, bind) = open_route_for_capability_test(
13231 &handler,
13232 &module_ctx,
13233 &mut module_rx,
13234 702,
13235 2,
13236 "prov",
13237 None,
13238 )
13239 .await;
13240 let ModuleControlRequest::RouteBind {
13241 route_channel: channel,
13242 epoch: route_epoch,
13243 ..
13244 } = bind
13245 else {
13246 panic!("expected route.bind");
13247 };
13248 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
13249 handler
13250 .forwarding
13251 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
13252 .unwrap()
13253 else {
13254 panic!("the acknowledged route must be bound");
13255 };
13256 RouteClosedFixture {
13257 handler,
13258 #[cfg(unix)]
13259 supervisor,
13260 module_ctx,
13261 #[cfg(unix)]
13262 module_rx,
13263 client_rx,
13264 route,
13265 }
13266 }
13267
13268 impl RouteClosedFixture {
13269 #[cfg(unix)]
13270 async fn cut(&mut self) {
13271 let notices = async {
13272 let mut draining = self.module_rx.recv().await.unwrap();
13273 assert!(matches!(
13274 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
13275 &draining.body
13276 )
13277 .unwrap(),
13278 subc_protocol::session::ModuleControlCommand::Draining {
13279 reason: RouteCloseReason::Restart,
13280 ..
13281 }
13282 ));
13283 draining.flushed.take().unwrap().send(()).unwrap();
13284 let mut closing = self.client_rx.recv().await.unwrap();
13285 assert!(matches!(
13286 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
13287 ClientControlPush::RouteClosing {
13288 reason: RouteCloseReason::Restart,
13289 ..
13290 }
13291 ));
13292 closing.flushed.take().unwrap().send(()).unwrap();
13293 };
13294 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
13295 result.unwrap();
13296 }
13297
13298 fn disconnect(&mut self) -> ClientControlPush {
13299 self.handler
13300 .cleanup_connection(self.module_ctx.connection_id)
13301 .unwrap();
13302 let closed = self.client_rx.try_recv().unwrap();
13303 assert_eq!(closed.header.ty, FrameType::Push);
13304 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
13305 assert!(matches!(
13306 &push,
13307 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
13308 if channels == &[self.route.client_channel]
13309 ));
13310 assert_eq!(
13311 self.client_rx.try_recv().unwrap().header.ty,
13312 FrameType::Goodbye
13313 );
13314 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
13315 push
13316 }
13317 }
13318
13319 #[cfg(unix)]
13320 #[tokio::test(start_paused = true)]
13321 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
13322 let mut fixture = route_closed_fixture().await;
13323 fixture.cut().await;
13324 assert!(matches!(
13325 fixture.disconnect(),
13326 ClientControlPush::RouteClosed {
13327 reason: RouteCloseReason::Restart,
13328 drained: true,
13329 ..
13330 }
13331 ));
13332 }
13333
13334 #[cfg(unix)]
13335 #[tokio::test(start_paused = true)]
13336 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
13337 let mut fixture = route_closed_fixture().await;
13338 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
13339 let started = tokio::time::Instant::now();
13340 fixture.cut().await;
13341 assert!(started.elapsed() >= Duration::from_secs(2));
13342 assert_eq!(fixture.route.flow.in_flight(), 1);
13343 assert!(matches!(
13344 fixture.disconnect(),
13345 ClientControlPush::RouteClosed {
13346 reason: RouteCloseReason::Restart,
13347 drained: false,
13348 ..
13349 }
13350 ));
13351 }
13352
13353 #[tokio::test]
13354 async fn module_connection_crash_reports_crash_and_not_drained() {
13355 let mut fixture = route_closed_fixture().await;
13356 assert!(matches!(
13360 fixture.disconnect(),
13361 ClientControlPush::RouteClosed {
13362 reason: RouteCloseReason::Crash,
13363 drained: false,
13364 ..
13365 }
13366 ));
13367 }
13368
13369 #[tokio::test]
13370 async fn dropping_router_connection_releases_registration() {
13371 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13372 let registry = Arc::new(Registry::default());
13373 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
13374 let router = Router::with_control_handler(Arc::clone(&control));
13375 let connection = router.begin_connection();
13376 let connection_id = connection.id();
13377 let (ctx, mut rx) = route_ctx(connection_id);
13378
13379 router
13380 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
13381 .await
13382 .unwrap();
13383 let response = rx.recv().await.unwrap();
13384 let ack = parse_ack(&response);
13385 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13386 assert!(registry.get_module("aft").unwrap().is_some());
13387
13388 drop(connection);
13389
13390 assert!(registry.get_module("aft").unwrap().is_none());
13391 assert_eq!(registry.active_registration_count().unwrap(), 0);
13392
13393 control
13394 .cleanup_connection(ConnectionId::new(u64::MAX))
13395 .unwrap();
13396 let captured = crate::router::test_log::captured_logs(&logs);
13397 println!("captured registry lifecycle logs:\n{captured}");
13398 let events: Vec<_> = captured
13399 .lines()
13400 .filter(|line| {
13401 line.contains("module registered module_id=aft ")
13402 || line.contains("module registration ended")
13403 })
13404 .collect();
13405 assert_eq!(
13406 events.len(),
13407 2,
13408 "unexpected registry lifecycle logs: {captured}"
13409 );
13410 assert!(events[0].contains("module registered module_id=aft "));
13411 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
13412 assert!(events[1].contains(&format!(
13413 "module_id=aft connection_id={}",
13414 connection_id.get()
13415 )));
13416 assert!(events[1].contains("reason=connection_closed"));
13417 assert!(events[1].contains("module registration ended"));
13418 assert_eq!(
13419 captured
13420 .lines()
13421 .filter(|line| line.contains("module registered module_id=aft "))
13422 .count(),
13423 1,
13424 "legacy registration admission line must appear once: {captured}"
13425 );
13426 }
13427
13428 #[tokio::test]
13429 async fn hello_registration_keeps_legacy_module_registered_line_once() {
13430 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13431 let registry = Arc::new(Registry::default());
13432 let control = ControlHandler::new(Arc::clone(®istry));
13433 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
13434 hello_via_sink(
13435 &control,
13436 &ctx,
13437 &mut rx,
13438 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
13439 )
13440 .await;
13441
13442 let captured = crate::router::test_log::captured_logs(&logs);
13443 let admissions: Vec<_> = captured
13444 .lines()
13445 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
13446 .collect();
13447 assert_eq!(
13448 admissions.len(),
13449 1,
13450 "legacy admission line must remain exactly once: {captured}"
13451 );
13452 assert!(admissions[0].contains("routable_provider=true"));
13453 assert!(admissions[0].contains("connection_id=777"));
13454 }
13455
13456 fn capability_manifest(
13457 module_id: &str,
13458 provides: &[&str],
13459 must_never_reach: &[&str],
13460 ) -> ModuleManifest {
13461 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
13462 manifest.capabilities = Some(CapabilityDeclarations {
13463 provides: provides
13464 .iter()
13465 .map(|capability| (*capability).to_string())
13466 .collect(),
13467 requires: Vec::new(),
13468 must_never_reach: must_never_reach
13469 .iter()
13470 .map(|capability| (*capability).to_string())
13471 .collect(),
13472 });
13473 manifest
13474 }
13475
13476 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
13477 Frame::build(
13478 FrameType::Hello,
13479 control_flags(),
13480 0,
13481 0,
13482 corr,
13483 serde_json::to_vec(&ModuleHelloBody {
13484 protocol_ver: manifest.protocol_ver,
13485 manifest,
13486 control_ops: None,
13487 launch_nonce: None,
13488 })
13489 .expect("capability test HELLO serializes"),
13490 )
13491 .expect("capability test HELLO frame builds")
13492 }
13493
13494 fn catalog_update_with_capabilities_frame(
13495 corr: u64,
13496 capabilities: CapabilityDeclarations,
13497 ) -> Frame {
13498 Frame::build(
13499 FrameType::Request,
13500 control_flags(),
13501 0,
13502 0,
13503 corr,
13504 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
13505 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
13506 capabilities: Some(capabilities),
13507 ready: None,
13508 })
13509 .expect("capability catalog.update serializes"),
13510 )
13511 .expect("capability catalog.update frame builds")
13512 }
13513
13514 async fn register_capability_manifest(
13515 handler: &ControlHandler,
13516 ctx: &RouteCtx,
13517 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13518 manifest: ModuleManifest,
13519 corr: u64,
13520 ) {
13521 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
13522 }
13523
13524 async fn open_route_for_capability_test(
13525 handler: &ControlHandler,
13526 target_ctx: &RouteCtx,
13527 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13528 client_connection_id: u64,
13529 corr: u64,
13530 target_module_id: &str,
13531 consumer_identity: Option<ConsumerIdentity>,
13532 ) -> (
13533 mpsc::Receiver<crate::router::OutboundFrame>,
13534 ModuleControlRequest,
13535 ) {
13536 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
13537 let route_handler = handler.clone();
13538 let target_module_id = target_module_id.to_string();
13539 let route_task = tokio::spawn(async move {
13540 route_handler
13541 .handle_control_frame(
13542 &client_ctx,
13543 route_open_frame_with_admission_facts(
13544 corr,
13545 &target_module_id,
13546 unique_project_root("admission-facts"),
13547 consumer_identity,
13548 None,
13549 ),
13550 )
13551 .await
13552 .expect("capability test route.open succeeds")
13553 });
13554 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
13555 .await
13556 .expect("capability test route.open must reach route.bind")
13557 .expect("target control receiver stays open");
13558 let bind_request: ModuleControlRequest =
13559 serde_json::from_slice(&bind.body).expect("route.bind decodes");
13560 handler
13561 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
13562 .await
13563 .expect("capability test route.bind ACK succeeds");
13564 assert!(route_task.await.expect("route.open task joins").is_empty());
13565 let opened = client_rx
13566 .recv()
13567 .await
13568 .expect("successful route.open publishes a response");
13569 assert!(matches!(
13570 serde_json::from_slice::<ClientControlResponse>(&opened.body),
13571 Ok(ClientControlResponse::RouteOpen { .. })
13572 ));
13573 (client_rx, bind_request)
13574 }
13575
13576 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
13577 assert_eq!(frame.header.ty, FrameType::Push);
13578 assert_eq!(frame.header.channel, 0);
13579 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
13580 .expect("route.closed control push decodes");
13581 let ClientControlPush::RouteClosed { channels, .. } = &push else {
13582 panic!("expected route.closed");
13583 };
13584 assert_eq!(channels.len(), 1, "exactly one violating route closed");
13585 let channels = channels.clone();
13586 assert_eq!(
13587 push,
13588 ClientControlPush::RouteClosed {
13589 module_id: target_module_id.to_string(),
13590 channels,
13591 reason: RouteCloseReason::CapabilityDenied,
13592 drained: false,
13593 abandoned: 0,
13594 excluded_subscriptions: 0,
13595 terminal: Some(false),
13596 }
13597 );
13598 }
13599
13600 #[tokio::test]
13601 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
13602 let registry = Arc::new(Registry::default());
13603 let forwarding = Arc::new(ForwardingTable::default());
13604 let supervisor = SupervisorHandle::new();
13605 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13606 let handler =
13607 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13608 .with_supervisor(supervisor);
13609 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
13610 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
13611 register_capability_manifest(
13612 &handler,
13613 &target_ctx,
13614 &mut target_rx,
13615 capability_manifest("target", &["credentials-provider/v1"], &[]),
13616 1,
13617 )
13618 .await;
13619 register_capability_manifest(
13620 &handler,
13621 &opener_ctx,
13622 &mut opener_rx,
13623 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13624 2,
13625 )
13626 .await;
13627
13628 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
13629 let replies = handler
13630 .handle_control_frame(
13631 &client_ctx,
13632 route_open_frame_with_admission_facts(
13633 3,
13634 "target",
13635 unique_project_root("admission-facts"),
13636 Some(ConsumerIdentity {
13637 module_id: "opener".to_string(),
13638 launch_nonce: "opener-nonce".to_string(),
13639 }),
13640 None,
13641 ),
13642 )
13643 .await
13644 .expect("denied route.open returns a typed frame");
13645 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
13646 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13647 assert!(
13648 target_rx.try_recv().is_err(),
13649 "forbidden route.open must not relay route.bind"
13650 );
13651 }
13652
13653 #[tokio::test]
13654 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
13655 let registry = Arc::new(Registry::default());
13656 let forwarding = Arc::new(ForwardingTable::default());
13657 let supervisor = SupervisorHandle::new();
13658 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13659 let handler =
13660 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13661 .with_supervisor(supervisor);
13662 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
13663 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
13664 register_capability_manifest(
13665 &handler,
13666 &target_ctx,
13667 &mut target_rx,
13668 capability_manifest("target", &["credentials-provider/v1"], &[]),
13669 1,
13670 )
13671 .await;
13672 register_capability_manifest(
13673 &handler,
13674 &old_opener_ctx,
13675 &mut old_opener_rx,
13676 capability_manifest("opener", &[], &[]),
13677 2,
13678 )
13679 .await;
13680 let (mut client_rx, _) = open_route_for_capability_test(
13681 &handler,
13682 &target_ctx,
13683 &mut target_rx,
13684 712,
13685 3,
13686 "target",
13687 Some(ConsumerIdentity {
13688 module_id: "opener".to_string(),
13689 launch_nonce: "opener-nonce".to_string(),
13690 }),
13691 )
13692 .await;
13693 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13694
13695 handler
13696 .cleanup_connection(old_opener_ctx.connection_id)
13697 .expect("old opener registration cleans up");
13698 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
13699 register_capability_manifest(
13700 &handler,
13701 &new_opener_ctx,
13702 &mut new_opener_rx,
13703 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13704 4,
13705 )
13706 .await;
13707
13708 assert_capability_denied_push(
13709 client_rx
13710 .try_recv()
13711 .expect("HELLO deny addition must emit route.closed")
13712 .frame,
13713 "target",
13714 );
13715 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13716 assert!(matches!(
13717 target_rx.try_recv(),
13718 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
13719 ));
13720 }
13721
13722 #[tokio::test]
13723 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
13724 let registry = Arc::new(Registry::default());
13725 let forwarding = Arc::new(ForwardingTable::default());
13726 let supervisor = SupervisorHandle::new();
13727 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13728 let handler =
13729 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13730 .with_supervisor(supervisor);
13731 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
13732 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
13733 register_capability_manifest(
13734 &handler,
13735 &target_ctx,
13736 &mut target_rx,
13737 capability_manifest("target", &[], &[]),
13738 1,
13739 )
13740 .await;
13741 register_capability_manifest(
13742 &handler,
13743 &opener_ctx,
13744 &mut opener_rx,
13745 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13746 2,
13747 )
13748 .await;
13749 let (mut client_rx, _) = open_route_for_capability_test(
13750 &handler,
13751 &target_ctx,
13752 &mut target_rx,
13753 722,
13754 3,
13755 "target",
13756 Some(ConsumerIdentity {
13757 module_id: "opener".to_string(),
13758 launch_nonce: "opener-nonce".to_string(),
13759 }),
13760 )
13761 .await;
13762 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13763
13764 let replies = handler
13765 .handle_control_frame(
13766 &target_ctx,
13767 catalog_update_with_capabilities_frame(
13768 4,
13769 CapabilityDeclarations {
13770 provides: vec!["credentials-provider/v1".to_string()],
13771 requires: Vec::new(),
13772 must_never_reach: Vec::new(),
13773 },
13774 ),
13775 )
13776 .await
13777 .expect("claim catalog.update succeeds");
13778 assert!(matches!(
13779 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
13780 Ok(ModuleControlResponseToModule::CatalogUpdate {})
13781 ));
13782 assert_capability_denied_push(
13783 client_rx
13784 .try_recv()
13785 .expect("claim addition must emit route.closed")
13786 .frame,
13787 "target",
13788 );
13789 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13790 assert!(matches!(
13791 target_rx.try_recv(),
13792 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
13793 ));
13794 }
13795
13796 #[tokio::test]
13797 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
13798 let registry = Arc::new(Registry::default());
13799 let forwarding = Arc::new(ForwardingTable::default());
13800 let supervisor = SupervisorHandle::new();
13801 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13802 let handler =
13803 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13804 .with_supervisor(supervisor);
13805 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
13806 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
13807 register_capability_manifest(
13808 &handler,
13809 &target_ctx,
13810 &mut target_rx,
13811 capability_manifest("target", &["credentials-provider/v1"], &[]),
13812 1,
13813 )
13814 .await;
13815 register_capability_manifest(
13816 &handler,
13817 &opener_ctx,
13818 &mut opener_rx,
13819 capability_manifest("opener", &[], &[]),
13820 2,
13821 )
13822 .await;
13823 let (mut client_rx, _) = open_route_for_capability_test(
13824 &handler,
13825 &target_ctx,
13826 &mut target_rx,
13827 732,
13828 3,
13829 "target",
13830 Some(ConsumerIdentity {
13831 module_id: "opener".to_string(),
13832 launch_nonce: "opener-nonce".to_string(),
13833 }),
13834 )
13835 .await;
13836 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13837
13838 handler
13839 .handle_control_frame(
13840 &target_ctx,
13841 catalog_update_with_capabilities_frame(
13842 4,
13843 CapabilityDeclarations {
13844 provides: Vec::new(),
13845 requires: Vec::new(),
13846 must_never_reach: Vec::new(),
13847 },
13848 ),
13849 )
13850 .await
13851 .expect("claim removal catalog.update succeeds");
13852 assert_eq!(
13853 forwarding.active_binding_count().unwrap(),
13854 1,
13855 "removing an attested target claim must leave the route census unchanged"
13856 );
13857 assert!(
13858 client_rx.try_recv().is_err(),
13859 "claim removal must not emit route.closed capability_denied"
13860 );
13861 assert!(
13862 target_rx.try_recv().is_err(),
13863 "claim removal must not send the target a route GOODBYE"
13864 );
13865 }
13866
13867 #[tokio::test]
13870 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
13871 let registry = Arc::new(Registry::default());
13872 let forwarding = Arc::new(ForwardingTable::default());
13873 let supervisor = SupervisorHandle::new();
13874 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13875 let handler =
13876 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13877 .with_supervisor(supervisor);
13878 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
13879 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
13880 register_capability_manifest(
13881 &handler,
13882 &target_ctx,
13883 &mut target_rx,
13884 capability_manifest("target", &["credentials-provider/v1"], &[]),
13885 1,
13886 )
13887 .await;
13888 register_capability_manifest(
13889 &handler,
13890 &opener_ctx,
13891 &mut opener_rx,
13892 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13893 2,
13894 )
13895 .await;
13896
13897 let (_client_rx, bind) = open_route_for_capability_test(
13898 &handler,
13899 &target_ctx,
13900 &mut target_rx,
13901 742,
13902 3,
13903 "target",
13904 None,
13905 )
13906 .await;
13907 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
13908 panic!("direct scope-honesty route must bind");
13909 };
13910 assert_eq!(principal, Some(Principal::Direct));
13911 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13912 }
13913
13914 #[tokio::test]
13917 async fn must_never_reach_self_route_is_capability_forbidden() {
13918 let registry = Arc::new(Registry::default());
13919 let forwarding = Arc::new(ForwardingTable::default());
13920 let supervisor = SupervisorHandle::new();
13921 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
13922 let handler =
13923 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13924 .with_supervisor(supervisor);
13925 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
13926 register_capability_manifest(
13927 &handler,
13928 &self_ctx,
13929 &mut self_rx,
13930 capability_manifest(
13931 "self-provider",
13932 &["credentials-provider/v1"],
13933 &["credentials-provider/v1"],
13934 ),
13935 1,
13936 )
13937 .await;
13938
13939 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
13940 let replies = handler
13941 .handle_control_frame(
13942 &client_ctx,
13943 route_open_frame_with_admission_facts(
13944 2,
13945 "self-provider",
13946 unique_project_root("admission-facts"),
13947 Some(ConsumerIdentity {
13948 module_id: "self-provider".to_string(),
13949 launch_nonce: "self-nonce".to_string(),
13950 }),
13951 None,
13952 ),
13953 )
13954 .await
13955 .expect("self-route refusal returns a typed frame");
13956 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
13957 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13958 assert!(
13959 self_rx.try_recv().is_err(),
13960 "self denial must not relay route.bind"
13961 );
13962 }
13963
13964 #[test]
13965 fn unsupported_channel_zero_frame_returns_error() {
13966 let handler = ControlHandler::default();
13967 let request = Frame::build(
13968 FrameType::Request,
13969 control_flags(),
13970 0,
13971 0,
13972 21,
13973 b"opaque".to_vec(),
13974 )
13975 .unwrap();
13976
13977 let response = handler
13978 .handle_control(ConnectionId::new(1), request)
13979 .unwrap();
13980
13981 assert_eq!(response[0].header.ty, FrameType::Error);
13982 assert_eq!(
13983 parse_error(&response[0])["code"],
13984 "unsupported_control_frame"
13985 );
13986 }
13987
13988 mod swap {
13993 use super::*;
13994
13995 const INCUMBENT: ConnectionId = ConnectionId::new(30);
13996 const CANDIDATE: ConnectionId = ConnectionId::new(40);
13997
13998 struct Swap {
13999 registry: Arc<Registry>,
14000 forwarding: Arc<ForwardingTable>,
14001 handler: ControlHandler,
14002 incumbent_ctx: RouteCtx,
14003 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14004 candidate_ctx: RouteCtx,
14005 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14006 }
14007
14008 async fn swap_with_incumbent() -> Swap {
14009 let registry = Arc::new(Registry::default());
14010 let forwarding = Arc::new(ForwardingTable::default());
14011 let handler =
14012 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14013 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
14014 hello_via_sink(
14015 &handler,
14016 &incumbent_ctx,
14017 &mut incumbent_rx,
14018 hello_frame("aft", PROTOCOL_VERSION, 7),
14019 )
14020 .await;
14021 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
14022 Swap {
14023 registry,
14024 forwarding,
14025 handler,
14026 incumbent_ctx,
14027 incumbent_rx,
14028 candidate_ctx,
14029 candidate_rx,
14030 }
14031 }
14032
14033 fn register_candidate(swap: &Swap, ready: Option<bool>) {
14034 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
14035 candidate_manifest.ready = ready;
14036 let registration = swap
14037 .registry
14038 .register_candidate_with_control_ops(
14039 candidate_manifest,
14040 PROTOCOL_VERSION,
14041 CANDIDATE,
14042 module_baseline_control_ops(),
14043 )
14044 .unwrap();
14045 swap.forwarding
14046 .register_candidate_module_connection(
14047 CANDIDATE,
14048 "aft".to_string(),
14049 PROTOCOL_VERSION,
14050 manifest_concurrency(®istration.manifest),
14051 swap.candidate_ctx.egress.clone(),
14052 )
14053 .unwrap();
14054 }
14055
14056 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
14057 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
14058 swap.registry.promote_candidate("aft").unwrap().unwrap();
14059 cutover.incumbent.unwrap()
14060 }
14061
14062 #[tokio::test]
14071 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
14072 let mut swap = swap_with_incumbent().await;
14073 let hold = Arc::new(std::sync::Barrier::new(2));
14074 let handler = swap
14075 .handler
14076 .clone()
14077 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
14078 handler
14079 .supervisor
14080 .open_swap("aft", "candidate-nonce".to_string());
14081 let hello = {
14082 let handler = handler.clone();
14083 let candidate_ctx = swap.candidate_ctx.clone();
14084 std::thread::spawn(move || {
14085 handler
14086 .handle_hello(
14087 candidate_ctx.connection_id,
14088 Some(candidate_ctx.egress.clone()),
14089 hello_frame_with_nonce(
14090 "aft",
14091 PROTOCOL_VERSION,
14092 8,
14093 Some("candidate-nonce"),
14094 ),
14095 )
14096 .unwrap()
14097 })
14098 };
14099
14100 hold.wait();
14101 assert!(swap
14104 .registry
14105 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
14106 .unwrap()
14107 .is_some());
14108 assert!(swap
14109 .registry
14110 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
14111 .unwrap()
14112 .is_none());
14113 assert!(swap
14114 .forwarding
14115 .module_endpoint_for_connection(CANDIDATE)
14116 .unwrap()
14117 .is_none());
14118
14119 hold.wait();
14120 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
14121 assert_eq!(
14122 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
14123 FrameType::HelloAck
14124 );
14125 let published = swap
14126 .registry
14127 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
14128 .unwrap()
14129 .expect("candidate is published once its HELLO completes");
14130 assert_eq!(published.connection_id, CANDIDATE);
14131 assert!(swap
14132 .forwarding
14133 .module_endpoint_for_connection(CANDIDATE)
14134 .unwrap()
14135 .is_some());
14136 }
14137
14138 fn keyed_total(counters: &Value, key: &str) -> u64 {
14139 counters[key]
14140 .as_object()
14141 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
14142 .unwrap_or(0)
14143 }
14144
14145 #[tokio::test]
14146 async fn replacement_logs_old_end_and_new_admission_once() {
14147 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14148 let mut swap = swap_with_incumbent().await;
14149 swap.handler
14150 .supervisor
14151 .open_swap("aft", "candidate-nonce".to_string());
14152 hello_via_sink(
14153 &swap.handler,
14154 &swap.candidate_ctx,
14155 &mut swap.candidate_rx,
14156 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
14157 )
14158 .await;
14159 cutover(&swap);
14160 swap.handler.cleanup_connection(INCUMBENT).unwrap();
14161
14162 let captured = crate::router::test_log::captured_logs(&logs);
14163 println!("captured replacement registry logs:\n{captured}");
14164 let new_admissions: Vec<_> = captured
14165 .lines()
14166 .filter(|line| {
14167 line.contains("connection_id=40")
14168 && line.contains("swap candidate registered; not routable until cutover")
14169 })
14170 .collect();
14171 assert_eq!(
14172 new_admissions.len(),
14173 1,
14174 "unexpected admission logs: {captured}"
14175 );
14176 assert!(new_admissions[0].contains("module_id=aft"));
14177 assert!(new_admissions[0].contains("connection_id=40"));
14178 assert!(
14179 new_admissions[0].contains("swap candidate registered; not routable until cutover")
14180 );
14181 assert!(new_admissions[0].contains("ready=true"));
14182
14183 let old_admissions: Vec<_> = captured
14184 .lines()
14185 .filter(|line| {
14186 line.contains("module registered module_id=aft ")
14187 && line.contains("connection_id=30")
14188 })
14189 .collect();
14190 assert_eq!(
14191 old_admissions.len(),
14192 1,
14193 "unexpected incumbent admission: {captured}"
14194 );
14195
14196 let old_ends: Vec<_> = captured
14197 .lines()
14198 .filter(|line| {
14199 line.contains("connection_id=30") && line.contains("module registration ended")
14200 })
14201 .collect();
14202 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
14203 assert!(old_ends[0].contains("module_id=aft"));
14204 assert!(old_ends[0].contains("connection_id=30"));
14205 assert!(old_ends[0].contains("reason=replaced"));
14206 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
14207 assert_eq!(
14208 captured
14209 .lines()
14210 .filter(|line| line.contains("module registration promoted"))
14211 .count(),
14212 1,
14213 "unexpected promotion logs: {captured}"
14214 );
14215 }
14216
14217 #[tokio::test]
14223 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
14224 let mut swap = swap_with_incumbent().await;
14225 let handler = swap.handler.clone();
14226
14227 let cotenant = ConnectionId::new(31);
14229 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
14230 let (cotenant_task, cotenant_bind) = relay_route_open(
14231 &handler,
14232 cotenant,
14233 &cotenant_ctx.egress,
14234 &mut swap.incumbent_rx,
14235 100,
14236 "aft",
14237 "swap-cotenant",
14238 )
14239 .await;
14240 handler
14241 .handle_control_frame(
14242 &swap.incumbent_ctx,
14243 route_bind_ack(cotenant_bind.header.corr),
14244 )
14245 .await
14246 .unwrap();
14247 assert!(cotenant_task.await.unwrap().is_empty());
14248 let (cotenant_channel, cotenant_epoch) =
14249 published_route(&cotenant_rx.recv().await.unwrap());
14250
14251 let caller = ConnectionId::new(32);
14253 let (caller_ctx, mut caller_rx) = route_ctx(caller);
14254 let (caller_task, caller_bind) = relay_route_open(
14255 &handler,
14256 caller,
14257 &caller_ctx.egress,
14258 &mut swap.incumbent_rx,
14259 101,
14260 "aft",
14261 "swap-caller",
14262 )
14263 .await;
14264 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
14265
14266 register_candidate(&swap, None);
14267 cutover(&swap);
14268
14269 let ack = handler
14271 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
14272 .await;
14273 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
14274 if module_loop_error.is_some() {
14275 handler.cleanup_connection(INCUMBENT).unwrap();
14278 }
14279
14280 assert!(
14282 cotenant_rx.try_recv().is_err(),
14283 "the co-tenant route on the incumbent was torn down by one late ack: \
14284 {module_loop_error:?}"
14285 );
14286 assert!(matches!(
14287 swap.forwarding
14288 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
14289 .unwrap(),
14290 DataRoute::Client(DataRouteState::Bound(_))
14291 ));
14292 assert_eq!(module_loop_error, None);
14293 assert!(swap
14294 .registry
14295 .get_module_by_connection(INCUMBENT)
14296 .unwrap()
14297 .is_some());
14298
14299 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
14301 .await
14302 .expect("the incumbent is told to drop the abandoned binding")
14303 .unwrap()
14304 .frame;
14305 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14306 assert_eq!(goodbye.header.channel, abandoned_channel);
14307 assert_eq!(goodbye.header.epoch, abandoned_epoch);
14308 assert!(swap.incumbent_rx.try_recv().is_err());
14309
14310 let response = caller_task.await.unwrap();
14312 assert_eq!(response.len(), 1);
14313 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14314 assert!(caller_rx.try_recv().is_err());
14315
14316 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
14320 let counters = handler.counters().snapshot();
14321 assert_eq!(
14322 keyed_total(&counters, "route_open_accepted_by_principal"),
14323 1
14324 );
14325 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
14326 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
14327 }
14328
14329 #[tokio::test]
14333 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
14334 let mut swap = swap_with_incumbent().await;
14335 register_candidate(&swap, None);
14336 let incumbent = cutover(&swap);
14337 swap.forwarding
14338 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
14339 .unwrap()
14340 .expect("the incumbent is still registered");
14341
14342 let client = ConnectionId::new(33);
14343 let (client_ctx, mut client_rx) = route_ctx(client);
14344 let route_handler = swap.handler.clone();
14345 let open_ctx = RouteCtx {
14346 connection_id: client,
14347 egress: client_ctx.egress.clone(),
14348 };
14349 let mut route_task = tokio::spawn(async move {
14350 route_handler
14351 .handle_control_frame(
14352 &open_ctx,
14353 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
14354 )
14355 .await
14356 .unwrap()
14357 });
14358 let bind = tokio::select! {
14359 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
14360 response = &mut route_task => {
14361 let response = response.unwrap();
14362 panic!(
14363 "post-cutover route.open was refused instead of relayed to the candidate: {}",
14364 parse_error(&response[0])["code"]
14365 );
14366 }
14367 };
14368 swap.handler
14369 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
14370 .await
14371 .unwrap();
14372 assert!(route_task.await.unwrap().is_empty());
14373 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
14374 match swap
14375 .forwarding
14376 .lookup_data_route(client, channel, epoch)
14377 .unwrap()
14378 {
14379 DataRoute::Client(DataRouteState::Bound(route)) => {
14380 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
14381 }
14382 other => panic!("expected a bound route on the candidate, got {other:?}"),
14383 }
14384 assert!(swap.incumbent_rx.try_recv().is_err());
14385 }
14386
14387 #[tokio::test]
14392 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
14393 let swap = swap_with_incumbent().await;
14394 register_candidate(&swap, Some(false));
14395 let update = Frame::build(
14396 FrameType::Request,
14397 control_flags(),
14398 0,
14399 0,
14400 55,
14401 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14402 provides: manifest("aft", PROTOCOL_VERSION).provides,
14403 capabilities: None,
14404 ready: Some(true),
14405 })
14406 .unwrap(),
14407 )
14408 .unwrap();
14409
14410 let replies = swap
14411 .handler
14412 .handle_control_frame(&swap.candidate_ctx, update)
14413 .await
14414 .unwrap();
14415
14416 assert_eq!(replies.len(), 1);
14417 assert_eq!(
14418 replies[0].header.ty,
14419 FrameType::Response,
14420 "candidate catalog.update was refused: {:?}",
14421 serde_json::from_slice::<Value>(&replies[0].body).ok()
14422 );
14423 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
14424 assert_eq!(
14425 swap.registry
14426 .get_module("aft")
14427 .unwrap()
14428 .unwrap()
14429 .connection_id,
14430 INCUMBENT
14431 );
14432 }
14433 }
14434
14435 mod swap_admission {
14439 use super::*;
14440
14441 const INCUMBENT_NONCE: &str = "incumbent-nonce";
14442 const CANDIDATE_NONCE: &str = "candidate-nonce";
14443
14444 fn handler_with_incumbent(
14445 module_id: &str,
14446 reserved: bool,
14447 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
14448 let registry = Arc::new(Registry::default());
14449 let supervisor = SupervisorHandle::new();
14450 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
14451 if reserved {
14452 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
14453 }
14454 let handler =
14455 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
14456 let incumbent = handler
14457 .handle_control(
14458 ConnectionId::new(1),
14459 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
14460 )
14461 .unwrap();
14462 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
14463 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
14464 (registry, supervisor, handler)
14465 }
14466
14467 #[test]
14473 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
14474 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
14475
14476 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
14477 let replies = handler
14478 .handle_control(
14479 ConnectionId::new(connection),
14480 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
14481 )
14482 .unwrap();
14483 assert_eq!(replies[0].header.ty, FrameType::Error);
14484 assert_eq!(
14485 parse_error(&replies[0])["code"],
14486 "swap_token_invalid",
14487 "nonce {nonce:?}"
14488 );
14489 }
14490 assert!(registry.get_candidate("aft").unwrap().is_none());
14491 assert_eq!(
14492 registry.get_module("aft").unwrap().unwrap().connection_id,
14493 ConnectionId::new(1)
14494 );
14495
14496 let admitted = handler
14499 .handle_control(
14500 ConnectionId::new(4),
14501 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
14502 )
14503 .unwrap();
14504 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
14505 assert_eq!(
14506 registry
14507 .get_candidate("aft")
14508 .unwrap()
14509 .unwrap()
14510 .connection_id,
14511 ConnectionId::new(4)
14512 );
14513 assert_eq!(
14514 registry.get_module("aft").unwrap().unwrap().connection_id,
14515 ConnectionId::new(1),
14516 "the candidate must not take the active slot"
14517 );
14518 let replayed = handler
14519 .handle_control(
14520 ConnectionId::new(5),
14521 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
14522 )
14523 .unwrap();
14524 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
14525
14526 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
14530 let squatter = handler
14531 .handle_control(
14532 ConnectionId::new(6),
14533 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
14534 )
14535 .unwrap();
14536 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
14537 assert!(
14538 registry.get_module("aft").unwrap().is_none(),
14539 "a squatter took the active slot of an id being swapped"
14540 );
14541 }
14542
14543 #[test]
14548 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
14549 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
14550
14551 let replies = handler
14552 .handle_control(
14553 ConnectionId::new(2),
14554 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14555 )
14556 .unwrap();
14557
14558 assert_eq!(
14559 replies[0].header.ty,
14560 FrameType::HelloAck,
14561 "reserved candidate refused: {:?}",
14562 serde_json::from_slice::<Value>(&replies[0].body).ok()
14563 );
14564 assert_eq!(
14565 registry
14566 .get_candidate("vault")
14567 .unwrap()
14568 .unwrap()
14569 .connection_id,
14570 ConnectionId::new(2)
14571 );
14572 }
14573
14574 #[test]
14577 fn without_an_open_swap_the_ordinary_gates_decide() {
14578 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
14579 supervisor.close_swap("vault");
14580
14581 let candidate = handler
14582 .handle_control(
14583 ConnectionId::new(2),
14584 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14585 )
14586 .unwrap();
14587 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
14588 let duplicate = handler
14589 .handle_control(
14590 ConnectionId::new(3),
14591 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
14592 )
14593 .unwrap();
14594 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14595 assert!(registry.get_candidate("vault").unwrap().is_none());
14596 }
14597 }
14598
14599 mod scopes {
14603 mod idle_tests {
14604 use super::*;
14605 include!("scope_idle_tests.rs");
14606 }
14607 mod route_open_timings {
14608 use super::*;
14609 include!("route_open_timing_tests.rs");
14610 }
14611 mod activity {
14612 include!("activity_scope_tests.rs");
14613 }
14614 mod delta {
14615 include!("scope_handler_delta_tests.rs");
14616 }
14617 mod call_modes {
14618 include!("scope_call_mode_tests.rs");
14619 }
14620
14621 use subc_protocol::scope::{
14622 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
14623 ScopeStatus,
14624 };
14625
14626 use super::*;
14627
14628 const OWNER: &str = "prefrontal-core";
14629
14630 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
14631 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
14632 }
14633
14634 async fn call(
14635 handler: &ControlHandler,
14636 ctx: &RouteCtx,
14637 request: &ModuleControlRequestFromModule,
14638 ) -> Frame {
14639 let body = serde_json::to_vec(request).unwrap();
14640 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
14641 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
14642 assert_eq!(replies.len(), 1, "{replies:?}");
14643 replies.pop().unwrap()
14644 }
14645
14646 async fn sync(
14647 handler: &ControlHandler,
14648 ctx: &RouteCtx,
14649 generation: u64,
14650 scopes: Vec<ScopeRecord>,
14651 ) -> Result<ModuleControlResponseToModule, String> {
14652 let reply = call(
14653 handler,
14654 ctx,
14655 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
14656 )
14657 .await;
14658 match reply.header.ty {
14659 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
14660 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
14661 }
14662 }
14663
14664 async fn describe(
14665 handler: &ControlHandler,
14666 ctx: &RouteCtx,
14667 owner: &str,
14668 scope_ref: &str,
14669 ) -> ModuleControlResponseToModule {
14670 let reply = call(
14671 handler,
14672 ctx,
14673 &ModuleControlRequestFromModule::ScopeDescribe {
14674 owner: Principal::Reserved {
14675 module_id: owner.to_string(),
14676 },
14677 scope_ref: scope_ref.to_string(),
14678 },
14679 )
14680 .await;
14681 assert_eq!(
14682 reply.header.ty,
14683 FrameType::Response,
14684 "{:?}",
14685 parse_error(&reply)
14686 );
14687 serde_json::from_slice(&reply.body).unwrap()
14688 }
14689
14690 #[tokio::test]
14691 async fn scope_describe_never_echoes_bind_proof_or_hash() {
14692 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
14693 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
14694 let mut rig = rig().await;
14695 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
14696 .await;
14697 let _bound = rig
14698 .bound(
14699 Some(AFT),
14700 PLEXUS,
14701 Some(
14702 rig.selector("s", Some(1))
14703 .with_bind_proof(Some(PROOF.into())),
14704 ),
14705 )
14706 .await;
14707 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
14708 let output = serde_json::to_string(&reply).unwrap();
14709 assert!(output.contains("\"status\":\"live\""));
14710 assert!(!output.contains(PROOF));
14711 assert!(!output.contains(HASH));
14712 }
14713
14714 async fn module(
14716 handler: &ControlHandler,
14717 connection: u64,
14718 module_id: &str,
14719 nonce: Option<&str>,
14720 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
14721 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
14722 hello_via_sink(
14723 handler,
14724 &ctx,
14725 &mut rx,
14726 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
14727 )
14728 .await;
14729 (ctx, rx)
14730 }
14731
14732 #[tokio::test]
14735 async fn a_client_connection_cannot_sync_or_describe() {
14736 let handler = ControlHandler::new(Arc::new(Registry::default()));
14737 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
14738 for request in [
14739 ModuleControlRequestFromModule::ScopeSync {
14740 generation: 1,
14741 scopes: vec![head("s", 1)],
14742 },
14743 ModuleControlRequestFromModule::ScopeDescribe {
14744 owner: Principal::Direct,
14745 scope_ref: "s".to_string(),
14746 },
14747 ] {
14748 let reply = call(&handler, &ctx, &request).await;
14749 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
14750 }
14751 assert!(
14752 !handler
14753 .scopes
14754 .read()
14755 .unwrap()
14756 .describe(
14757 &Principal::Reserved {
14758 module_id: OWNER.to_string()
14759 },
14760 "s"
14761 )
14762 .owner_synced
14763 );
14764 }
14765
14766 #[tokio::test]
14769 async fn a_module_without_a_supervised_launch_cannot_sync() {
14770 let handler = ControlHandler::new(Arc::new(Registry::default()));
14771 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
14772 assert_eq!(
14773 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
14774 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14775 );
14776 }
14777
14778 #[tokio::test]
14779 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
14780 let supervisor = SupervisorHandle::new();
14781 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14782 let handler = ControlHandler::new(Arc::new(Registry::default()))
14783 .with_supervisor(supervisor.clone());
14784 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
14785 sync(&handler, &incumbent, 1, vec![head("s", 1)])
14786 .await
14787 .expect("the current launch syncs");
14788
14789 supervisor.open_swap(OWNER, "n2".to_string());
14792 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
14793 assert_eq!(
14794 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
14795 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14796 );
14797 sync(&handler, &incumbent, 2, vec![head("s", 1)])
14798 .await
14799 .expect("the serving owner syncs during the swap");
14800
14801 supervisor.close_swap(OWNER);
14804 assert_eq!(
14805 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
14806 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14807 );
14808 sync(&handler, &incumbent, 3, vec![head("s", 1)])
14809 .await
14810 .expect("the serving owner syncs after the rollback");
14811 handler.cleanup_connection(candidate.connection_id).unwrap();
14812
14813 supervisor.open_swap(OWNER, "n3".to_string());
14818 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
14819 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
14820 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
14821 .await
14822 .expect("the promoted launch takes authority");
14823 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
14824 panic!("unexpected reply {reply:?}");
14825 };
14826 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
14827 assert_eq!(
14828 sync(&handler, &incumbent, 4, Vec::new()).await,
14829 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14830 );
14831 }
14832
14833 #[tokio::test]
14836 async fn closing_the_authority_connection_frees_sync_authority() {
14837 let supervisor = SupervisorHandle::new();
14838 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14839 let handler = ControlHandler::new(Arc::new(Registry::default()))
14840 .with_supervisor(supervisor.clone());
14841 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
14842 sync(&handler, &first, 10, vec![head("s", 1)])
14843 .await
14844 .unwrap();
14845 handler.cleanup_connection(first.connection_id).unwrap();
14846
14847 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
14848 sync(&handler, &second, 1, vec![head("s", 1)])
14849 .await
14850 .expect("the next connection takes the released authority");
14851 }
14852
14853 #[tokio::test]
14854 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
14855 let supervisor = SupervisorHandle::new();
14856 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14857 let handler =
14858 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
14859 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
14860 sync(&handler, &first, 10, vec![head("s", 1)])
14861 .await
14862 .unwrap();
14863 handler
14864 .handle_control_frame(
14865 &first,
14866 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
14867 )
14868 .await
14869 .unwrap();
14870 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
14871 sync(&handler, &second, 1, vec![head("s", 1)])
14872 .await
14873 .expect("GOODBYE releases authority even if the old socket remains open");
14874 }
14875
14876 #[tokio::test]
14877 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
14878 let registry = Arc::new(Registry::default());
14879 let supervisor_handle = SupervisorHandle::new();
14880 let supervisor =
14881 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
14882 .with_handle(supervisor_handle.clone())
14883 .with_daemon_incarnation("incarnation-7".to_string());
14884 supervisor
14887 .supervise_configured(
14888 ModuleSpec {
14889 module_id: OWNER.to_string(),
14890 program: PathBuf::from("/nonexistent/prefrontal-core"),
14891 args: Vec::new(),
14892 env: Vec::new(),
14893 reserved: false,
14894 reserved_prefixes: Vec::new(),
14895 protocol: ModuleProtocol::Subc,
14896 overlap: Default::default(),
14897 },
14898 false,
14899 )
14900 .unwrap();
14901 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
14902 let handler =
14903 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14904 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
14905
14906 let ModuleControlResponseToModule::ScopeDescribe {
14908 status,
14909 daemon_incarnation,
14910 owner_synced,
14911 owner_configured,
14912 scope,
14913 ..
14914 } = describe(&handler, &reader, OWNER, "s").await
14915 else {
14916 panic!("not a describe reply");
14917 };
14918 assert_eq!(status, ScopeStatus::NotLive);
14919 assert_eq!(daemon_incarnation, "incarnation-7");
14920 assert!(!owner_synced);
14921 assert!(owner_configured);
14922 assert!(scope.is_none());
14923
14924 let ModuleControlResponseToModule::ScopeDescribe {
14927 status,
14928 owner_configured,
14929 ..
14930 } = describe(&handler, &reader, "ghost", "s").await
14931 else {
14932 panic!("not a describe reply");
14933 };
14934 assert_eq!(status, ScopeStatus::NotLive);
14935 assert!(!owner_configured);
14936
14937 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
14939 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
14940 let ModuleControlResponseToModule::ScopeDescribe {
14941 status,
14942 scope_epoch,
14943 owner_synced,
14944 scope,
14945 ..
14946 } = describe(&handler, &reader, OWNER, "s").await
14947 else {
14948 panic!("not a describe reply");
14949 };
14950 assert_eq!(status, ScopeStatus::Live);
14951 assert_eq!(scope_epoch, Some(4));
14952 assert!(owner_synced);
14953 let stamp = scope.expect("a live scope carries its stamp");
14954 assert!(
14955 stamp.owner_authorized,
14956 "prefrontal-core is the default authority"
14957 );
14958 assert_eq!(stamp.kind, ScopeKind::Head);
14959 }
14960
14961 #[tokio::test]
14962 async fn scope_authority_owners_decides_owner_authorized() {
14963 let supervisor = SupervisorHandle::new();
14964 supervisor.set_spawn_nonce("broca", "b1".to_string());
14965 let handler = ControlHandler::new(Arc::new(Registry::default()))
14966 .with_supervisor(supervisor)
14967 .with_scope_authority_owners(vec!["broca".to_string()]);
14968 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
14969 let mut gated = head("s", 1);
14970 gated.attributes.agent_id = Some("agent".to_string());
14971 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
14972 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
14973 describe(&handler, &broca, "broca", "s").await
14974 else {
14975 panic!("not a describe reply");
14976 };
14977 assert!(scope.unwrap().owner_authorized);
14978 }
14979
14980 #[tokio::test]
14984 async fn scope_ops_and_the_scopes_capability_are_advertised() {
14985 let handler = ControlHandler::new(Arc::new(Registry::default()));
14986 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
14987 let ack = hello_via_sink(
14988 &handler,
14989 &ctx,
14990 &mut rx,
14991 hello_frame("m", PROTOCOL_VERSION, 1),
14992 )
14993 .await;
14994 let ack = parse_ack(&ack);
14995 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
14996 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
14997 }
14998 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
14999
15000 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
15001 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15002 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15003 let reply = handler
15004 .handle_control_frame(&client, frame)
15005 .await
15006 .unwrap()
15007 .pop()
15008 .unwrap();
15009 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15010 serde_json::from_slice(&reply.body).unwrap()
15011 else {
15012 panic!("not a server.describe reply");
15013 };
15014 assert!(
15015 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
15016 "{capabilities:?}"
15017 );
15018 }
15019
15020 const PLEXUS: &str = "plexus";
15023 const OTHER: &str = "other";
15024 const AFT: &str = "aft";
15025 const BROCA: &str = "broca";
15026 const MAGIC: &str = "magic-context";
15027
15028 fn nonce(module_id: &str) -> String {
15029 format!("nonce-{module_id}")
15030 }
15031
15032 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15033 let (tx, rx) = mpsc::channel(64);
15034 (
15035 RouteCtx {
15036 connection_id: ConnectionId::new(connection),
15037 egress: FrameSink::new(tx),
15038 },
15039 rx,
15040 )
15041 }
15042
15043 struct Rig {
15047 handler: ControlHandler,
15048 forwarding: Arc<ForwardingTable>,
15049 owner: RouteCtx,
15050 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15051 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
15052 generation: u64,
15053 next_connection: u64,
15054 _supervisor: Supervisor,
15055 }
15056
15057 async fn rig() -> Rig {
15058 rig_with_flow_support(true).await
15059 }
15060
15061 async fn rig_with_flow_support(flow_support: bool) -> Rig {
15062 rig_with_scope_capabilities(flow_support, false).await
15063 }
15064
15065 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
15066 let mut provides = Vec::new();
15067 if flow_support {
15068 provides.push("flow-scopes/v1");
15069 }
15070 if run_support {
15071 provides.push("agent-run-scopes/v1");
15072 }
15073 rig_with_target_capabilities(&provides).await
15074 }
15075
15076 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
15077 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
15078 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
15079 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
15082 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
15083 Frame::build(
15084 FrameType::Hello,
15085 control_flags(),
15086 0,
15087 0,
15088 connection,
15089 serde_json::to_vec(&body).unwrap(),
15090 )
15091 .unwrap()
15092 }
15093
15094 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
15095 let registry = Arc::new(Registry::default());
15096 let forwarding = Arc::new(ForwardingTable::default());
15097 let supervisor_handle = SupervisorHandle::new();
15098 let supervisor =
15099 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
15100 .with_handle(supervisor_handle.clone());
15101 supervisor
15102 .supervise_configured(
15103 ModuleSpec {
15104 module_id: OWNER.to_string(),
15105 program: PathBuf::from("/nonexistent/prefrontal-core"),
15106 args: Vec::new(),
15107 env: Vec::new(),
15108 reserved: false,
15109 reserved_prefixes: Vec::new(),
15110 protocol: ModuleProtocol::Subc,
15111 overlap: Default::default(),
15112 },
15113 false,
15114 )
15115 .unwrap();
15116 for module_id in [OWNER, AFT, BROCA, MAGIC] {
15117 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
15118 }
15119 let handler =
15120 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15121 .with_supervisor(supervisor_handle);
15122 let (owner, mut owner_rx) = wide_ctx(1);
15123 hello_via_sink(
15124 &handler,
15125 &owner,
15126 &mut owner_rx,
15127 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
15128 )
15129 .await;
15130 let mut modules = BTreeMap::new();
15131 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
15132 let (ctx, mut rx) = wide_ctx(connection);
15133 let hello = hello_with_capabilities(module_id, connection, provides);
15134 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
15135 modules.insert(module_id.to_string(), (ctx, rx));
15136 }
15137 Rig {
15138 handler,
15139 forwarding,
15140 owner,
15141 _owner_rx: owner_rx,
15142 modules,
15143 generation: 0,
15144 next_connection: 100,
15145 _supervisor: supervisor,
15146 }
15147 }
15148
15149 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
15150 ScopeCarrier::new(Principal::Reserved {
15151 module_id: module_id.to_string(),
15152 })
15153 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
15154 }
15155
15156 fn session(scope_epoch: u64) -> ScopeRecord {
15159 let mut record = head("s", scope_epoch);
15160 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
15161 record.attributes.agent_id = Some("agent-1".to_string());
15162 record.attributes.delegates = true;
15163 record
15164 }
15165
15166 impl Rig {
15167 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
15168 self.generation += 1;
15169 sync(&self.handler, &self.owner, self.generation, scopes)
15170 .await
15171 .expect("the owner's sync is accepted");
15172 }
15173
15174 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15175 ScopeSelector::new(
15176 Principal::Reserved {
15177 module_id: OWNER.to_string(),
15178 },
15179 scope_ref.to_string(),
15180 scope_epoch,
15181 )
15182 }
15183
15184 fn open_frame(
15185 &mut self,
15186 opener: Option<&str>,
15187 target: &str,
15188 scope: Option<ScopeSelector>,
15189 ) -> (
15190 RouteCtx,
15191 mpsc::Receiver<crate::router::OutboundFrame>,
15192 Frame,
15193 ) {
15194 self.next_connection += 1;
15195 let (ctx, rx) = wide_ctx(self.next_connection);
15196 let root = unique_project_root("scoped-open");
15197 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
15198 target: RouteTarget::ToolProvider {
15199 module_id: target.to_string(),
15200 },
15201 identity: BindIdentity::new(
15202 root.path().to_path_buf(),
15203 "unit".to_string(),
15204 "session".to_string(),
15205 ),
15206 consumer_identity: opener.map(|module_id| ConsumerIdentity {
15207 module_id: module_id.to_string(),
15208 launch_nonce: nonce(module_id),
15209 }),
15210 consumer_capabilities: None,
15211 role_versions: None,
15212 admission_facts: None,
15213 scope,
15214 })
15215 .unwrap();
15216 let frame = Frame::build(
15217 FrameType::Request,
15218 control_flags(),
15219 0,
15220 0,
15221 self.next_connection,
15222 body,
15223 )
15224 .unwrap();
15225 (ctx, rx, frame)
15226 }
15227
15228 async fn refused(
15230 &mut self,
15231 opener: Option<&str>,
15232 target: &str,
15233 scope: Option<ScopeSelector>,
15234 ) -> String {
15235 self.refusal_body(opener, target, scope).await["code"]
15236 .as_str()
15237 .unwrap()
15238 .to_string()
15239 }
15240
15241 async fn refusal_body(
15242 &mut self,
15243 opener: Option<&str>,
15244 target: &str,
15245 scope: Option<ScopeSelector>,
15246 ) -> Value {
15247 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
15248 let replies = tokio::time::timeout(
15249 Duration::from_secs(2),
15250 self.handler.handle_control_frame(&ctx, frame),
15251 )
15252 .await
15253 .expect("the open must be refused before waiting for a bind ack")
15254 .unwrap();
15255 assert_eq!(replies.len(), 1, "{replies:?}");
15256 assert_eq!(replies[0].header.ty, FrameType::Error);
15257 let (_, module_rx) = self.modules.get_mut(target).unwrap();
15258 assert!(
15259 module_rx.try_recv().is_err(),
15260 "a refused open relays nothing"
15261 );
15262 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
15263 parse_error(&replies[0])
15264 }
15265
15266 async fn relayed(
15268 &mut self,
15269 opener: Option<&str>,
15270 target: &str,
15271 scope: Option<ScopeSelector>,
15272 ) -> Relayed {
15273 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
15274 let handler = self.handler.clone();
15275 let task_ctx = ctx.clone();
15276 let task = tokio::spawn(async move {
15277 handler
15278 .handle_control_frame(&task_ctx, frame)
15279 .await
15280 .unwrap()
15281 });
15282 let (_, module_rx) = self.modules.get_mut(target).unwrap();
15283 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
15284 .await
15285 .expect("the target receives the relayed route.bind")
15286 .unwrap()
15287 .frame;
15288 Relayed {
15289 target: target.to_string(),
15290 client: ctx,
15291 client_rx: rx,
15292 task,
15293 bind,
15294 }
15295 }
15296
15297 async fn ack(&self, relayed: &Relayed) {
15298 let (module, _) = &self.modules[&relayed.target];
15299 self.handler
15300 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
15301 .await
15302 .unwrap();
15303 }
15304
15305 async fn bound(
15307 &mut self,
15308 opener: Option<&str>,
15309 target: &str,
15310 scope: Option<ScopeSelector>,
15311 ) -> Bound {
15312 let relayed = self.relayed(opener, target, scope).await;
15313 self.ack(&relayed).await;
15314 let Relayed {
15315 target,
15316 client,
15317 mut client_rx,
15318 task,
15319 bind,
15320 } = relayed;
15321 assert!(
15322 task.await.unwrap().is_empty(),
15323 "the open is answered by commit"
15324 );
15325 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
15326 Bound {
15327 target,
15328 client,
15329 client_rx,
15330 channel,
15331 epoch,
15332 bind,
15333 }
15334 }
15335
15336 fn live(&self, route: &Bound) -> bool {
15337 matches!(
15338 self.forwarding
15339 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
15340 .unwrap(),
15341 DataRoute::Client(DataRouteState::Bound(_))
15342 )
15343 }
15344 }
15345
15346 struct Relayed {
15347 target: String,
15348 client: RouteCtx,
15349 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15350 task: tokio::task::JoinHandle<Vec<Frame>>,
15351 bind: Frame,
15352 }
15353
15354 struct Bound {
15355 target: String,
15356 client: RouteCtx,
15357 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15358 channel: u16,
15359 epoch: u32,
15360 bind: Frame,
15361 }
15362
15363 impl Bound {
15364 fn closed_reason(&mut self) -> RouteCloseReason {
15367 let mut reason = None;
15368 let mut goodbye = false;
15369 while let Ok(outbound) = self.client_rx.try_recv() {
15370 let frame = outbound.frame;
15371 match frame.header.ty {
15372 FrameType::Goodbye => {
15373 assert_eq!(
15374 (frame.header.channel, frame.header.epoch),
15375 (self.channel, self.epoch)
15376 );
15377 goodbye = true;
15378 }
15379 FrameType::Push => {
15380 let ClientControlPush::RouteClosed {
15381 reason: r,
15382 module_id,
15383 ..
15384 } = serde_json::from_slice(&frame.body).unwrap()
15385 else {
15386 panic!("unexpected push");
15387 };
15388 assert_eq!(module_id, self.target);
15389 reason = Some(r);
15390 }
15391 other => panic!("unexpected frame {other:?}"),
15392 }
15393 }
15394 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
15395 reason.expect("the client is told why the route closed")
15396 }
15397
15398 fn untouched(&mut self) -> bool {
15399 self.client_rx.try_recv().is_err()
15400 }
15401
15402 fn stamp(&self) -> Option<ScopeStamp> {
15403 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
15404 ModuleControlRequest::RouteBind { scope, .. } => scope,
15405 other => panic!("expected a route.bind, got {other:?}"),
15406 }
15407 }
15408 }
15409
15410 #[tokio::test]
15411 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
15412 ) {
15413 let mut rig = rig().await;
15414 let mut record = session(1);
15415 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
15416 record.child_owners = vec![Principal::Reserved {
15417 module_id: MAGIC.to_string(),
15418 }];
15419 rig.sync(vec![record]).await;
15420 let scope = || Some(rig_selector("s", Some(1)));
15421
15422 rig.bound(Some(OWNER), PLEXUS, scope()).await;
15425 rig.bound(Some(AFT), OTHER, scope()).await;
15426 rig.bound(Some(BROCA), PLEXUS, scope()).await;
15427
15428 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
15432 assert_eq!(
15433 rig.refused(opener, target, scope()).await,
15434 error_codes::SCOPE_NOT_CARRIER,
15435 "{opener:?} -> {target}"
15436 );
15437 }
15438 }
15439
15440 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15441 ScopeSelector::new(
15442 Principal::Reserved {
15443 module_id: OWNER.to_string(),
15444 },
15445 scope_ref.to_string(),
15446 scope_epoch,
15447 )
15448 }
15449
15450 #[tokio::test]
15451 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
15452 let mut rig = rig().await;
15453 rig.sync(vec![session(1)]).await;
15454 for opener in [OWNER, AFT] {
15455 assert_eq!(
15456 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
15457 .await,
15458 error_codes::SCOPE_EPOCH_REQUIRED,
15459 "{opener}"
15460 );
15461 }
15462 }
15463
15464 #[tokio::test]
15465 async fn admission_separates_not_synced_not_live_and_ended() {
15466 let mut rig = rig().await;
15467 let code = rig
15469 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15470 .await;
15471 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
15472 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
15473
15474 let ghost = ScopeSelector::new(
15476 Principal::Reserved {
15477 module_id: "ghost".to_string(),
15478 },
15479 "s".to_string(),
15480 Some(1),
15481 );
15482 assert_eq!(
15483 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
15484 error_codes::SCOPE_NOT_LIVE
15485 );
15486
15487 rig.sync(vec![session(2)]).await;
15488 assert_eq!(
15489 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
15490 .await,
15491 error_codes::SCOPE_NOT_LIVE
15492 );
15493 for epoch in [1, 3] {
15494 assert_eq!(
15495 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
15496 .await,
15497 error_codes::SCOPE_ENDED,
15498 "epoch {epoch}"
15499 );
15500 }
15501 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
15503 .await;
15504 }
15505
15506 #[tokio::test]
15507 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
15508 let mut rig = rig().await;
15509 rig.sync(vec![session(1)]).await;
15510 let route = rig
15511 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15512 .await;
15513 let stamp = route.stamp().expect("a scoped bind carries the stamp");
15514 assert_eq!(stamp.scope_ref, "s");
15515 assert_eq!(stamp.scope_epoch, 1);
15516 assert_eq!(stamp.kind, ScopeKind::Head);
15517 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
15518 assert!(stamp.attributes.delegates);
15519 assert!(stamp.owner_authorized);
15520 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15521 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
15522
15523 let (broca, mut broca_rx) = wide_ctx(50);
15526 hello_via_sink(
15527 &rig.handler,
15528 &broca,
15529 &mut broca_rx,
15530 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
15531 )
15532 .await;
15533 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
15534 .await
15535 .unwrap();
15536 let own = ScopeSelector::new(
15537 Principal::Reserved {
15538 module_id: BROCA.to_string(),
15539 },
15540 "b".to_string(),
15541 Some(1),
15542 );
15543 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
15544 assert!(!route.stamp().unwrap().owner_authorized);
15545 }
15546
15547 #[tokio::test]
15548 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
15549 let mut rig = rig().await;
15550 let mut record = head("s", 1);
15551 record.carriers = vec![carrier(AFT, None)];
15552 let flow_id = "Flow:run-7/step_2!~";
15553 record.attributes.flow_id = Some(flow_id.to_string());
15554 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
15555 .await
15556 .unwrap();
15557 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
15558 panic!("not a sync reply");
15559 };
15560 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
15561 let route = rig
15562 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15563 .await;
15564 assert!(rig.live(&route), "the stamped bind committed");
15565 let stamp = route.stamp().expect("a flow scope carries a stamp");
15566 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
15567 assert_eq!(stamp.attributes.agent_id, None);
15568 assert!(!stamp.attributes.delegates);
15569 assert!(stamp.owner_authorized);
15570 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15571 assert_eq!(unscoped.stamp(), None);
15572 }
15573
15574 #[tokio::test]
15575 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
15576 let mut rig = rig_with_flow_support(false).await;
15577 let mut record = session(1);
15578 record.attributes.flow_id = Some("flow:7".to_string());
15579 rig.sync(vec![record]).await;
15580 for opener in [OWNER, AFT] {
15581 let body = rig
15582 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
15583 .await;
15584 assert_eq!(body["code"], "target_flow_unsupported");
15585 let message = body["message"].as_str().unwrap();
15586 for required in [PLEXUS, "flow-scopes/v1"] {
15587 assert!(message.contains(required), "{message}");
15588 }
15589 }
15590 }
15591
15592 #[tokio::test]
15593 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
15594 let mut rig = rig_with_flow_support(true).await;
15595 let mut record = session(1);
15596 record.attributes.flow_id = Some("flow:7".to_string());
15597 rig.sync(vec![record]).await;
15598 for opener in [OWNER, AFT] {
15599 let route = rig
15600 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
15601 .await;
15602 assert!(rig.live(&route));
15603 assert_eq!(
15604 route.stamp().unwrap().attributes.flow_id.as_deref(),
15605 Some("flow:7")
15606 );
15607 }
15608 }
15609
15610 #[tokio::test]
15611 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
15612 use std::future::Future;
15613
15614 let mut rig = rig_with_flow_support(true).await;
15615 let mut record = session(1);
15616 record.attributes.flow_id = Some("flow:7".to_string());
15617 rig.sync(vec![record]).await;
15618 let (client, mut client_rx, frame) =
15619 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
15620 for _ in 0..64 {
15623 client.egress.try_send(route_bind_ack(1)).unwrap();
15624 }
15625 let handler = rig.handler.clone();
15626 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15627 std::future::poll_fn(|cx| {
15628 assert!(open.as_mut().poll(cx).is_pending());
15629 std::task::Poll::Ready(())
15630 })
15631 .await;
15632 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15633
15634 let old_connection = rig.modules[PLEXUS].0.connection_id;
15635 rig.handler.cleanup_connection(old_connection).unwrap();
15636 let (replacement, mut replacement_rx) = wide_ctx(200);
15637 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
15638 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
15639 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
15640 let hello = Frame::build(
15641 FrameType::Hello,
15642 control_flags(),
15643 0,
15644 0,
15645 200,
15646 serde_json::to_vec(&body).unwrap(),
15647 )
15648 .unwrap();
15649 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
15650
15651 client_rx.try_recv().unwrap();
15652 let replies = tokio::time::timeout(Duration::from_secs(2), open)
15653 .await
15654 .expect("the replacement is refused without waiting for a bind ack")
15655 .unwrap();
15656 assert_eq!(replies.len(), 1);
15657 let body = parse_error(&replies[0]);
15658 assert_eq!(body["code"], "target_flow_unsupported");
15659 for required in [PLEXUS, "flow-scopes/v1"] {
15660 assert!(body["message"].as_str().unwrap().contains(required));
15661 }
15662 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
15663 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
15664 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15665 assert!(rig
15666 .handler
15667 .registry
15668 .get_module_by_connection(replacement.connection_id)
15669 .unwrap()
15670 .is_some());
15671 }
15672
15673 #[tokio::test]
15674 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
15675 use std::future::Future;
15676 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
15677 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15678 const NEXT_HASH: &str =
15679 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
15680 let mut rig = rig().await;
15681 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
15682 record.carriers.push(ScopeCarrier::new(Principal::Direct));
15683 rig.sync(vec![record.clone()]).await;
15684 let (client, mut rx, frame) = rig.open_frame(
15685 None,
15686 PLEXUS,
15687 Some(
15688 rig.selector("s", Some(1))
15689 .with_bind_proof(Some(PROOF.into())),
15690 ),
15691 );
15692 for _ in 0..64 {
15695 client.egress.try_send(route_bind_ack(1)).unwrap();
15696 }
15697 let handler = rig.handler.clone();
15698 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15699 std::future::poll_fn(|cx| {
15700 assert!(open.as_mut().poll(cx).is_pending());
15701 std::task::Poll::Ready(())
15702 })
15703 .await;
15704 record.bind_proof_sha256 = Some(NEXT_HASH.into());
15705 rig.sync(vec![record]).await;
15706 rx.try_recv().unwrap();
15707 let replies = tokio::time::timeout(Duration::from_secs(2), open)
15708 .await
15709 .unwrap()
15710 .unwrap();
15711 assert_eq!(
15712 parse_error(&replies[0])["code"],
15713 error_codes::SCOPE_BIND_PROOF_MISMATCH
15714 );
15715 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
15716 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
15717 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15718 }
15719
15720 #[tokio::test]
15721 async fn scope_bind_proof_reserved_openers_ignore_proof() {
15722 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15723 let mut rig = rig().await;
15724 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15725 .await;
15726 for opener in [OWNER, AFT] {
15727 for proof in [None, Some("malformed+proof=")] {
15728 let route = rig
15729 .bound(
15730 Some(opener),
15731 PLEXUS,
15732 Some(
15733 rig.selector("s", Some(1))
15734 .with_bind_proof(proof.map(str::to_string)),
15735 ),
15736 )
15737 .await;
15738 assert!(rig.live(&route));
15739 }
15740 }
15741 }
15742
15743 #[tokio::test]
15744 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
15745 use std::future::Future;
15746 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15747 const NEXT_HASH: &str =
15748 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
15749 let mut rig = rig().await;
15750 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15751 .await;
15752 let (client, mut rx, frame) =
15753 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
15754 for _ in 0..64 {
15755 client.egress.try_send(route_bind_ack(1)).unwrap();
15756 }
15757 let handler = rig.handler.clone();
15758 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15759 std::future::poll_fn(|cx| {
15760 assert!(open.as_mut().poll(cx).is_pending());
15761 std::task::Poll::Ready(())
15762 })
15763 .await;
15764 rig.sync(vec![
15765 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
15766 ])
15767 .await;
15768 rx.try_recv().unwrap();
15769 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15770 let bind = std::future::poll_fn(|cx| {
15771 assert!(open.as_mut().poll(cx).is_pending());
15772 module_rx.poll_recv(cx)
15773 })
15774 .await
15775 .unwrap()
15776 .frame;
15777 handler
15778 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
15779 .await
15780 .unwrap();
15781 let replies = tokio::time::timeout(Duration::from_secs(2), open)
15782 .await
15783 .unwrap()
15784 .unwrap();
15785 assert_eq!(replies.len(), 1);
15786 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
15789 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15790 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
15791 let route = rig
15792 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
15793 .await;
15794 assert!(rig.live(&route));
15795 }
15796
15797 #[tokio::test]
15798 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
15799 {
15800 let mut rig = rig_with_flow_support(false).await;
15801 rig.sync(vec![session(1)]).await;
15802 let route = rig
15803 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15804 .await;
15805 assert!(rig.live(&route));
15806 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
15807 let mut record = session(1);
15808 record.attributes.flow_id = Some("flow:7".to_string());
15809 rig.sync(vec![record]).await;
15810 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
15811 assert!(rig.live(&unscoped));
15812 assert_eq!(unscoped.stamp(), None);
15813 }
15814
15815 #[tokio::test]
15816 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
15817 let mut rig = rig().await;
15818 let mut record = session(1);
15819 record.attributes.flow_id = Some("flow:7".to_string());
15820 rig.sync(vec![record.clone()]).await;
15821 let before = rig.forwarding.published_scope_tag(OWNER, "s");
15822 let mut owner_route = rig
15823 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
15824 .await;
15825 let mut carrier_route = rig
15826 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
15827 .await;
15828 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15829 rig.sync(vec![record.clone()]).await;
15830 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
15831 assert!(rig.live(&owner_route) && owner_route.untouched());
15832 assert!(rig.live(&carrier_route) && carrier_route.untouched());
15833
15834 record.attributes.flow_id = Some("flow:8".to_string());
15835 rig.sync(vec![record]).await;
15836 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
15837 let before = before.unwrap();
15838 assert_eq!(after.scope_epoch, before.scope_epoch);
15839 assert!(after.version > before.version);
15840 for route in [&mut owner_route, &mut carrier_route] {
15841 assert!(!rig.live(route));
15842 assert_eq!(
15843 route.closed_reason(),
15844 RouteCloseReason::ScopeDelegationChanged
15845 );
15846 }
15847 assert!(rig.live(&unscoped) && unscoped.untouched());
15848 for target in [PLEXUS, OTHER] {
15851 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
15852 let goodbye = module_rx
15853 .try_recv()
15854 .expect("the provider sees the drain")
15855 .frame;
15856 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15857 assert!(module_rx.try_recv().is_err());
15858 }
15859 let rebound = rig
15860 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15861 .await;
15862 assert_eq!(
15863 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
15864 Some("flow:8")
15865 );
15866 }
15867
15868 #[tokio::test]
15873 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
15874 let mut rig = rig().await;
15875 rig.sync(vec![session(1)]).await;
15876 let mut cotenant = rig
15877 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
15878 .await;
15879
15880 let mut changed = session(1);
15881 changed.child_owners.push(Principal::Reserved {
15882 module_id: MAGIC.to_string(),
15883 });
15884 let mut ended = None;
15885 for (code, next) in [
15886 (error_codes::SCOPE_CHANGED, vec![changed]),
15887 (error_codes::SCOPE_ENDED, Vec::new()),
15888 ] {
15889 let relayed = rig
15890 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15891 .await;
15892 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
15893 ended = Some(next.is_empty());
15894 rig.sync(next).await;
15895 rig.ack(&relayed).await;
15896 let replies = relayed.task.await.unwrap();
15897 assert_eq!(replies.len(), 1, "{replies:?}");
15898 assert_eq!(parse_error(&replies[0])["code"], code);
15899 assert_eq!(
15900 subc_protocol::error_codes::is_retryable_route_open(code),
15901 code == error_codes::SCOPE_CHANGED
15902 );
15903 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15904 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15906 let mut goodbyes = Vec::new();
15909 while let Ok(outbound) = plexus_rx.try_recv() {
15910 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
15911 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
15912 }
15913 assert!(
15914 goodbyes.contains(&(bind_channel, bind_epoch)),
15915 "{goodbyes:?}"
15916 );
15917 assert!(rig
15918 .handler
15919 .registry
15920 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
15921 .unwrap()
15922 .is_some());
15923 }
15924 assert_eq!(ended, Some(true));
15925 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
15928 }
15929
15930 #[tokio::test]
15933 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
15934 struct Case {
15935 name: &'static str,
15936 change: fn(&mut ScopeRecord),
15937 closed: [Option<RouteCloseReason>; 4],
15940 }
15941 use RouteCloseReason::*;
15942 let cases = [
15943 Case {
15944 name: "a carrier entry removed",
15945 change: |r| {
15946 r.carriers.retain(|c| {
15947 c.principal
15948 != Principal::Reserved {
15949 module_id: AFT.to_string(),
15950 }
15951 })
15952 },
15953 closed: [None, Some(ScopeCarrierRemoved), None, None],
15954 },
15955 Case {
15956 name: "a target removed from a carrier",
15957 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
15958 closed: [None, None, None, Some(ScopeCarrierRemoved)],
15959 },
15960 Case {
15961 name: "a bare carrier narrowed to targets",
15962 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
15963 closed: [None, Some(ScopeCarrierRemoved), None, None],
15964 },
15965 Case {
15966 name: "delegates turned off",
15967 change: |r| r.attributes.delegates = false,
15968 closed: [Some(ScopeDelegationChanged); 4],
15969 },
15970 Case {
15971 name: "agent_id changed",
15972 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
15973 closed: [Some(ScopeDelegationChanged); 4],
15974 },
15975 Case {
15976 name: "a carrier added, child owners changed, the record re-sent",
15977 change: |r| {
15978 r.carriers.push(carrier(MAGIC, None));
15979 r.child_owners.push(Principal::Reserved {
15980 module_id: MAGIC.to_string(),
15981 });
15982 },
15983 closed: [None; 4],
15984 },
15985 Case {
15986 name: "a target added",
15987 change: |r| {
15988 r.carriers[1]
15989 .targets
15990 .as_mut()
15991 .unwrap()
15992 .push("third".to_string())
15993 },
15994 closed: [None; 4],
15995 },
15996 Case {
15997 name: "delegates turned on",
15998 change: |r| r.attributes.delegates = true,
15999 closed: [None; 4],
16000 },
16001 ];
16002 for case in cases {
16003 let mut rig = rig().await;
16004 rig.sync(vec![session(1)]).await;
16005 let scope = || Some(rig_selector("s", Some(1)));
16006 let mut routes = [
16007 rig.bound(Some(OWNER), PLEXUS, scope()).await,
16008 rig.bound(Some(AFT), PLEXUS, scope()).await,
16009 rig.bound(Some(BROCA), PLEXUS, scope()).await,
16010 rig.bound(Some(BROCA), OTHER, scope()).await,
16011 ];
16012 let mut record = session(1);
16013 (case.change)(&mut record);
16014 rig.sync(vec![record]).await;
16015 for (index, expected) in case.closed.iter().enumerate() {
16016 let route = &mut routes[index];
16017 match expected {
16018 Some(reason) => {
16019 assert!(!rig.live(route), "{}: route {index} still live", case.name);
16020 assert_eq!(
16021 route.closed_reason(),
16022 *reason,
16023 "{}: route {index}",
16024 case.name
16025 );
16026 }
16027 None => {
16028 assert!(rig.live(route), "{}: route {index} closed", case.name);
16029 assert!(
16030 route.untouched(),
16031 "{}: route {index} was told something",
16032 case.name
16033 );
16034 }
16035 }
16036 }
16037 }
16038 }
16039
16040 #[tokio::test]
16041 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
16042 for next in [Vec::new(), vec![session(2)]] {
16044 let mut rig = rig().await;
16045 rig.sync(vec![session(1)]).await;
16046 let mut route = rig
16047 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16048 .await;
16049 rig.sync(next).await;
16050 assert!(!rig.live(&route));
16051 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
16052 }
16053
16054 let mut rig = rig().await;
16057 let mut child = session(1);
16058 child.scope_ref = "child".to_string();
16059 child.kind = ScopeKind::Worker;
16060 child.parent = Some(ScopeParent::new(
16061 Principal::Reserved {
16062 module_id: OWNER.to_string(),
16063 },
16064 "s".to_string(),
16065 1,
16066 ));
16067 rig.sync(vec![session(1), child.clone()]).await;
16068 let mut child_route = rig
16069 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
16070 .await;
16071 assert_eq!(
16072 child_route.stamp().unwrap().parent_state,
16073 Some(ParentState::Linked)
16074 );
16075 rig.sync(vec![child]).await;
16076 assert!(!rig.live(&child_route));
16077 assert_eq!(
16078 child_route.closed_reason(),
16079 RouteCloseReason::ScopeParentEnded
16080 );
16081 }
16082
16083 #[tokio::test]
16084 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
16085 ) {
16086 let mut rig = rig().await;
16087 rig.sync(vec![session(1)]).await;
16088 let mut route = rig
16089 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16090 .await;
16091 let before = rig.forwarding.published_scope_tag(OWNER, "s");
16092 rig.sync(vec![session(1)]).await;
16093 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
16094 assert!(rig.live(&route) && route.untouched());
16095
16096 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
16100 .forwarding
16101 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16102 .unwrap()
16103 else {
16104 panic!("the route is bound");
16105 };
16106 binding.flow.acquire_tagged(9, false).await.unwrap();
16107 let mut widened = session(1);
16108 widened.carriers.push(carrier(MAGIC, None));
16109 rig.sync(vec![widened]).await;
16110 assert!(rig.live(&route) && route.untouched());
16111 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16112 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
16113 assert_eq!(binding.flow.in_flight(), 1);
16116 binding
16117 .flow
16118 .acquire_tagged(10, false)
16119 .await
16120 .expect("the flow is still open");
16121 }
16122
16123 #[tokio::test]
16126 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
16127 let mut rig = rig().await;
16128 rig.sync(vec![session(1)]).await;
16129 let mut on_incumbent = rig
16130 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16131 .await;
16132
16133 let (candidate, _candidate_rx) = wide_ctx(9);
16136 let registration = rig
16137 .handler
16138 .registry
16139 .register_candidate_with_control_ops(
16140 manifest(PLEXUS, PROTOCOL_VERSION),
16141 PROTOCOL_VERSION,
16142 candidate.connection_id,
16143 module_baseline_control_ops(),
16144 )
16145 .unwrap();
16146 rig.forwarding
16147 .register_candidate_module_connection(
16148 candidate.connection_id,
16149 PLEXUS.to_string(),
16150 PROTOCOL_VERSION,
16151 manifest_concurrency(®istration.manifest),
16152 candidate.egress.clone(),
16153 )
16154 .unwrap();
16155 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
16156 rig.handler
16157 .registry
16158 .promote_candidate(PLEXUS)
16159 .unwrap()
16160 .unwrap();
16161 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
16162
16163 rig.sync(Vec::new()).await;
16164 assert!(!rig.live(&on_incumbent));
16165 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
16166 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16167 let goodbye = incumbent_rx
16168 .try_recv()
16169 .expect("the superseded endpoint is told")
16170 .frame;
16171 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16172 }
16173 }
16174}
16175
16176#[cfg(test)]
16177mod concurrency_default_exposure_tests {
16178 use super::*;
16179
16180 fn hello_body(role_json: &str) -> Vec<u8> {
16181 format!(
16182 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
16183 )
16184 .into_bytes()
16185 }
16186
16187 fn manifest_from(body: &[u8]) -> ModuleManifest {
16188 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
16189 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
16190 .expect("manifest parses")
16191 }
16192
16193 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
16194
16195 #[test]
16196 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
16197 let body = hello_body(&format!(
16198 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
16199 ));
16200 let manifest = manifest_from(&body);
16201 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16204 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
16205 }
16206
16207 #[test]
16208 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
16209 let body = hello_body(&format!(
16210 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
16211 ));
16212 let manifest = manifest_from(&body);
16213 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16214 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16215 }
16216
16217 #[test]
16218 fn non_management_roles_are_never_reported() {
16219 let body = hello_body(
16220 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
16221 );
16222 let manifest = manifest_from(&body);
16223 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16224 }
16225}