1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_event_declarations,
25 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
26 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
27 },
28 scope::{
29 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
30 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
31 SCOPE_SYNC_OP,
32 },
33 session::{
34 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
35 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
36 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
37 ROLE_VERSIONS_FIELD,
38 },
39 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
40 Priority, RouteTarget, PROTOCOL_VERSION,
41};
42use tokio::time::{timeout_at, Instant};
43use tracing::{debug, info, warn};
44
45use crate::{
46 caller_identity::CallerIdentity,
47 capability_requirements::{
48 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
49 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
50 RuntimeModule,
51 },
52 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
53 forwarding::{
54 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
55 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
56 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
57 },
58 observability::{
59 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
60 },
61 privileged_prompt::{prompt_text, PrivilegedOperation},
62 provenance::{
63 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
64 },
65 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
66 router::{RouteCtx, RouterError},
67 scopes::{HelloLaunchNonces, ScopeTable},
68 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
69 stderr_tail::{CaptureState, TailEntry},
70 supervise::{
71 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
72 SupervisorHandle, SwapHelloAdmission,
73 },
74 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
75};
76
77pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
83
84const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
85const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
86const CAP_PING_PONG: &str = "ping_pong_v1";
87const CAP_SESSION_ATTACH: &str = "session_attach_v1";
88const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
89
90const SUBC_CONTROL_OPS: &[&str] = &[
91 ops::ACTIVITY_RECORD,
92 ops::ACTIVITY_QUERY,
93 ops::ACTIVITY_SUBSCRIBE,
94 ops::SERVER_DESCRIBE,
95 ops::CATALOG_LIST,
96 ops::ROUTE_OPEN,
97 ops::ROUTE_POLL,
98 ops::ROUTE_CLOSING,
99 ops::ROUTE_CLOSED,
100 ops::SUPERVISOR_LIST,
101 ops::SUPERVISOR_RESTART,
102 ops::SUPERVISOR_SWAP,
103 ops::SUPERVISOR_RELOAD,
104 ops::SUPERVISOR_RESCAN,
105 ops::SUPERVISOR_RELEASE_RESERVED,
106 ops::SUPERVISOR_SET_ENABLED,
107 ops::SUPERVISOR_HEALTH_PROBE,
108 ops::SUPERVISOR_HEALTH,
109 ops::SUPERVISOR_STDERR_TAIL,
110 ops::SUPERVISOR_TERMINALS,
111 ops::SUPERVISOR_ROUTES,
112 ops::SUPERVISOR_PROVENANCE,
113 ops::SUPERVISOR_SPAWN_SNAPSHOT,
114 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
115];
116
117const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
118 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
119 "supervisor.live_roots",
120 SCOPE_SYNC_OP,
121 SCOPE_APPLY_OP,
122 SCOPE_DESCRIBE_OP,
123 "operator.confirm",
124];
125
126const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
130
131const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
132
133pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
141
142pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
157
158pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
176
177const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
178const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
179
180fn reload_verdict(
181 configured: &Path,
182 spawned_from: Option<&Path>,
183 image: subc_control::RunningImageAgreement,
184) -> PendingReloadVerdict {
185 let path = match spawned_from {
186 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
187 ReloadPathAgreement::Match
188 }
189 Some(spawned_from) => ReloadPathAgreement::Mismatch {
190 configured: configured.to_path_buf(),
191 spawned_from: spawned_from.to_path_buf(),
192 },
193 None => ReloadPathAgreement::Unavailable {
194 reason: if matches!(
195 image,
196 subc_control::RunningImageAgreement::Unavailable {
197 reason: subc_control::RunningImageUnavailableReason::NotRunning
198 }
199 ) {
200 ReloadPathUnavailableReason::NotRunning
201 } else {
202 ReloadPathUnavailableReason::SpawnedPathUnavailable
203 },
204 },
205 };
206 PendingReloadVerdict {
207 path,
208 image,
209 spec: None,
210 }
211}
212
213fn launch_spec_agreement(
214 configured: &crate::supervise::ModuleSpec,
215 running: bool,
216 spawned: Option<&crate::supervise::LaunchSpecDigest>,
217) -> subc_control::LaunchSpecAgreement {
218 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
219 if !running {
220 return Agreement::Unavailable {
221 reason: Reason::NotRunning,
222 };
223 }
224 match spawned {
225 None => Agreement::Unavailable {
226 reason: Reason::SpawnedSpecUnavailable,
227 },
228 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
229 Agreement::Match
230 }
231 Some(_) => Agreement::Mismatch,
232 }
233}
234
235fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
236 if configured == spawned_from {
237 return true;
238 }
239 #[cfg(windows)]
240 {
241 match (
247 std::fs::canonicalize(configured),
248 std::fs::canonicalize(spawned_from),
249 ) {
250 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
251 _ => false,
252 }
253 }
254 #[cfg(not(windows))]
255 {
256 false
257 }
258}
259
260#[derive(Clone)]
261struct DaemonProvenanceFacts {
262 build: DaemonBuildProvenance,
263 pid: Option<u32>,
264 started_at_ms: Option<u64>,
265 start_clock: Option<crate::clock::StartClock>,
266 executable_path: Option<PathBuf>,
267 executable_identity: Option<SpawnedFileIdentity>,
268 process_start_time: Option<u64>,
269 probe: ExecutableIdentityProbe,
270}
271
272impl Default for DaemonProvenanceFacts {
273 fn default() -> Self {
274 Self {
275 build: DaemonBuildProvenance {
276 build_git_sha: None,
277 build_lock_digest: None,
278 },
279 pid: None,
280 started_at_ms: None,
281 start_clock: None,
282 executable_path: None,
283 executable_identity: None,
284 process_start_time: None,
285 probe: ExecutableIdentityProbe::default(),
286 }
287 }
288}
289
290#[derive(Debug, Clone)]
291struct SupervisorRescanContext {
292 supervisor: Supervisor,
293 config_path: PathBuf,
294 configured_port: Option<u16>,
295 storage_config: Option<crate::daemon_config::StorageConfig>,
296 admission_facts_carrier_module_id: Option<String>,
297 admission_facts_targets: Option<Vec<String>>,
298 scope_authority_owners: Vec<String>,
299 privileged_operator_prompt: PrivilegedOperatorPrompt,
300}
301
302const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
323 "reloading",
324 "supervisor_not_live",
325 "registration_not_active",
326 "no_forwarding_connection",
327 "relay_send_failed",
328];
329
330#[derive(Clone)]
332pub struct ControlHandler {
333 registry: Arc<Registry>,
334 activity: Option<crate::activity::ActivityLedger>,
335 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
336 forwarding: Arc<ForwardingTable>,
337 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
338 supervisor: SupervisorHandle,
339 subc_capabilities: Arc<[String]>,
340 route_bind_relay_timeout: Duration,
344 route_bind_relay_timeouts: BTreeMap<String, Duration>,
348 route_bind_breakers: RouteBindBreakers,
351 route_bind_concurrency: RouteBindConcurrency,
354 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
359 route_bind_breaker_threshold: u32,
361 route_bind_breaker_cooldown: Duration,
363 health_probe_timeout: Duration,
364 storage_config: Option<crate::daemon_config::StorageConfig>,
367 machine_id: Option<crate::machine_id::MachineId>,
371 admission_facts_carrier_module_id: Option<String>,
372 admission_facts_targets: Option<Vec<String>>,
373 scopes: Arc<RwLock<ScopeTable>>,
377 scope_deadline_added: Arc<tokio::sync::Notify>,
381 scope_authority_owners: Vec<String>,
384 privileged_operator_prompt: PrivilegedOperatorPrompt,
386 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
389 #[cfg(test)]
390 caller_lookup: Option<TestCallerLookup>,
391 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
394 rescan: Option<SupervisorRescanContext>,
395 connected_clients: ConnectedClients,
396 counters: DaemonCounters,
397 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
398 daemon_provenance: DaemonProvenanceFacts,
399 #[cfg(test)]
400 control_dispatch_delay: Option<Duration>,
401 #[cfg(test)]
402 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
403 #[cfg(test)]
407 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
408 #[cfg(test)]
414 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
415}
416
417impl fmt::Debug for ControlHandler {
418 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
419 f.debug_struct("ControlHandler")
420 .field("registry", &self.registry)
421 .field("forwarding", &self.forwarding)
422 .field("process_liveness", &self.process_liveness.is_some())
423 .field("supervisor", &self.supervisor)
424 .field("subc_capabilities", &self.subc_capabilities)
425 .finish()
426 }
427}
428
429struct RouteOpenRequest {
430 target: RouteTarget,
431 identity: BindIdentity,
432 consumer_identity: Option<ConsumerIdentity>,
433 consumer_capabilities: Option<Vec<String>>,
434 role_versions: Option<BTreeMap<String, String>>,
435 admission_facts: Option<serde_json::Value>,
436 scope: Option<ScopeSelector>,
437}
438
439#[derive(Clone, Copy)]
440enum RouteOpenPhase {
441 Admission,
442 ProjectRoot,
443 EgressReservation,
444 BindSend,
445 BindAckWait,
446}
447
448struct RouteOpenTimings {
452 module_id: String,
453 principal: String,
454 admission: Duration,
455 project_root: Duration,
456 egress_reservation: Duration,
457 bind_send: Duration,
458 bind_ack_wait: Duration,
459 phase: Option<RouteOpenPhase>,
460 phase_started_at: Instant,
461}
462
463impl RouteOpenTimings {
464 fn new() -> Self {
465 Self {
466 module_id: String::new(),
467 principal: "unresolved".to_string(),
469 admission: Duration::ZERO,
470 project_root: Duration::ZERO,
471 egress_reservation: Duration::ZERO,
472 bind_send: Duration::ZERO,
473 bind_ack_wait: Duration::ZERO,
474 phase: Some(RouteOpenPhase::Admission),
475 phase_started_at: Instant::now(),
476 }
477 }
478
479 fn finish(&mut self) {
480 let elapsed = self.phase_started_at.elapsed();
481 match self.phase.take() {
482 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
483 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
484 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
485 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
486 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
487 None => {}
488 }
489 }
490
491 fn enter(&mut self, phase: RouteOpenPhase) {
492 self.finish();
493 self.phase = Some(phase);
494 self.phase_started_at = Instant::now();
495 }
496}
497
498struct RouteBindReservationGuard {
499 forwarding: Arc<ForwardingTable>,
500 endpoint: ModuleEndpointId,
501 relay_corr: u64,
502 armed: bool,
503}
504
505struct ModuleControlRpcGuard {
506 forwarding: Arc<ForwardingTable>,
507 endpoint: ModuleEndpointId,
508 corr: u64,
509 armed: bool,
510}
511
512impl ModuleControlRpcGuard {
513 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
514 Self {
515 forwarding,
516 endpoint,
517 corr,
518 armed: true,
519 }
520 }
521
522 fn disarm(&mut self) {
523 self.armed = false;
524 }
525}
526
527impl Drop for ModuleControlRpcGuard {
528 fn drop(&mut self) {
529 if self.armed {
530 let _ = self
531 .forwarding
532 .cancel_module_control_rpc(self.endpoint, self.corr);
533 }
534 }
535}
536
537impl RouteBindReservationGuard {
538 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
539 Self {
540 forwarding,
541 endpoint,
542 relay_corr,
543 armed: true,
544 }
545 }
546
547 fn release_and_disarm(&mut self) {
548 if !self.armed {
549 return;
550 }
551 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
552 self.endpoint,
553 self.relay_corr,
554 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
555 ) {
556 send_goodbye_target_best_effort(
557 &self.forwarding.counters(),
558 &target,
559 "canceled route.bind",
560 );
561 }
562 self.armed = false;
563 }
564
565 fn disarm(&mut self) {
566 self.armed = false;
567 }
568}
569
570impl Drop for RouteBindReservationGuard {
571 fn drop(&mut self) {
572 self.release_and_disarm();
573 }
574}
575
576#[derive(Debug, Clone, Default)]
606pub(crate) struct RouteBindBreakers {
607 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
608}
609
610#[derive(Debug, Clone, Default)]
611pub(crate) struct RouteBindConcurrency {
612 modules: Arc<Mutex<HashMap<String, usize>>>,
613}
614
615struct RouteBindConcurrencyGuard {
616 concurrency: RouteBindConcurrency,
617 module_id: String,
618}
619
620impl RouteBindConcurrency {
621 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
624 let mut modules = self
625 .modules
626 .lock()
627 .expect("route.bind concurrency mutex poisoned");
628 let in_flight = modules.entry(module_id.to_string()).or_default();
629 if *in_flight >= limit {
630 return Err(*in_flight);
631 }
632 *in_flight += 1;
633 Ok(RouteBindConcurrencyGuard {
634 concurrency: self.clone(),
635 module_id: module_id.to_string(),
636 })
637 }
638}
639
640impl Drop for RouteBindConcurrencyGuard {
641 fn drop(&mut self) {
642 let mut modules = self
643 .concurrency
644 .modules
645 .lock()
646 .expect("route.bind concurrency mutex poisoned");
647 let remove = {
648 let in_flight = modules
649 .get_mut(&self.module_id)
650 .expect("admitted route.bind has a concurrency entry");
651 *in_flight -= 1;
652 *in_flight == 0
653 };
654 if remove {
655 modules.remove(&self.module_id);
656 }
657 }
658}
659
660#[derive(Debug, Default)]
661struct ModuleBreakerState {
662 consecutive_timeouts: u32,
664 cooldown_until: Option<Instant>,
667 probe_in_flight: Option<Arc<()>>,
672}
673
674enum RouteBindAdmission<'a> {
676 Admitted {
677 guard: RouteBindBreakerGuard<'a>,
678 probe: bool,
681 },
682 Refused {
683 consecutive_timeouts: u32,
684 retry_in: Duration,
688 probe_in_flight: bool,
689 },
690}
691
692struct RouteBindBreakerGuard<'a> {
699 breakers: RouteBindBreakers,
700 module_id: &'a str,
701 probe_token: Option<Arc<()>>,
702 settled: bool,
703}
704
705impl RouteBindBreakerGuard<'_> {
706 fn record_accepted(&mut self) -> bool {
710 self.settled = true;
711 self.breakers.record_accepted(self.module_id)
712 }
713
714 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
717 self.settled = true;
718 self.breakers.record_timeout(
719 self.module_id,
720 self.probe_token.as_ref(),
721 threshold,
722 cooldown,
723 )
724 }
725
726 fn record_inconclusive(&mut self) {
735 self.settled = true;
736 self.breakers
737 .record_inconclusive(self.module_id, self.probe_token.as_ref());
738 }
739}
740
741impl Drop for RouteBindBreakerGuard<'_> {
742 fn drop(&mut self) {
743 if !self.settled {
744 self.breakers
745 .record_inconclusive(self.module_id, self.probe_token.as_ref());
746 }
747 }
748}
749
750struct BreakerOpened {
754 consecutive_timeouts: u32,
755 reopened_after_probe: bool,
758}
759
760impl RouteBindBreakers {
761 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
762 self.modules
763 .lock()
764 .expect("route.bind breaker mutex poisoned")
765 }
766
767 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
770 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
771 probe: probe_token.is_some(),
772 guard: RouteBindBreakerGuard {
773 breakers: self.clone(),
774 module_id,
775 probe_token,
776 settled: false,
777 },
778 };
779
780 let mut modules = self.lock();
781 let Some(state) = modules.get_mut(module_id) else {
782 return admitted(None);
783 };
784 let Some(cooldown_until) = state.cooldown_until else {
785 return admitted(None);
786 };
787 if state.probe_in_flight.is_some() {
788 return RouteBindAdmission::Refused {
789 consecutive_timeouts: state.consecutive_timeouts,
790 retry_in: Duration::ZERO,
791 probe_in_flight: true,
792 };
793 }
794 let now = Instant::now();
795 if now < cooldown_until {
796 return RouteBindAdmission::Refused {
797 consecutive_timeouts: state.consecutive_timeouts,
798 retry_in: cooldown_until - now,
799 probe_in_flight: false,
800 };
801 }
802 let token = Arc::new(());
803 state.probe_in_flight = Some(Arc::clone(&token));
804 admitted(Some(token))
805 }
806
807 fn record_accepted(&self, module_id: &str) -> bool {
808 self.lock()
809 .remove(module_id)
810 .is_some_and(|state| state.cooldown_until.is_some())
811 }
812
813 fn record_timeout(
814 &self,
815 module_id: &str,
816 probe_token: Option<&Arc<()>>,
817 threshold: u32,
818 cooldown: Duration,
819 ) -> Option<BreakerOpened> {
820 let mut modules = self.lock();
821 let state = modules.entry(module_id.to_string()).or_default();
822 let was_open = state.cooldown_until.is_some();
823 let was_probe = Self::owns_probe(state, probe_token);
824 if was_probe {
825 state.probe_in_flight = None;
826 }
827 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
828 if state.consecutive_timeouts < threshold {
829 return None;
830 }
831 state.cooldown_until = Some(Instant::now() + cooldown);
832 Some(BreakerOpened {
833 consecutive_timeouts: state.consecutive_timeouts,
834 reopened_after_probe: was_open && was_probe,
835 })
836 }
837
838 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
839 state
843 .probe_in_flight
844 .as_ref()
845 .zip(token)
846 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
847 }
848
849 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
850 if let Some(state) = self.lock().get_mut(module_id) {
851 if Self::owns_probe(state, probe_token) {
852 state.probe_in_flight = None;
853 }
854 }
855 }
856
857 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
872 self.lock()
873 .remove(module_id)
874 .map(|state| state.consecutive_timeouts)
875 .filter(|discarded| *discarded > 0)
876 }
877
878 fn open_snapshot(&self) -> Option<serde_json::Value> {
885 let now = Instant::now();
886 let modules = self.lock();
887 let open = modules
888 .iter()
889 .filter_map(|(module_id, state)| {
890 let cooldown_until = state.cooldown_until?;
891 Some((
892 module_id.clone(),
893 serde_json::json!({
894 "consecutive_timeouts": state.consecutive_timeouts,
895 "cooldown_remaining_ms":
896 cooldown_until.saturating_duration_since(now).as_millis() as u64,
897 "probe_in_flight": state.probe_in_flight.is_some(),
898 }),
899 ))
900 })
901 .collect::<serde_json::Map<String, serde_json::Value>>();
902 (!open.is_empty()).then_some(serde_json::Value::Object(open))
903 }
904}
905
906impl ControlHandler {
907 pub fn new(registry: Arc<Registry>) -> Self {
908 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
909 }
910
911 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
912 let counters = forwarding.counters();
913 let route_bind_breakers = forwarding.route_bind_breakers();
917 let route_bind_concurrency = forwarding.route_bind_concurrency();
918 let route_outages = forwarding.route_outages();
919 Self {
920 registry,
921 activity: None,
922 activity_streams: Arc::new(Mutex::new(HashMap::new())),
923 forwarding,
924 process_liveness: None,
925 supervisor: SupervisorHandle::new(),
926 subc_capabilities: Arc::from([
927 CAP_MANIFEST_REGISTRATION.to_string(),
928 CAP_CHANNEL_LIFECYCLE.to_string(),
929 CAP_PING_PONG.to_string(),
930 CAP_SESSION_ATTACH.to_string(),
931 CAP_ADMISSION_FACTS_RELAY.to_string(),
932 CAP_SCOPES_V1.to_string(),
933 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
934 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
935 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
936 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
937 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
938 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
939 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
940 ]),
941 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
942 route_bind_relay_timeouts: BTreeMap::new(),
943 route_bind_breakers,
944 route_bind_concurrency,
945 route_outages,
946 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
947 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
948 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
949 storage_config: None,
950 machine_id: None,
951 admission_facts_carrier_module_id: None,
952 admission_facts_targets: None,
953 scopes: Arc::new(RwLock::new(ScopeTable::new(
954 crate::daemon_config::default_scope_authority_owners(),
955 ))),
956 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
957 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
958 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
959 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
960 #[cfg(test)]
961 caller_lookup: None,
962 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
963 rescan: None,
964 connected_clients: ConnectedClients::new(),
965 counters,
966 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
967 daemon_provenance: DaemonProvenanceFacts::default(),
968 #[cfg(test)]
969 control_dispatch_delay: None,
970 #[cfg(test)]
971 provenance_probe_override: None,
972 #[cfg(test)]
973 route_bind_relay_sent_hold: None,
974 #[cfg(test)]
975 hello_registry_to_forwarding_hold: None,
976 }
977 }
978
979 pub fn with_storage_config(
982 mut self,
983 storage_config: Option<crate::daemon_config::StorageConfig>,
984 ) -> Self {
985 self.storage_config = storage_config;
986 self
987 }
988
989 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
992 self.activity = Some(crate::activity::ActivityLedger::open(
993 run_dir.join("activity.jsonl"),
994 incarnation,
995 ));
996 self.supervisor.set_activity(self.activity.clone());
997 self
998 }
999
1000 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1003 self.machine_id = machine_id;
1004 self
1005 }
1006
1007 pub fn with_admission_facts_config(
1011 mut self,
1012 carrier_module_id: Option<String>,
1013 targets: Option<Vec<String>>,
1014 ) -> Self {
1015 self.admission_facts_carrier_module_id = carrier_module_id;
1016 self.admission_facts_targets = targets;
1017 self
1018 }
1019
1020 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1024 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1025 self.scope_authority_owners = owners;
1026 self
1027 }
1028
1029 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1031 self.privileged_operator_prompt = mode;
1032 self
1033 }
1034
1035 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1038 self.route_bind_relay_timeout = timeout;
1039 self
1040 }
1041
1042 pub fn with_route_bind_relay_timeouts(
1048 mut self,
1049 timeouts: impl IntoIterator<Item = (String, Duration)>,
1050 ) -> Self {
1051 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1052 self
1053 }
1054
1055 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1061 self.route_bind_relay_timeouts
1062 .get(module_id)
1063 .copied()
1064 .unwrap_or(self.route_bind_relay_timeout)
1065 }
1066
1067 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1075 self.route_bind_breaker_threshold = threshold.max(1);
1076 self.route_bind_breaker_cooldown = cooldown;
1077 self
1078 }
1079
1080 #[cfg(test)]
1081 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1082 self.health_probe_timeout = timeout;
1083 self
1084 }
1085
1086 #[cfg(test)]
1087 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1088 self.control_dispatch_delay = Some(delay);
1089 self
1090 }
1091
1092 #[cfg(test)]
1096 pub(crate) fn with_route_bind_relay_sent_hold(
1097 mut self,
1098 hold: Arc<tokio::sync::Barrier>,
1099 ) -> Self {
1100 self.route_bind_relay_sent_hold = Some(hold);
1101 self
1102 }
1103
1104 #[cfg(test)]
1109 pub(crate) fn with_hello_registry_to_forwarding_hold(
1110 mut self,
1111 hold: Arc<std::sync::Barrier>,
1112 ) -> Self {
1113 self.hello_registry_to_forwarding_hold = Some(hold);
1114 self
1115 }
1116
1117 #[cfg(test)]
1118 fn hold_between_registry_and_forwarding(&self) {
1119 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1120 hold.wait();
1121 hold.wait();
1122 }
1123 }
1124
1125 pub fn with_process_liveness(
1126 mut self,
1127 process_liveness: Arc<dyn ModuleProcessLiveness>,
1128 ) -> Self {
1129 self.process_liveness = Some(process_liveness);
1130 self
1131 }
1132
1133 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1134 self.supervisor = supervisor;
1135 self.supervisor.set_activity(self.activity.clone());
1136 self
1137 }
1138
1139 pub fn with_daemon_provenance(
1140 mut self,
1141 pid: u32,
1142 started_at_ms: u64,
1143 executable_path: Option<PathBuf>,
1144 build_git_sha: Option<String>,
1145 build_lock_digest: Option<String>,
1146 ) -> Self {
1147 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1148 let process_start_time = process_start_time(pid);
1149 self.daemon_provenance = DaemonProvenanceFacts {
1150 build: DaemonBuildProvenance {
1151 build_git_sha,
1152 build_lock_digest,
1153 },
1154 pid: Some(pid),
1155 started_at_ms: Some(started_at_ms),
1156 start_clock: None,
1157 executable_path,
1158 executable_identity,
1159 process_start_time,
1160 probe: ExecutableIdentityProbe::default(),
1161 };
1162 self
1163 }
1164
1165 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1166 self.daemon_provenance.start_clock = Some(clock);
1167 self
1168 }
1169
1170 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1172 self.scopes
1173 .write()
1174 .unwrap_or_else(|poisoned| poisoned.into_inner())
1175 .set_wall_clock(Arc::new(clock));
1176 self
1177 }
1178
1179 #[cfg(test)]
1180 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1181 self.provenance_probe_override = Some(result);
1182 self
1183 }
1184
1185 pub fn with_capability_config(
1189 self,
1190 modules: impl IntoIterator<Item = (String, bool)>,
1191 reserved_capabilities: BTreeMap<String, String>,
1192 ) -> Self {
1193 self.capability_evaluator
1194 .configure(modules, reserved_capabilities);
1195 self
1196 }
1197
1198 pub fn with_supervisor_rescan(
1199 mut self,
1200 supervisor: Supervisor,
1201 config_path: impl Into<PathBuf>,
1202 configured_port: Option<u16>,
1203 ) -> Self {
1204 self.rescan = Some(SupervisorRescanContext {
1205 supervisor,
1206 config_path: config_path.into(),
1207 configured_port,
1208 storage_config: self.storage_config.clone(),
1209 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1210 admission_facts_targets: self.admission_facts_targets.clone(),
1211 scope_authority_owners: self.scope_authority_owners.clone(),
1212 privileged_operator_prompt: self.privileged_operator_prompt,
1213 });
1214 self
1215 }
1216
1217 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1218 self.connected_clients = connected_clients;
1219 self
1220 }
1221
1222 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1223 Arc::clone(&self.forwarding)
1224 }
1225
1226 pub(crate) fn counters(&self) -> DaemonCounters {
1227 self.counters.clone()
1228 }
1229
1230 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1233 tokio::spawn(async move {
1234 loop {
1235 self.capability_evaluator
1236 .wait_for_change_or_deadline()
1237 .await;
1238 self.refresh_capability_requirements();
1239 }
1240 });
1241 }
1242
1243 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1249 tokio::spawn(async move {
1250 loop {
1251 let has_deadlines = self
1252 .scopes
1253 .read()
1254 .unwrap_or_else(|poisoned| poisoned.into_inner())
1255 .has_deadlines();
1256 if !has_deadlines {
1257 self.scope_deadline_added.notified().await;
1258 continue;
1259 }
1260 tokio::time::sleep(Duration::from_secs(1)).await;
1261 if let Err(error) = self.sweep_expired_scopes() {
1262 warn!(%error, "scope expiry sweep failed");
1263 }
1264 }
1265 });
1266 }
1267
1268 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1269 let mut table = self
1270 .scopes
1271 .write()
1272 .unwrap_or_else(|poisoned| poisoned.into_inner());
1273 let swept = table.sweep_expired();
1274 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1275 drop(table);
1276 self.close_scope_drained_routes(drained);
1277 Ok(())
1278 }
1279
1280 fn publish_scope_changes(
1281 &self,
1282 changes: &[crate::scopes::ScopeTagChange],
1283 expired: &[crate::scopes::ScopeExpired],
1284 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1285 if changes.is_empty() {
1286 return Ok(Vec::new());
1287 }
1288 let drained = self
1289 .forwarding
1290 .publish_scope_changes(changes)
1291 .map_err(RouterError::Forwarding)?;
1292 let mut counts = HashMap::new();
1293 for route in &drained {
1294 *counts
1295 .entry((
1296 route.scope.owner.as_str(),
1297 route.scope.scope_ref.as_str(),
1298 route.scope.tag.scope_epoch,
1299 ))
1300 .or_insert(0usize) += 1;
1301 }
1302 for scope in expired {
1303 let routes_closed = counts
1304 .get(&(
1305 scope.owner.as_str(),
1306 scope.scope_ref.as_str(),
1307 scope.scope_epoch,
1308 ))
1309 .copied()
1310 .unwrap_or(0);
1311 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1312 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1313 routes_closed, "scope expired");
1314 }
1315 Ok(drained)
1316 }
1317
1318 fn runtime_capability_snapshot(
1319 &self,
1320 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1321 let runtime = self
1322 .supervisor
1323 .list()
1324 .into_iter()
1325 .map(|module| {
1326 let status = module.status().map_err(|err| {
1327 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1328 })?;
1329 Ok(RuntimeModule {
1330 module_id: status.module_id,
1331 state: status.state,
1332 enabled: status.enabled,
1333 })
1334 })
1335 .collect::<Result<Vec<_>, RouterError>>()?;
1336 let registrations = self.registry.list_registered_modules().map_err(|err| {
1342 RouterError::backend(
1343 0,
1344 0,
1345 format!("failed to list capability registrations: {err}"),
1346 )
1347 })?;
1348 let registrations = registrations
1349 .into_iter()
1350 .map(|registration| RegisteredModule {
1351 module_id: registration.manifest.module_id,
1352 module_version: registration.manifest.module_version,
1353 capabilities: registration.manifest.capabilities,
1354 })
1355 .collect();
1356 Ok((runtime, registrations))
1357 }
1358
1359 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1366 let cached_registration = RegisteredModule {
1367 module_id: registration.manifest.module_id.clone(),
1368 module_version: registration.manifest.module_version.clone(),
1369 capabilities: registration.manifest.capabilities.clone(),
1370 };
1371 if self.capability_evaluator.record_hello(&cached_registration) {
1372 warn!(
1373 module_id = %cached_registration.module_id,
1374 "capability claims drifted from the cached manifest"
1375 );
1376 }
1377 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1378 self.enforce_capability_denies();
1379 }
1380 self.refresh_capability_requirements();
1381 }
1382
1383 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1387 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1388 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1389 self.supervisor.set_swap_promotion_observer(observer);
1390 }
1391
1392 pub fn refresh_capability_requirements(&self) {
1393 match self.runtime_capability_snapshot() {
1394 Ok((runtime, registrations)) => {
1395 log_requirement_events(
1396 self.capability_evaluator
1397 .evaluate_now(&runtime, ®istrations),
1398 );
1399 }
1400 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1401 }
1402 }
1403
1404 fn enforce_capability_denies(&self) {
1408 let registrations = match self.registry.list_registered_modules() {
1409 Ok(snapshot) => snapshot,
1410 Err(err) => {
1411 warn!(error = %err, "failed to read registrations for capability deny census");
1412 return;
1413 }
1414 };
1415 let manifests = registrations
1416 .into_iter()
1417 .map(|registration| {
1418 (
1419 registration.manifest.module_id.clone(),
1420 registration.manifest,
1421 )
1422 })
1423 .collect::<BTreeMap<_, _>>();
1424 let census = match self.forwarding.route_census(None) {
1425 Ok(census) => census,
1426 Err(err) => {
1427 warn!(error = %err, "failed to read route census for capability deny enforcement");
1428 return;
1429 }
1430 };
1431
1432 for (target_module_id, routes) in census {
1433 let Some(target_manifest) = manifests.get(&target_module_id) else {
1434 continue;
1435 };
1436 let mut closed_routes = Vec::new();
1437 let mut module_goodbyes = Vec::new();
1438 for route in routes {
1439 let Principal::Reserved {
1440 module_id: opening_module_id,
1441 } = &route.principal
1442 else {
1443 continue;
1444 };
1445 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1446 continue;
1447 };
1448 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1449 continue;
1450 };
1451
1452 match self.forwarding.release_client_route(
1453 route.goodbye_target.connection_id,
1454 route.goodbye_target.channel,
1455 route.goodbye_target.epoch,
1456 ) {
1457 Ok(RouteRelease::Removed(module_goodbye)) => {
1458 warn!(
1459 opening_module_id,
1460 target_module_id,
1461 capability,
1462 "force-closing route because an attested capability deny edge now matches"
1463 );
1464 closed_routes.push(route);
1465 module_goodbyes.push(module_goodbye);
1466 }
1467 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1468 Err(err) => warn!(
1469 opening_module_id,
1470 target_module_id,
1471 capability,
1472 error = %err,
1473 "failed to force-close capability-denied route"
1474 ),
1475 }
1476 }
1477
1478 if closed_routes.is_empty() {
1479 continue;
1480 }
1481 send_route_control_pushes(
1482 &self.forwarding,
1483 closed_routes,
1484 ClientControlPush::RouteClosed {
1485 module_id: target_module_id,
1486 channels: Vec::new(),
1487 reason: RouteCloseReason::CapabilityDenied,
1488 drained: false,
1489 abandoned: 0,
1490 excluded_subscriptions: 0,
1491 terminal: Some(false),
1492 },
1493 );
1494 self.emit_route_goodbyes(module_goodbyes);
1495 }
1496 }
1497
1498 fn not_ready_reason(
1504 &self,
1505 registration: &crate::registry::ModuleRegistration,
1506 ) -> Option<NotReadyReason> {
1507 if !registration.ready {
1508 return Some(NotReadyReason {
1509 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1510 capability: None,
1511 });
1512 }
1513 self.first_unprovided_required_capability(registration)
1514 .map(|capability| NotReadyReason {
1515 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1516 capability: Some(capability),
1517 })
1518 }
1519
1520 fn first_unprovided_required_capability(
1538 &self,
1539 registration: &crate::registry::ModuleRegistration,
1540 ) -> Option<String> {
1541 let required = registration
1542 .manifest
1543 .capabilities
1544 .iter()
1545 .flat_map(|declarations| declarations.requires.iter())
1546 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1547 .map(|requirement| requirement.capability.as_str())
1548 .collect::<BTreeSet<_>>();
1549 if required.is_empty() {
1550 return None;
1551 }
1552 let module_id = registration.manifest.module_id.as_str();
1553 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1554 if required
1555 .iter()
1556 .any(|capability| verdict(capability).is_none())
1557 {
1558 self.refresh_capability_requirements();
1559 }
1560 required
1561 .into_iter()
1562 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1563 .map(str::to_string)
1564 }
1565
1566 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1567 self.capability_evaluator
1568 .statuses()
1569 .into_iter()
1570 .map(capability_requirement_status)
1571 .collect()
1572 }
1573
1574 fn deregister_connection(
1581 &self,
1582 connection_id: ConnectionId,
1583 reason: RegistrationEndReason,
1584 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1585 self.registry
1586 .deregister_connection_with_reason(connection_id, reason)
1587 }
1588
1589 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1590 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1591 return None;
1592 }
1593 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1594 parse_client_control_request(&frame.body)
1595 else {
1596 return None;
1597 };
1598 Some(target_module_id(&target).to_string())
1599 }
1600
1601 pub(crate) fn route_open_capacity_refusal(
1602 &self,
1603 ctx: &RouteCtx,
1604 frame: &Frame,
1605 target_module_id: &str,
1606 in_flight: usize,
1607 limit: usize,
1608 ) -> Result<Frame, RouterError> {
1609 self.route_open_admission_refusal_frame(
1610 ctx,
1611 frame,
1612 target_module_id,
1613 "open_admission_full",
1614 (in_flight, limit),
1615 format!(
1616 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1617 ),
1618 )
1619 }
1620
1621 fn route_open_target_capacity_refusal(
1622 &self,
1623 ctx: &RouteCtx,
1624 frame: &Frame,
1625 target_module_id: &str,
1626 in_flight: usize,
1627 ) -> Result<Frame, RouterError> {
1628 self.route_open_admission_refusal_frame(
1629 ctx,
1630 frame,
1631 target_module_id,
1632 "target_binds_full",
1633 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1634 format!(
1635 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1636 ),
1637 )
1638 }
1639
1640 fn route_open_admission_refusal_frame(
1648 &self,
1649 ctx: &RouteCtx,
1650 frame: &Frame,
1651 target_module_id: &str,
1652 reason: &'static str,
1653 (in_flight, limit): (usize, usize),
1654 message: impl Into<String>,
1655 ) -> Result<Frame, RouterError> {
1656 let code = error_codes::TARGET_UNAVAILABLE;
1657 self.counters.increment_route_open_refused(code);
1658 info!(
1659 target: "control",
1660 code,
1661 reason,
1662 module_id = ?target_module_id,
1663 connection_id = ctx.connection_id.get(),
1664 in_flight,
1665 limit,
1666 "route.open refused"
1667 );
1668 control_error_frame(frame, code, message.into())
1669 }
1670
1671 #[cfg(test)]
1677 pub fn handle_control(
1678 &self,
1679 connection_id: ConnectionId,
1680 frame: Frame,
1681 ) -> Result<Vec<Frame>, RouterError> {
1682 match frame.header.ty {
1683 FrameType::Ping => Ok(vec![pong(&frame)?]),
1684 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1685 FrameType::Goodbye => self.handle_goodbye(connection_id),
1686 ty => Ok(vec![control_error_frame(
1687 &frame,
1688 "unsupported_control_frame",
1689 format!("unsupported channel-0 frame {ty:?}"),
1690 )?]),
1691 }
1692 }
1693
1694 pub async fn handle_control_frame(
1695 &self,
1696 ctx: &RouteCtx,
1697 frame: Frame,
1698 ) -> Result<Vec<Frame>, RouterError> {
1699 self.handle_control_frame_timed(ctx, frame, None).await
1700 }
1701
1702 pub(crate) fn record_caller_endpoints(
1703 &self,
1704 connection: ConnectionId,
1705 endpoints: Option<(SocketAddr, SocketAddr)>,
1706 ) {
1707 let mut callers = self
1708 .caller_endpoints
1709 .lock()
1710 .unwrap_or_else(|p| p.into_inner());
1711 callers.remove(&connection);
1712 if let Some(endpoints) = endpoints {
1713 callers.insert(connection, endpoints);
1714 }
1715 }
1716
1717 #[cfg(test)]
1718 pub(crate) fn with_caller_lookup_for_test(
1719 mut self,
1720 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1721 ) -> Self {
1722 self.caller_lookup = Some(Arc::new(lookup));
1723 self
1724 }
1725
1726 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1727 let endpoints = self
1728 .caller_endpoints
1729 .lock()
1730 .unwrap_or_else(|p| p.into_inner())
1731 .get(&connection)
1732 .copied();
1733 #[cfg(test)]
1734 let lookup = self.caller_lookup.clone();
1735 tokio::task::spawn_blocking(move || {
1736 #[cfg(test)]
1737 if let Some(lookup) = lookup {
1738 return lookup(endpoints);
1739 }
1740 endpoints
1741 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1742 .unwrap_or_default()
1743 })
1744 .await
1745 .unwrap_or_default()
1746 }
1747
1748 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1749 frame.header.channel == 0
1750 && frame.header.ty == FrameType::Request
1751 && self
1752 .forwarding
1753 .module_endpoint_for_connection(ctx.connection_id)
1754 .is_ok_and(|endpoint| endpoint.is_none())
1755 && parse_client_control_request(&frame.body)
1756 .ok()
1757 .is_some_and(|request| privileged_operation(&request).is_some())
1758 }
1759
1760 pub(crate) async fn handle_control_frame_timed(
1761 &self,
1762 ctx: &RouteCtx,
1763 frame: Frame,
1764 dispatch_started_at: Option<StdInstant>,
1765 ) -> Result<Vec<Frame>, RouterError> {
1766 match frame.header.ty {
1767 FrameType::Ping => Ok(vec![pong(&frame)?]),
1768 FrameType::Hello => {
1769 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1770 }
1771 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1772 FrameType::Cancel => {
1773 if self
1776 .forwarding
1777 .operator_confirms()
1778 .cancel(ctx.connection_id, frame.header.corr)
1779 || self
1780 .supervisor
1781 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1782 || self.cancel_activity_subscription(ctx, &frame)?
1783 {
1784 Ok(Vec::new())
1785 } else {
1786 Ok(vec![control_error_frame(
1787 &frame,
1788 "unknown_subscription",
1789 "no subscription has this correlation id",
1790 )?])
1791 }
1792 }
1793 FrameType::Request => {
1794 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1797 if op
1798 .as_ref()
1799 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1800 {
1801 return self.handle_activity_record(ctx, frame).await;
1802 }
1803 if op.as_ref().is_some_and(|probe| {
1804 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1805 }) {
1806 return self.handle_activity_read(ctx, frame).await;
1807 }
1808 if op
1809 .as_ref()
1810 .is_some_and(|probe| probe.op == "operator.confirm")
1811 {
1812 return self.handle_operator_confirm(ctx, frame);
1813 }
1814 if self
1815 .forwarding
1816 .module_endpoint_for_connection(ctx.connection_id)
1817 .map_err(RouterError::Forwarding)?
1818 .is_some()
1819 {
1820 if !is_known_module_request_op(&frame.body) {
1821 return Ok(vec![control_error_frame(
1822 &frame,
1823 "unsupported_control_frame",
1824 "module-originated channel-0 REQUEST is not supported",
1825 )?]);
1826 }
1827 let request = match parse_module_control_request_from_module(&frame.body) {
1828 Ok(request) => request,
1829 Err((err, ControlRequestBodyError::UnknownOp)) => {
1830 return Ok(vec![control_error_frame(
1831 &frame,
1832 "unsupported_control_frame",
1833 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1834 )?])
1835 }
1836 Err((err, ControlRequestBodyError::InvalidBody)) => {
1837 return Ok(vec![control_error_frame(
1838 &frame,
1839 "invalid_control_body",
1840 format!("malformed module control body: {err}"),
1841 )?])
1842 }
1843 };
1844 let op = module_control_request_op(&request);
1845 let corr = frame.header.corr;
1846 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1847 let result =
1848 self.handle_module_control_request(ctx.connection_id, frame, request);
1849 log_slow_control_dispatch(
1850 dispatch_started_at,
1851 op,
1852 ctx.connection_id,
1853 corr,
1854 None,
1855 );
1856 return result;
1857 }
1858
1859 if is_known_module_request_op(&frame.body) {
1860 return Ok(vec![control_error_frame(
1861 &frame,
1862 "not_registered",
1863 "catalog.update requires an active module registration owned by this connection",
1864 )?]);
1865 }
1866
1867 let request = match parse_client_control_request(&frame.body) {
1868 Ok(request) => request,
1869 Err((err, ControlRequestBodyError::UnknownOp)) => {
1870 return Ok(vec![control_error_frame(
1871 &frame,
1872 "unknown_control_op",
1873 format!("unknown client control op: {err}"),
1874 )?])
1875 }
1876 Err((err, ControlRequestBodyError::InvalidBody)) => {
1877 return Ok(vec![control_error_frame(
1878 &frame,
1879 "invalid_control_body",
1880 format!("malformed client control body: {err}"),
1881 )?])
1882 }
1883 };
1884 let op = client_control_request_op(&request);
1885 let corr = frame.header.corr;
1886 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1887 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1888 #[cfg(test)]
1889 if let Some(delay) = self.control_dispatch_delay {
1890 tokio::time::sleep(delay).await;
1891 }
1892 let result = self
1893 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1894 .await;
1895 if let Some(timings) = route_open_timings.as_mut() {
1896 timings.finish();
1897 }
1898 log_slow_control_dispatch(
1899 dispatch_started_at,
1900 op,
1901 ctx.connection_id,
1902 corr,
1903 route_open_timings.as_ref(),
1904 );
1905 result
1906 }
1907 FrameType::Push => {
1908 let Some(endpoint) = self
1909 .forwarding
1910 .module_endpoint_for_connection(ctx.connection_id)
1911 .map_err(RouterError::Forwarding)?
1912 else {
1913 return Ok(vec![control_error_frame(
1914 &frame,
1915 "unsupported_control_frame",
1916 "client-originated channel-0 PUSH is not supported",
1917 )?]);
1918 };
1919 self.handle_status_update(endpoint, frame)
1920 }
1921 FrameType::Response | FrameType::Error
1922 if self
1923 .forwarding
1924 .module_endpoint_for_connection(ctx.connection_id)
1925 .map_err(RouterError::Forwarding)?
1926 .is_some() =>
1927 {
1928 self.handle_module_relay_response(ctx.connection_id, frame)
1929 }
1930 ty => Ok(vec![control_error_frame(
1931 &frame,
1932 "unsupported_control_frame",
1933 format!("unsupported channel-0 frame {ty:?}"),
1934 )?]),
1935 }
1936 }
1937
1938 pub fn cleanup_connection(
1939 &self,
1940 connection_id: ConnectionId,
1941 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1942 self.caller_endpoints
1943 .lock()
1944 .unwrap_or_else(|p| p.into_inner())
1945 .remove(&connection_id);
1946 self.forwarding
1947 .operator_confirms()
1948 .connection_closed(connection_id);
1949 self.cleanup_connection_with_end_reason(
1950 connection_id,
1951 RegistrationEndReason::ConnectionClosed,
1952 )
1953 }
1954
1955 fn cleanup_connection_with_end_reason(
1956 &self,
1957 connection_id: ConnectionId,
1958 requested_reason: RegistrationEndReason,
1959 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1960 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1961 self.registry
1962 .get_module_by_connection(connection_id)?
1963 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
1964 .and_then(|module| module.registration_end_reason().ok().flatten())
1965 .unwrap_or(requested_reason)
1966 } else {
1967 requested_reason
1968 };
1969 let crash_closed = self
1970 .registry
1971 .get_module_by_connection(connection_id)?
1972 .and_then(|registration| {
1973 self.forwarding
1974 .module_endpoint_for_connection(connection_id)
1975 .ok()
1976 .flatten()
1977 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1978 .map(|routes| (registration.manifest.module_id, routes))
1979 });
1980 let crash_closed = crash_closed.map(|(module_id, routes)| {
1981 let terminal = match self.supervisor.get(&module_id) {
1982 None => false,
1983 Some(module) => match module.will_recover_after_connection_loss() {
1984 Ok(will_recover) => !will_recover,
1985 Err(err) => {
1986 warn!(
1987 %module_id,
1988 error = %err,
1989 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1990 );
1991 false
1992 }
1993 },
1994 };
1995 let (reason, drained) = match self
1999 .forwarding
2000 .daemon_drain_result_for_connection(connection_id)
2001 {
2002 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2003 Ok(None) => (RouteCloseReason::Crash, false),
2004 Err(err) => {
2005 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2006 (RouteCloseReason::Crash, false)
2007 }
2008 };
2009 (module_id, routes, reason, drained, terminal)
2010 });
2011 let registrations = self.deregister_connection(connection_id, end_reason);
2012 let cleanup = if crash_closed.is_some() {
2013 self.forwarding.cleanup_connection_counted(connection_id)
2014 } else {
2015 self.forwarding
2021 .cleanup_connection(connection_id)
2022 .map(|released| crate::forwarding::ConnectionCleanup {
2023 released,
2024 abandoned_relays: 0,
2025 })
2026 };
2027 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2032 let abandoned = cleanup
2033 .as_ref()
2034 .map_or(0, |cleanup| cleanup.abandoned_relays);
2035 send_route_control_pushes(
2036 &self.forwarding,
2037 routes,
2038 ClientControlPush::RouteClosed {
2039 module_id,
2040 channels: Vec::new(),
2041 reason,
2042 drained,
2043 abandoned,
2044 excluded_subscriptions: 0,
2045 terminal: Some(terminal),
2046 },
2047 );
2048 }
2049 if let Ok(cleanup) = cleanup {
2050 self.emit_route_goodbyes(cleanup.released);
2051 }
2052 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2056 crate::supervise::notify_registration_release();
2057 self.capability_evaluator.wake_deadline_loop();
2058 self.refresh_capability_requirements();
2059 }
2060 self.supervisor.remove_spawn_subscribers(connection_id);
2061 self.activity_streams
2062 .lock()
2063 .unwrap_or_else(|p| p.into_inner())
2064 .retain(|(id, _), task| {
2065 if *id == connection_id {
2066 task.abort();
2067 false
2068 } else {
2069 true
2070 }
2071 });
2072 self.hello_launch_nonces
2075 .lock()
2076 .unwrap_or_else(|poisoned| poisoned.into_inner())
2077 .forget(connection_id);
2078 self.scopes
2079 .write()
2080 .unwrap_or_else(|poisoned| poisoned.into_inner())
2081 .release_connection(connection_id);
2082 registrations
2083 }
2084
2085 pub(crate) fn handle_route_goodbye(
2086 &self,
2087 connection_id: ConnectionId,
2088 route_channel: u16,
2089 route_epoch: u32,
2090 ) -> Result<bool, RouterError> {
2091 debug!(
2092 connection_id = connection_id.get(),
2093 route_channel, route_epoch, "handling route GOODBYE"
2094 );
2095 let RouteRelease::Removed(released_route) = self
2096 .forwarding
2097 .release_client_route(connection_id, route_channel, route_epoch)
2098 .map_err(RouterError::Forwarding)?
2099 else {
2100 return Ok(false);
2101 };
2102 self.emit_route_goodbyes(vec![released_route]);
2103 Ok(true)
2104 }
2105
2106 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2107 for released in released_routes {
2108 let frame = match Frame::build_with_version(
2109 released.negotiated_ver,
2110 FrameType::Goodbye,
2111 control_flags(),
2112 released.channel,
2113 released.epoch,
2114 0,
2115 Vec::new(),
2116 ) {
2117 Ok(frame) => frame,
2118 Err(err) => {
2119 warn!(
2120 route_channel = released.channel,
2121 error = %err,
2122 "failed to build route GOODBYE frame"
2123 );
2124 continue;
2125 }
2126 };
2127 if !released.close_on_delivery_failure() {
2128 crate::forwarding::send_module_route_goodbye(
2129 &self.counters,
2130 &released.sink,
2131 frame,
2132 released.module_id.as_deref(),
2133 "client route released",
2134 );
2135 continue;
2136 }
2137 if let Err(err) = released.sink.try_send(frame) {
2138 warn!(
2139 target_connection_id = released.connection_id.get(),
2140 route_channel = released.channel,
2141 error = %err,
2142 "route GOODBYE was not delivered to client; closing target connection"
2143 );
2144 if self
2145 .forwarding
2146 .escalate_client_delivery_failure(
2147 released.connection_id,
2148 released.channel,
2149 released.epoch,
2150 CloseReason::new(
2151 "route_goodbye_delivery_failed",
2152 format!(
2153 "failed to enqueue route GOODBYE for channel {}: {err}",
2154 released.channel
2155 ),
2156 ),
2157 crate::forwarding::UndeliveredFrame {
2158 module_id: released.module_id.as_deref(),
2159 sink: &released.sink,
2160 },
2161 )
2162 .unwrap_or(false)
2163 {
2164 self.counters.increment_goodbye_relay_client_failed();
2165 }
2166 }
2167 }
2168 }
2169
2170 fn send_abandoned_route_bind_goodbye(
2182 &self,
2183 module_sink: &crate::FrameSink,
2184 negotiated_ver: u8,
2185 module_channel: u16,
2186 module_epoch: u32,
2187 ) {
2188 let frame = match Frame::build_with_version(
2189 negotiated_ver,
2190 FrameType::Goodbye,
2191 control_flags(),
2192 module_channel,
2193 module_epoch,
2194 0,
2195 Vec::new(),
2196 ) {
2197 Ok(frame) => frame,
2198 Err(err) => {
2199 warn!(
2200 route_channel = module_channel,
2201 error = %err,
2202 "failed to build GOODBYE for abandoned route.bind"
2203 );
2204 return;
2205 }
2206 };
2207 crate::forwarding::send_module_route_goodbye(
2208 &self.counters,
2209 module_sink,
2210 frame,
2211 None,
2212 "abandoned route.bind",
2213 );
2214 }
2215
2216 fn handle_hello(
2217 &self,
2218 connection_id: ConnectionId,
2219 sink: Option<crate::FrameSink>,
2220 frame: Frame,
2221 ) -> Result<Vec<Frame>, RouterError> {
2222 debug!(
2223 connection_id = connection_id.get(),
2224 corr = frame.header.corr,
2225 "handling HELLO"
2226 );
2227 if self
2231 .registry
2232 .get_module_by_connection(connection_id)
2233 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2234 .is_some()
2235 {
2236 return Ok(vec![control_error_frame(
2237 &frame,
2238 "invalid_hello",
2239 "connection is already registered as a module",
2240 )?]);
2241 }
2242 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2243 Ok(value) => value,
2244 Err(err) => {
2245 return Ok(vec![control_error_frame(
2246 &frame,
2247 "invalid_hello",
2248 format!("malformed HELLO body: {err}"),
2249 )?])
2250 }
2251 };
2252 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2253 return Ok(vec![control_error_frame(
2254 &frame,
2255 "invalid_capability_grammar",
2256 err.to_string(),
2257 )?]);
2258 }
2259 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2260 return Ok(vec![control_error_frame(
2261 &frame,
2262 "invalid_manifest",
2263 err.to_string(),
2264 )?]);
2265 }
2266 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2267 return Ok(vec![control_error_frame(
2268 &frame,
2269 "invalid_event_declaration",
2270 err.to_string(),
2271 )?]);
2272 }
2273 if let Some(provenance) = hello_value
2274 .get("manifest")
2275 .and_then(|manifest| manifest.get("provenance"))
2276 {
2277 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2278 return Ok(vec![control_error_frame(
2279 &frame,
2280 "invalid_manifest",
2281 format!("malformed manifest provenance: {err}"),
2282 )?]);
2283 }
2284 }
2285 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2286 Ok(hello) => hello,
2287 Err(err) => {
2288 return Ok(vec![control_error_frame(
2289 &frame,
2290 "invalid_hello",
2291 format!("malformed HELLO body: {err}"),
2292 )?])
2293 }
2294 };
2295
2296 if hello.protocol_ver != hello.manifest.protocol_ver {
2297 return Ok(vec![control_error_frame(
2298 &frame,
2299 "invalid_manifest",
2300 format!(
2301 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2302 hello.protocol_ver, hello.manifest.protocol_ver
2303 ),
2304 )?]);
2305 }
2306
2307 if hello.manifest.module_id.trim().is_empty() {
2308 return Ok(vec![control_error_frame(
2309 &frame,
2310 "invalid_manifest",
2311 "manifest module_id must not be empty",
2312 )?]);
2313 }
2314
2315 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2316 Ok(negotiated_ver) => negotiated_ver,
2317 Err(message) => {
2318 return Ok(vec![control_error_frame(
2319 &frame,
2320 "version_unsupported",
2321 message,
2322 )?])
2323 }
2324 };
2325
2326 let swap_admission = self
2334 .supervisor
2335 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2336 if swap_admission == SwapHelloAdmission::Refused {
2337 warn!(
2338 module_id = %hello.manifest.module_id,
2339 connection_id = connection_id.get(),
2340 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2341 );
2342 return Ok(vec![control_error_frame(
2343 &frame,
2344 "swap_token_invalid",
2345 format!(
2346 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2347 hello.manifest.module_id
2348 ),
2349 )?]);
2350 }
2351 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2352
2353 if let Some(rejection) = (!swap_candidate)
2361 .then(|| {
2362 self.supervisor.reserved_hello_rejection(
2363 &hello.manifest.module_id,
2364 hello.launch_nonce.as_deref(),
2365 )
2366 })
2367 .flatten()
2368 {
2369 let message = match rejection {
2370 ReservedHelloRejection::Exact { module_id } => format!(
2371 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2372 ),
2373 ReservedHelloRejection::Prefix {
2374 prefix,
2375 owner_module_id,
2376 } => format!(
2377 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2378 hello.manifest.module_id
2379 ),
2380 };
2381 return Ok(vec![control_error_frame(
2382 &frame,
2383 "reserved_module",
2384 message,
2385 )?]);
2386 }
2387
2388 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2389 &hello.manifest.module_id,
2390 hello.manifest.capabilities.as_ref(),
2391 );
2392 if let Some(refusal) = reserved_capability_refusals.first() {
2393 let capability = refusal.capability.clone();
2394 let bound_module = refusal.claimants[0].clone();
2395 log_duplicate_claim_events(reserved_capability_refusals);
2396 return Ok(vec![control_error_frame(
2397 &frame,
2398 "reserved_capability",
2399 format!(
2400 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2401 capability, bound_module, hello.manifest.module_id
2402 ),
2403 )?]);
2404 }
2405
2406 if self
2409 .forwarding
2410 .connection_has_client_routes(connection_id)
2411 .map_err(RouterError::Forwarding)?
2412 {
2413 return Ok(vec![control_error_frame(
2414 &frame,
2415 "invalid_hello",
2416 "connection has open client routes and cannot also register as a module",
2417 )?]);
2418 }
2419
2420 self.hello_launch_nonces
2425 .lock()
2426 .unwrap_or_else(|poisoned| poisoned.into_inner())
2427 .record(connection_id, hello.launch_nonce.as_deref());
2428 let control_ops = effective_module_control_ops(hello.control_ops);
2429 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2432 if swap_candidate {
2433 return self.register_swap_candidate(
2434 connection_id,
2435 sink,
2436 &frame,
2437 hello.manifest,
2438 negotiated_ver,
2439 control_ops,
2440 hello_ack,
2441 );
2442 }
2443 let registration = match self.registry.register_unpublished_with_control_ops(
2450 hello.manifest,
2451 negotiated_ver,
2452 connection_id,
2453 control_ops,
2454 ) {
2455 Ok(registration) => registration,
2456 Err(RegistryError::DuplicateModuleId { module_id }) => {
2457 return Ok(vec![control_error_frame(
2458 &frame,
2459 "duplicate_module_id",
2460 format!(
2461 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2462 ),
2463 )?])
2464 }
2465 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2466 return Ok(vec![control_error_frame(
2467 &frame,
2468 "invalid_module_id",
2469 err.to_string(),
2470 )?])
2471 }
2472 Err(err) => {
2473 return Ok(vec![control_error_frame(
2474 &frame,
2475 "registry_error",
2476 err.to_string(),
2477 )?])
2478 }
2479 };
2480 #[cfg(test)]
2481 self.hold_between_registry_and_forwarding();
2482
2483 let reply = if let Some(sink) = sink {
2484 let concurrency = manifest_concurrency(®istration.manifest);
2500 if let Err(err) = self.forwarding.register_module_connection_acked(
2501 connection_id,
2502 registration.manifest.module_id.clone(),
2503 negotiated_ver,
2504 concurrency,
2505 sink,
2506 hello_ack,
2507 ) {
2508 if matches!(
2513 self.deregister_connection(
2514 connection_id,
2515 RegistrationEndReason::RegistrationFailed,
2516 ),
2517 Ok(r) if !r.is_empty()
2518 ) {
2519 crate::supervise::notify_registration_release();
2520 }
2521 return Ok(vec![control_error_frame(
2522 &frame,
2523 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2524 "invalid_hello"
2525 } else {
2526 forwarding_error_code(&err)
2527 },
2528 err.to_string(),
2529 )?]);
2530 }
2531 Vec::new()
2532 } else {
2533 vec![hello_ack]
2536 };
2537
2538 if !self
2543 .registry
2544 .publish_registration(connection_id)
2545 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2546 {
2547 debug!(
2548 module_id = %registration.manifest.module_id,
2549 connection_id = connection_id.get(),
2550 "registration was removed before it could be published"
2551 );
2552 }
2553
2554 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2563 info!(
2564 module_id = %registration.manifest.module_id,
2565 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2566 );
2567 }
2568
2569 self.apply_registration_capabilities(®istration);
2570
2571 self.supervisor
2572 .record_activity_registration(®istration.manifest.module_id);
2573
2574 info!(
2575 module_id = %registration.manifest.module_id,
2576 module_version = %registration.manifest.module_version,
2577 negotiated_ver,
2578 routable_provider = manifest_provides_routable_role(®istration.manifest),
2579 connection_id = connection_id.get(),
2580 "module registered"
2581 );
2582
2583 Ok(reply)
2584 }
2585
2586 #[allow(clippy::too_many_arguments)]
2603 fn register_swap_candidate(
2604 &self,
2605 connection_id: ConnectionId,
2606 sink: Option<crate::FrameSink>,
2607 frame: &Frame,
2608 manifest: ModuleManifest,
2609 negotiated_ver: u8,
2610 control_ops: Vec<String>,
2611 hello_ack: Frame,
2612 ) -> Result<Vec<Frame>, RouterError> {
2613 let module_id = manifest.module_id.clone();
2614 let registration = match self.registry.register_unpublished_candidate_with_control_ops(
2615 manifest,
2616 negotiated_ver,
2617 connection_id,
2618 control_ops,
2619 ) {
2620 Ok(registration) => registration,
2621 Err(RegistryError::DuplicateModuleId { module_id }) => {
2622 return Ok(vec![control_error_frame(
2623 frame,
2624 "duplicate_module_id",
2625 format!(
2626 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2627 ),
2628 )?])
2629 }
2630 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2631 return Ok(vec![control_error_frame(
2632 frame,
2633 "invalid_module_id",
2634 err.to_string(),
2635 )?])
2636 }
2637 Err(err) => {
2638 return Ok(vec![control_error_frame(
2639 frame,
2640 "registry_error",
2641 err.to_string(),
2642 )?])
2643 }
2644 };
2645 #[cfg(test)]
2646 self.hold_between_registry_and_forwarding();
2647 let reply = if let Some(sink) = sink {
2648 let concurrency = manifest_concurrency(®istration.manifest);
2652 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2653 connection_id,
2654 module_id.clone(),
2655 negotiated_ver,
2656 concurrency,
2657 sink,
2658 hello_ack,
2659 ) {
2660 if matches!(
2661 self.deregister_connection(
2662 connection_id,
2663 RegistrationEndReason::RegistrationFailed,
2664 ),
2665 Ok(r) if !r.is_empty()
2666 ) {
2667 crate::supervise::notify_registration_release();
2668 }
2669 return Ok(vec![control_error_frame(
2670 frame,
2671 forwarding_error_code(&err),
2672 err.to_string(),
2673 )?]);
2674 }
2675 Vec::new()
2676 } else {
2677 vec![hello_ack]
2678 };
2679 if !self
2680 .registry
2681 .publish_registration(connection_id)
2682 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2683 {
2684 debug!(
2685 module_id = %module_id,
2686 connection_id = connection_id.get(),
2687 "swap candidate registration was removed before it could be published"
2688 );
2689 }
2690 self.supervisor.mark_swap_candidate_admitted(&module_id);
2691 info!(
2692 module_id = %module_id,
2693 module_version = %registration.manifest.module_version,
2694 negotiated_ver,
2695 ready = registration.ready,
2696 connection_id = connection_id.get(),
2697 "swap candidate registered; not routable until cutover"
2698 );
2699 Ok(reply)
2700 }
2701
2702 fn build_hello_ack(
2703 &self,
2704 frame: &Frame,
2705 negotiated_ver: u8,
2706 module_id: &str,
2707 ) -> Result<Frame, RouterError> {
2708 let ack = ModuleHelloAckBody {
2709 negotiated_ver,
2710 subc_ops: module_subc_ops(),
2711 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2712 storage: self
2713 .storage_config
2714 .as_ref()
2715 .map(|cfg| cfg.descriptor_for(module_id)),
2716 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2717 };
2718 let body = serde_json::to_vec(&ack).map_err(|err| {
2719 RouterError::backend(
2720 0,
2721 frame.header.corr,
2722 format!("failed to encode HELLO_ACK: {err}"),
2723 )
2724 })?;
2725
2726 Frame::build_with_version(
2727 negotiated_ver,
2728 FrameType::HelloAck,
2729 control_flags(),
2730 0,
2731 0,
2732 frame.header.corr,
2733 body,
2734 )
2735 .map_err(RouterError::FrameBuild)
2736 }
2737
2738 async fn handle_client_control_request(
2739 &self,
2740 ctx: &RouteCtx,
2741 frame: Frame,
2742 request: ClientControlRequest,
2743 route_open_timings: &mut Option<RouteOpenTimings>,
2744 ) -> Result<Vec<Frame>, RouterError> {
2745 if let Some(operation) = privileged_operation(&request) {
2746 let caller = self.resolve_caller(ctx.connection_id).await;
2747 let op = client_control_request_op(&request);
2748 let text = prompt_text(operation, &caller);
2749 match self.privileged_operator_prompt {
2750 PrivilegedOperatorPrompt::Log => {
2751 info!(target: "subc_daemon::privileged_prompt", operation = op,
2752 target = ?operation, caller = %caller.description(), summary = %text,
2753 "privileged request would prompt the operator");
2754 }
2755 PrivilegedOperatorPrompt::Enforce => {
2756 let key =
2757 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2758 let outcome = self
2759 .forwarding
2760 .operator_confirms()
2761 .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2762 .await;
2763 if outcome.code != "confirmed" {
2764 return Ok(vec![outcome.frame(&frame)]);
2765 }
2766 }
2767 }
2768 }
2769 match request {
2770 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2771 ClientControlRequest::CatalogList { module_id } => {
2772 self.handle_catalog_list(frame, module_id)
2773 }
2774 ClientControlRequest::RouteOpen {
2775 target,
2776 identity,
2777 consumer_identity,
2778 consumer_capabilities,
2779 role_versions,
2780 admission_facts,
2781 scope,
2782 } => {
2783 self.handle_route_open(
2784 ctx,
2785 frame,
2786 RouteOpenRequest {
2787 target,
2788 identity,
2789 consumer_identity,
2790 consumer_capabilities,
2791 role_versions,
2792 admission_facts,
2793 scope,
2794 },
2795 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2796 )
2797 .await
2798 }
2799 ClientControlRequest::RoutePoll {
2800 route_channel,
2801 route_epoch,
2802 kind,
2803 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2804 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2805 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2806 self.handle_supervisor_spawn_snapshot(frame)
2807 }
2808 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2809 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2810 }
2811 ClientControlRequest::SupervisorRestart {
2812 module_id,
2813 drain_timeout_ms,
2814 } => {
2815 self.log_supervisor_request_received(
2816 ctx,
2817 frame.header.corr,
2818 ops::SUPERVISOR_RESTART,
2819 Some(&module_id),
2820 None,
2821 )?;
2822 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2823 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2824 .await
2825 }
2826 ClientControlRequest::SupervisorSwap {
2827 module_id,
2828 ready_timeout_ms,
2829 } => {
2830 self.log_supervisor_request_received(
2831 ctx,
2832 frame.header.corr,
2833 ops::SUPERVISOR_SWAP,
2834 Some(&module_id),
2835 None,
2836 )?;
2837 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2838 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2839 .await
2840 }
2841 ClientControlRequest::SupervisorReload { module_id } => {
2842 self.log_supervisor_request_received(
2843 ctx,
2844 frame.header.corr,
2845 ops::SUPERVISOR_RELOAD,
2846 Some(&module_id),
2847 None,
2848 )?;
2849 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2850 self.handle_supervisor_reload(frame, module_id, requester)
2851 .await
2852 }
2853 ClientControlRequest::SupervisorRescan { preview } => {
2854 if !preview {
2855 self.log_supervisor_request_received(
2856 ctx,
2857 frame.header.corr,
2858 ops::SUPERVISOR_RESCAN,
2859 None,
2860 None,
2861 )?;
2862 }
2863 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2864 self.handle_supervisor_rescan(frame, preview, requester)
2865 .await
2866 }
2867 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2868 self.log_supervisor_request_received(
2869 ctx,
2870 frame.header.corr,
2871 ops::SUPERVISOR_RELEASE_RESERVED,
2872 Some(&module_id),
2873 None,
2874 )?;
2875 self.handle_supervisor_release_reserved(frame, module_id)
2876 .await
2877 }
2878 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2879 self.log_supervisor_request_received(
2880 ctx,
2881 frame.header.corr,
2882 ops::SUPERVISOR_SET_ENABLED,
2883 Some(&module_id),
2884 Some(enabled),
2885 )?;
2886 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2887 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2888 .await
2889 }
2890 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2891 self.handle_supervisor_health_probe(frame, module_id).await
2892 }
2893 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2894 ClientControlRequest::SupervisorRoutes { module_id } => {
2895 self.handle_supervisor_routes(frame, module_id)
2896 }
2897 ClientControlRequest::SupervisorProvenance { module_id } => {
2898 self.handle_supervisor_provenance(frame, module_id).await
2899 }
2900 ClientControlRequest::SupervisorStderrTail {
2901 module_id,
2902 max_lines,
2903 max_bytes,
2904 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2905 ClientControlRequest::SupervisorTerminals { module_id } => {
2906 self.handle_supervisor_terminals(frame, module_id).await
2907 }
2908 }
2909 }
2910
2911 fn log_supervisor_request_received(
2912 &self,
2913 ctx: &RouteCtx,
2914 corr: u64,
2915 op: &'static str,
2916 module_id: Option<&str>,
2917 enabled: Option<bool>,
2918 ) -> Result<(), RouterError> {
2919 let caller = self.supervisor_requester(ctx, corr)?;
2920 let caller = principal_label(&caller);
2921 info!(
2924 target: "control",
2925 op,
2926 module_id,
2927 enabled,
2928 connection_id = ctx.connection_id.get(),
2929 caller = %caller,
2930 "supervisor request received"
2931 );
2932 Ok(())
2933 }
2934
2935 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
2936 Ok(self
2937 .registry
2938 .get_module_by_connection(ctx.connection_id)
2939 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2940 .map(|registration| Principal::Reserved {
2941 module_id: registration.manifest.module_id,
2942 })
2943 .unwrap_or(Principal::Direct))
2944 }
2945
2946 fn handle_module_control_request(
2947 &self,
2948 connection_id: ConnectionId,
2949 frame: Frame,
2950 request: ModuleControlRequestFromModule,
2951 ) -> Result<Vec<Frame>, RouterError> {
2952 match request {
2953 ModuleControlRequestFromModule::CatalogUpdate {
2954 provides,
2955 capabilities,
2956 ready,
2957 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2958 ModuleControlRequestFromModule::LiveRoots {} => {
2959 let registered = self
2960 .registry
2961 .get_module_by_connection(connection_id)
2962 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2963 let Some(registration) = registered else {
2964 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2965 };
2966 let response = self
2967 .forwarding
2968 .live_roots(®istration.manifest.module_id)
2969 .map_err(RouterError::Forwarding)?;
2970 Ok(vec![control_response_body_frame(
2971 &frame,
2972 &response,
2973 "ModuleControlResponseToModule::LiveRoots",
2974 )?])
2975 }
2976 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2977 self.handle_scope_sync(connection_id, frame, generation, scopes)
2978 }
2979 ModuleControlRequestFromModule::ScopeApply {
2980 generation,
2981 upsert,
2982 end,
2983 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
2984 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2985 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2986 }
2987 }
2988 }
2989
2990 fn handle_operator_confirm(
2991 &self,
2992 ctx: &RouteCtx,
2993 frame: Frame,
2994 ) -> Result<Vec<Frame>, RouterError> {
2995 use crate::operator_confirm::{audit, Outcome};
2996 let registration = self
2997 .registry
2998 .get_module_by_connection(ctx.connection_id)
2999 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3000 let Some(registration) = registration else {
3001 let outcome = Outcome::refusal("not_registered");
3002 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
3003 return Ok(vec![outcome.frame(&frame)]);
3004 };
3005 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3006 Ok(request) => request,
3007 Err(_) => {
3008 let outcome = Outcome::refusal("invalid_control_body");
3009 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
3010 return Ok(vec![outcome.frame(&frame)]);
3011 }
3012 };
3013 let module_id = registration.manifest.module_id;
3014 let nonce = self
3018 .hello_launch_nonces
3019 .lock()
3020 .unwrap_or_else(|p| p.into_inner())
3021 .nonce(ctx.connection_id)
3022 .map(str::to_owned);
3023 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3024 self.supervisor
3025 .spawned_consumer_authorized(&module_id, nonce)
3026 });
3027 let confirms = self.forwarding.operator_confirms();
3028 self.forwarding
3029 .with_operator_route(
3030 ctx.connection_id,
3031 request.route_channel,
3032 request.route_epoch,
3033 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3034 )
3035 .map_err(RouterError::Forwarding)
3036 }
3037
3038 fn handle_scope_sync(
3042 &self,
3043 connection_id: ConnectionId,
3044 frame: Frame,
3045 generation: u64,
3046 scopes: Vec<ScopeRecord>,
3047 ) -> Result<Vec<Frame>, RouterError> {
3048 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3049 }
3050
3051 fn handle_scope_change(
3052 &self,
3053 connection_id: ConnectionId,
3054 frame: Frame,
3055 generation: u64,
3056 scopes: Vec<ScopeRecord>,
3057 end: Option<Vec<ScopeEnd>>,
3058 ) -> Result<Vec<Frame>, RouterError> {
3059 let is_apply = end.is_some();
3060 let Some(registration) = self
3061 .registry
3062 .get_module_by_connection(connection_id)
3063 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3064 else {
3065 return Ok(vec![control_error_frame(
3066 &frame,
3067 "not_registered",
3068 "scope.sync and scope.apply require an active module registration owned by this connection",
3069 )?]);
3070 };
3071 let owner = registration.manifest.module_id;
3072 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3073 let is_current_launch = |connection: ConnectionId| {
3074 self.hello_launch_nonces
3075 .lock()
3076 .unwrap_or_else(|poisoned| poisoned.into_inner())
3077 .presented(connection, current_nonce.as_deref())
3078 };
3079 let mut table = self
3084 .scopes
3085 .write()
3086 .unwrap_or_else(|poisoned| poisoned.into_inner());
3087 let outcome = match end {
3088 Some(end) => table.apply(
3089 &owner,
3090 connection_id,
3091 is_current_launch,
3092 generation,
3093 scopes,
3094 end,
3095 ),
3096 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3097 };
3098 let drained = match &outcome {
3099 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3100 Err(_) => Vec::new(),
3101 };
3102 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3103 drop(table);
3104 if has_deadlines {
3105 self.scope_deadline_added.notify_one();
3106 }
3107 match outcome {
3108 Ok(applied) => {
3109 let counts = ScopeOutcomeCounts::of(&applied.results);
3110 info!(
3111 owner = %owner,
3112 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3113 generation,
3114 records = applied.results.len(),
3115 created = counts.created,
3116 replaced = counts.replaced,
3117 updated = counts.updated,
3118 unchanged = counts.unchanged,
3119 refused = counts.refused,
3120 ended = applied.ended.len(),
3121 tag_changes = applied.tag_changes.len(),
3122 routes_closed = drained.len(),
3123 "scope change accepted"
3124 );
3125 for refused in applied
3131 .results
3132 .iter()
3133 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3134 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3135 {
3136 warn!(
3137 owner = %owner,
3138 generation,
3139 scope_ref = %refused.scope_ref,
3140 scope_epoch = refused.scope_epoch,
3141 code = refused.code.as_deref().unwrap_or(""),
3142 "scope record refused"
3143 );
3144 }
3145 self.close_scope_drained_routes(drained);
3146 let response = if is_apply {
3147 ModuleControlResponseToModule::ScopeApply {
3148 generation,
3149 results: applied.results,
3150 end_results: applied.end_results,
3151 ended: applied.ended,
3152 }
3153 } else {
3154 ModuleControlResponseToModule::ScopeSync {
3155 generation,
3156 results: applied.results,
3157 ended: applied.ended,
3158 }
3159 };
3160 Ok(vec![control_response_body_frame(
3161 &frame,
3162 &response,
3163 "scope change response",
3164 )?])
3165 }
3166 Err(refusal) => {
3167 info!(
3168 owner = %owner,
3169 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3170 generation,
3171 code = refusal.code,
3172 "scope change refused"
3173 );
3174 Ok(vec![control_error_frame(
3175 &frame,
3176 refusal.code,
3177 refusal.message,
3178 )?])
3179 }
3180 }
3181 }
3182
3183 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3189 if drained.is_empty() {
3190 return;
3191 }
3192 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3193 BTreeMap::new();
3194 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3195 for route in drained {
3196 warn!(
3197 module_id = %route.module_id,
3198 reason = ?route.reason,
3199 client_connection_id = route.client.connection_id.get(),
3200 route_channel = route.client.channel,
3201 "closing route because its scope changed"
3202 );
3203 pushes
3204 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3205 .or_insert_with(|| (route.reason, Vec::new()))
3206 .1
3207 .push(EndpointRoute {
3208 goodbye_target: route.client.clone(),
3209 principal: Principal::Unverified,
3210 bound_at: Instant::now(),
3211 draining: false,
3212 drain_reason: None,
3213 });
3214 goodbyes.push(route.module);
3215 goodbyes.push(route.client);
3216 }
3217 for ((module_id, _), (reason, routes)) in pushes {
3218 send_route_control_pushes(
3219 &self.forwarding,
3220 routes,
3221 ClientControlPush::RouteClosed {
3222 module_id,
3223 channels: Vec::new(),
3224 reason,
3225 drained: false,
3226 abandoned: 0,
3227 excluded_subscriptions: 0,
3228 terminal: Some(false),
3229 },
3230 );
3231 }
3232 self.emit_route_goodbyes(goodbyes);
3233 }
3234
3235 fn handle_scope_describe(
3238 &self,
3239 connection_id: ConnectionId,
3240 frame: Frame,
3241 owner: Principal,
3242 scope_ref: String,
3243 ) -> Result<Vec<Frame>, RouterError> {
3244 let registered = self
3245 .registry
3246 .get_module_by_connection(connection_id)
3247 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3248 let Some(registered) = registered else {
3249 return Ok(vec![control_error_frame(
3250 &frame,
3251 "not_registered",
3252 "scope.describe requires an active module registration owned by this connection",
3253 )?]);
3254 };
3255 let mut description = self
3256 .scopes
3257 .read()
3258 .unwrap_or_else(|poisoned| poisoned.into_inner())
3259 .describe(&owner, &scope_ref);
3260 if !registered
3263 .manifest
3264 .capabilities
3265 .as_ref()
3266 .is_some_and(|caps| {
3267 caps.provides
3268 .iter()
3269 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3270 })
3271 {
3272 if let Some(stamp) = description.stamp.as_mut() {
3273 stamp.attributes.on_behalf_of_agent = None;
3274 }
3275 }
3276 let owner_configured = match &owner {
3277 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3282 _ => false,
3283 };
3284 let response = ModuleControlResponseToModule::ScopeDescribe {
3285 status: description.status,
3286 scope_epoch: description.scope_epoch,
3287 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3288 owner_synced: description.owner_synced,
3289 owner_configured,
3290 scope: description.stamp,
3291 };
3292 Ok(vec![control_response_body_frame(
3293 &frame,
3294 &response,
3295 "ModuleControlResponseToModule::ScopeDescribe",
3296 )?])
3297 }
3298
3299 fn handle_catalog_update(
3300 &self,
3301 connection_id: ConnectionId,
3302 frame: Frame,
3303 provides: Vec<ProviderRole>,
3304 capabilities: Option<CapabilityDeclarations>,
3305 ready: Option<bool>,
3306 ) -> Result<Vec<Frame>, RouterError> {
3307 self.refresh_capability_requirements();
3308 let Some(registration) = self
3309 .registry
3310 .get_module_by_connection(connection_id)
3311 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3312 else {
3313 return Ok(vec![control_error_frame(
3314 &frame,
3315 "not_registered",
3316 "catalog.update requires an active module registration owned by this connection",
3317 )?]);
3318 };
3319
3320 if let Some(message) =
3321 catalog_update_frozen_field_message(®istration.manifest, &provides)
3322 {
3323 return Ok(vec![control_error_frame(
3324 &frame,
3325 "catalog_update_frozen_field",
3326 message,
3327 )?]);
3328 }
3329
3330 let mut candidate = registration.manifest.clone();
3331 candidate.provides = provides.clone();
3332 candidate.capabilities = capabilities
3333 .clone()
3334 .or_else(|| registration.manifest.capabilities.clone());
3335 if let Err(err) = candidate.validate_capability_grammar() {
3336 return Ok(vec![control_error_frame(
3337 &frame,
3338 "invalid_capability_grammar",
3339 err.to_string(),
3340 )?]);
3341 }
3342
3343 let mut conflicts = self
3346 .capability_evaluator
3347 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3348 if let Some(conflict) = conflicts.first() {
3349 let message = format!(
3350 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3351 conflict.capability, conflict.claimants[0], candidate.module_id
3352 );
3353 for conflict in &mut conflicts {
3354 conflict.source = DuplicateClaimSource::CatalogUpdate;
3355 }
3356 log_duplicate_claim_events(conflicts);
3357 return Ok(vec![control_error_frame(
3358 &frame,
3359 "reserved_capability",
3360 message,
3361 )?]);
3362 }
3363
3364 let updated = self
3365 .registry
3366 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3367 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3368 if updated.is_none() {
3369 return Ok(vec![control_error_frame(
3370 &frame,
3371 "not_registered",
3372 "catalog.update requires an active module registration owned by this connection",
3373 )?]);
3374 }
3375 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3376 log_duplicate_claim_events(
3377 self.capability_evaluator
3378 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3379 );
3380 }
3381 if capability_census_trigger(
3382 registration.manifest.capabilities.as_ref(),
3383 updated
3384 .as_ref()
3385 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3386 ) {
3387 self.enforce_capability_denies();
3388 }
3389 self.refresh_capability_requirements();
3390
3391 let response = ModuleControlResponseToModule::CatalogUpdate {};
3392 control_response_body_frame(
3393 &frame,
3394 &response,
3395 "ModuleControlResponseToModule::CatalogUpdate",
3396 )
3397 .map(|frame| vec![frame])
3398 }
3399
3400 pub(crate) fn with_activity_ledger_config(
3401 mut self,
3402 run_dir: &Path,
3403 incarnation: String,
3404 config: crate::activity::ActivityConfig,
3405 ) -> Self {
3406 self.activity = Some(crate::activity::ActivityLedger::open_config(
3407 run_dir.join("activity.jsonl"),
3408 incarnation,
3409 config,
3410 ));
3411 self.supervisor.set_activity(self.activity.clone());
3412 self
3413 }
3414
3415 pub(crate) async fn configure_activity(
3416 &self,
3417 config: crate::activity::ActivityConfig,
3418 ) -> Result<(), &'static str> {
3419 match &self.activity {
3420 Some(ledger) => ledger.configure(config).await,
3421 None => Ok(()),
3422 }
3423 }
3424
3425 fn cancel_activity_subscription(
3426 &self,
3427 ctx: &RouteCtx,
3428 frame: &Frame,
3429 ) -> Result<bool, RouterError> {
3430 let task = self
3431 .activity_streams
3432 .lock()
3433 .unwrap_or_else(|p| p.into_inner())
3434 .remove(&(ctx.connection_id, frame.header.corr));
3435 let Some(task) = task else {
3436 return Ok(false);
3437 };
3438 task.abort();
3439 let end = Frame::build_with_version(
3440 response_version(frame),
3441 FrameType::StreamEnd,
3442 control_flags(),
3443 0,
3444 0,
3445 frame.header.corr,
3446 Vec::new(),
3447 )
3448 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3449 let sink = ctx.egress.clone();
3450 tokio::spawn(async move {
3451 let _ = sink.send(end).await;
3452 });
3453 Ok(true)
3454 }
3455
3456 async fn handle_activity_read(
3457 &self,
3458 ctx: &RouteCtx,
3459 frame: Frame,
3460 ) -> Result<Vec<Frame>, RouterError> {
3461 use subc_control::{
3462 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3463 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3464 };
3465 #[derive(Deserialize)]
3466 struct Identity {
3467 #[serde(default)]
3468 consumer_identity: Option<ConsumerIdentity>,
3469 }
3470 let principal = match self
3474 .forwarding
3475 .module_id_for_connection(ctx.connection_id)
3476 .map_err(RouterError::Forwarding)?
3477 {
3478 Some(module_id) => Ok(Principal::Reserved { module_id }),
3479 None => {
3480 let identity: Identity = match serde_json::from_slice(&frame.body) {
3481 Ok(identity) => identity,
3482 Err(_) => {
3483 return Ok(vec![control_error_frame(
3484 &frame,
3485 ACTIVITY_MALFORMED,
3486 "malformed activity read",
3487 )?])
3488 }
3489 };
3490 self.route_open_principal(&frame, identity.consumer_identity)?
3491 }
3492 };
3493 let admitted = matches!(principal, Ok(Principal::Direct))
3494 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3495 if !admitted {
3496 return Ok(vec![control_error_frame(
3497 &frame,
3498 ACTIVITY_READ_FORBIDDEN,
3499 "activity reads require an operator or trusted mirror principal",
3500 )?]);
3501 }
3502 let Some(ledger) = &self.activity else {
3503 return Ok(vec![control_error_frame(
3504 &frame,
3505 ACTIVITY_UNAVAILABLE,
3506 "activity ledger unavailable",
3507 )?]);
3508 };
3509 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3510 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3511 {
3512 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3513 Ok(request) => ledger.query(request).await,
3514 Err(_) => Err(ACTIVITY_MALFORMED),
3515 };
3516 return match result {
3517 Ok(reply) => Ok(vec![control_response_body_frame(
3518 &frame,
3519 &reply,
3520 ops::ACTIVITY_QUERY,
3521 )?]),
3522 Err(code) => Ok(vec![control_error_frame(
3523 &frame,
3524 code,
3525 "activity query refused",
3526 )?]),
3527 };
3528 }
3529 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3530 Ok(request) => request,
3531 Err(_) => {
3532 return Ok(vec![control_error_frame(
3533 &frame,
3534 ACTIVITY_MALFORMED,
3535 "malformed activity subscription",
3536 )?])
3537 }
3538 };
3539 let mut subscription = match ledger.subscribe(request.since).await {
3540 Ok(subscription) => subscription,
3541 Err(code) => {
3542 return Ok(vec![control_error_frame(
3543 &frame,
3544 code,
3545 "activity subscription refused",
3546 )?])
3547 }
3548 };
3549 let key = (ctx.connection_id, frame.header.corr);
3550 let sink = ctx.egress.clone();
3551 let mut streams = self
3552 .activity_streams
3553 .lock()
3554 .unwrap_or_else(|p| p.into_inner());
3555 streams.retain(|_, task| !task.is_finished());
3556 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3557 return Ok(vec![control_error_frame(
3558 &frame,
3559 "duplicate_subscription",
3560 "activity subscription correlation id already in use",
3561 )?]);
3562 }
3563 let task = tokio::spawn(async move {
3564 loop {
3565 let entry = if let Some(entry) = subscription.replay.pop_front() {
3566 Some(entry)
3567 } else {
3568 subscription.receiver.recv().await
3569 };
3570 let Some(entry) = entry else {
3571 break;
3572 };
3573 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3574 return;
3575 };
3576 let Ok(next) = Frame::build_with_version(
3577 response_version(&frame),
3578 FrameType::StreamData,
3579 control_flags(),
3580 0,
3581 0,
3582 frame.header.corr,
3583 body,
3584 ) else {
3585 return;
3586 };
3587 if sink.send(next).await.is_err() {
3588 return;
3589 }
3590 }
3591 if let Some(code) = subscription.terminal_code() {
3594 if let Ok(end) =
3595 control_error_frame(&frame, code, "activity subscriber fell behind")
3596 {
3597 let _ = sink.send(end).await;
3598 }
3599 }
3600 });
3601 streams.insert(key, task.abort_handle());
3602 Ok(Vec::new())
3603 }
3604
3605 async fn handle_activity_record(
3606 &self,
3607 ctx: &RouteCtx,
3608 frame: Frame,
3609 ) -> Result<Vec<Frame>, RouterError> {
3610 use subc_control::{
3611 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3612 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3613 };
3614 let result = async {
3615 let module = self
3616 .forwarding
3617 .module_id_for_connection(ctx.connection_id)
3618 .ok()
3619 .flatten()
3620 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3621 let request: ActivityRecordRequest =
3622 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3623 crate::activity::validate(&module, &request)?;
3624 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3625 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3626 let at_ms = std::time::SystemTime::now()
3627 .duration_since(std::time::UNIX_EPOCH)
3628 .unwrap_or_default()
3629 .as_millis() as u64;
3630 ledger.record(module, request, actor, at_ms).await
3631 }
3632 .await;
3633 match result {
3634 Ok(entry) => Ok(vec![control_response_body_frame(
3635 &frame,
3636 &ActivityRecordResponse { seq: entry.seq },
3637 "activity.record",
3638 )?]),
3639 Err(code) => Ok(vec![control_error_frame(
3640 &frame,
3641 code,
3642 "activity recording refused",
3643 )?]),
3644 }
3645 }
3646
3647 fn activity_actor(
3648 &self,
3649 connection: ConnectionId,
3650 module: &str,
3651 on: Option<&subc_control::ActivityOn>,
3652 ) -> Result<subc_control::ActivityActor, &'static str> {
3653 use crate::forwarding::{DataRoute, DataRouteState};
3654 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3655 let principal = |id: &str| Principal::Reserved {
3656 module_id: id.to_owned(),
3657 };
3658 let mut actor = ActivityActor {
3659 principal: principal(module),
3660 scope_owner: None,
3661 agent_id: None,
3662 run_id: None,
3663 flow_id: None,
3664 project_root: None,
3665 };
3666 let scope = match on {
3667 None => return Ok(actor),
3668 Some(ActivityOn::Scope(scope)) => {
3669 let selector = subc_protocol::scope::ScopeSelector::new(
3670 principal(module),
3671 scope.scope_ref.clone(),
3672 Some(scope.epoch),
3673 );
3674 self.scopes
3675 .read()
3676 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3677 .attribute(&principal(module), module, &selector, true)
3678 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3679 }
3680 Some(ActivityOn::Route(route)) => {
3681 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3682 .forwarding
3683 .lookup_data_route(connection, route.channel, route.epoch)
3684 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3685 else {
3686 return Err(ACTIVITY_ACTOR_UNKNOWN);
3687 };
3688 if binding.module_id != module {
3689 return Err(ACTIVITY_ACTOR_UNKNOWN);
3690 }
3691 actor.principal = binding.principal.clone();
3692 actor.project_root = binding
3693 .project_root
3694 .as_ref()
3695 .map(|root| root.as_path().to_path_buf());
3696 let Some(bound) = &binding.scope else {
3697 return Ok(actor);
3698 };
3699 let owner = principal(&bound.owner);
3700 let selector = subc_protocol::scope::ScopeSelector::new(
3701 owner.clone(),
3702 bound.scope_ref.clone(),
3703 Some(bound.tag.scope_epoch),
3704 );
3705 let admitted = self
3706 .scopes
3707 .read()
3708 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3709 .attribute(&owner, module, &selector, true)
3710 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3711 if admitted.tag != bound.tag {
3712 return Err(ACTIVITY_ACTOR_UNKNOWN);
3713 }
3714 admitted
3715 }
3716 };
3717 actor.scope_owner = Some(scope.stamp.owner);
3718 actor.agent_id = scope.stamp.attributes.agent_id;
3719 actor.run_id = scope.stamp.attributes.run_id;
3720 actor.flow_id = scope.stamp.attributes.flow_id;
3721 Ok(actor)
3722 }
3723
3724 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3725 self.refresh_capability_requirements();
3726 let mut counters = self.counters.snapshot();
3732 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3733 obj.insert(
3734 "activity_unparseable_lines".into(),
3735 ledger.read_errors().into(),
3736 );
3737 obj.insert(
3738 "activity_lifecycle_append_failures".into(),
3739 ledger.lifecycle_failures().into(),
3740 );
3741 obj.insert(
3742 "activity_started_digest_skips".into(),
3743 ledger.lifecycle_digest_skips().into(),
3744 );
3745 }
3746 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3747 self.forwarding.client_route_concentration(),
3748 counters.as_object_mut(),
3749 ) {
3750 obj.insert(
3751 "client_connections_with_routes".into(),
3752 connections_with_routes.into(),
3753 );
3754 obj.insert("max_routes_on_one_connection".into(), max.into());
3755 }
3756 if let (Some(open_breakers), Some(obj)) = (
3762 self.route_bind_breakers.open_snapshot(),
3763 counters.as_object_mut(),
3764 ) {
3765 obj.insert("route_bind_breakers_open".into(), open_breakers);
3766 }
3767 let response = ClientControlResponse::ServerDescribe {
3768 protocol_ver: PROTOCOL_VERSION,
3769 subc_ops: subc_ops(),
3770 capabilities: self.subc_capabilities.as_ref().to_vec(),
3771 connected_clients: self.connected_clients.count(),
3772 counters: Some(counters),
3773 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3774 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3775 capability_requirements: self.capability_requirement_statuses(),
3776 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3777 };
3778 Ok(vec![control_response_body_frame(
3779 &frame,
3780 &response,
3781 "ClientControlResponse::ServerDescribe",
3782 )?])
3783 }
3784
3785 fn handle_catalog_list(
3786 &self,
3787 frame: Frame,
3788 module_id: Option<String>,
3789 ) -> Result<Vec<Frame>, RouterError> {
3790 let (generation, modules) = self.registry.list_modules().map_err(|err| {
3791 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3792 })?;
3793 let entries = modules
3794 .into_iter()
3795 .filter(|registration| {
3796 module_id
3797 .as_deref()
3798 .map(|wanted| registration.manifest.module_id == wanted)
3799 .unwrap_or(true)
3800 })
3801 .map(|registration| {
3802 let not_ready = self.not_ready_reason(®istration);
3803 let roles = registration.manifest.provides;
3804 CatalogEntry::new(
3805 registration.manifest.module_id,
3806 roles,
3807 registration.control_ops,
3808 )
3809 .with_ready(not_ready.is_none())
3810 .with_not_ready(not_ready)
3811 .with_module_version(Some(registration.manifest.module_version))
3812 .with_capabilities(registration.manifest.capabilities)
3813 .with_self_signals(registration.manifest.self_signals)
3814 .with_events(registration.manifest.events)
3815 .with_operation_delivery(registration.manifest.operation_delivery)
3816 })
3817 .collect();
3818 let response = ClientControlResponse::CatalogList {
3819 generation,
3820 modules: entries,
3821 subc_ops: subc_ops(),
3822 };
3823 Ok(vec![control_response_body_frame(
3824 &frame,
3825 &response,
3826 "ClientControlResponse::CatalogList",
3827 )?])
3828 }
3829
3830 fn route_open_principal(
3831 &self,
3832 frame: &Frame,
3833 consumer_identity: Option<ConsumerIdentity>,
3834 ) -> Result<Result<Principal, Frame>, RouterError> {
3835 let Some(consumer_identity) = consumer_identity else {
3836 return Ok(Ok(Principal::Direct));
3837 };
3838
3839 if self.supervisor.spawned_consumer_authorized(
3840 &consumer_identity.module_id,
3841 &consumer_identity.launch_nonce,
3842 ) {
3843 return Ok(Ok(Principal::Reserved {
3844 module_id: consumer_identity.module_id,
3845 }));
3846 }
3847
3848 Ok(Err(control_error_frame(
3849 frame,
3850 "bad_consumer_identity",
3851 format!(
3852 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3853 consumer_identity.module_id
3854 ),
3855 )?))
3856 }
3857
3858 fn route_open_refusal_frame(
3865 &self,
3866 ctx: &RouteCtx,
3867 frame: &Frame,
3868 module_id: &str,
3869 reason: &'static str,
3870 code: &'static str,
3871 message: impl Into<String>,
3872 ) -> Result<Frame, RouterError> {
3873 self.observe_route_open_refusal(ctx, module_id, reason, code);
3874 control_error_frame(frame, code, message.into())
3875 }
3876
3877 fn route_open_breaker_refusal_frame(
3895 &self,
3896 ctx: &RouteCtx,
3897 frame: &Frame,
3898 module_id: &str,
3899 consecutive_timeouts: u32,
3900 retry_in: Duration,
3901 probe_in_flight: bool,
3902 ) -> Result<Frame, RouterError> {
3903 self.counters
3904 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
3905 debug!(
3906 target: "control",
3907 code = "module_timeout",
3908 module_id = ?module_id,
3909 connection_id = ctx.connection_id.get(),
3910 consecutive_timeouts,
3911 retry_in_ms = retry_in.as_millis() as u64,
3912 probe_in_flight,
3913 "route.open refused by open bind-relay breaker"
3914 );
3915 let detail = if probe_in_flight {
3922 "one test route is already being tried; retry once it settles".to_string()
3923 } else {
3924 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
3925 };
3926 control_error_frame(
3927 frame,
3928 "module_timeout",
3929 format!(
3930 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
3931 timed out in a row); {detail}; its established routes are unaffected"
3932 ),
3933 )
3934 }
3935
3936 fn observe_route_open_refusal(
3949 &self,
3950 ctx: &RouteCtx,
3951 module_id: &str,
3952 reason: &'static str,
3953 code: &'static str,
3954 ) {
3955 self.counters.increment_route_open_refused(code);
3956 info!(
3957 target: "control",
3958 code,
3959 reason,
3960 module_id = ?module_id,
3961 connection_id = ctx.connection_id.get(),
3962 "route.open refused"
3963 );
3964 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
3965 self.route_outages.record_not_serving(module_id, reason);
3966 }
3967 }
3968
3969 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
4014 self.route_outages.record_accepted(module_id);
4015 self.counters.increment_route_open_accepted(principal);
4016 info!(
4017 target: "control",
4018 principal,
4019 module_id,
4020 connection_id = ctx.connection_id.get(),
4021 "route.open accepted"
4022 );
4023 }
4024
4025 fn supervised_absent_route_open_refusal_frame(
4026 &self,
4027 ctx: &RouteCtx,
4028 frame: &Frame,
4029 module_id: &str,
4030 code: &'static str,
4031 status: &crate::supervise::ModuleStatus,
4032 ) -> Result<Frame, RouterError> {
4033 self.counters.increment_route_open_refused(code);
4034 info!(
4035 target: "control",
4036 code,
4037 reason = "supervised_not_registered",
4038 module_id = ?module_id,
4039 connection_id = ctx.connection_id.get(),
4040 state = %status.state,
4041 enabled = status.enabled,
4042 live = status.live,
4043 "route.open refused"
4044 );
4045 self.route_outages
4049 .record_not_serving(module_id, "supervised_not_registered");
4050 let message = if let Some(pid) = status.waiting_for_previous_pid {
4051 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4052 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4053 })
4054 } else {
4055 format!(
4056 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4057 status.state, status.enabled, status.live
4058 )
4059 };
4060 control_error_frame(frame, code, message)
4061 }
4062
4063 async fn handle_route_open(
4064 &self,
4065 ctx: &RouteCtx,
4066 frame: Frame,
4067 request: RouteOpenRequest,
4068 timings: &mut RouteOpenTimings,
4069 ) -> Result<Vec<Frame>, RouterError> {
4070 let RouteOpenRequest {
4071 target,
4072 mut identity,
4073 consumer_identity,
4074 consumer_capabilities,
4075 role_versions,
4076 admission_facts,
4077 scope,
4078 } = request;
4079 let target_module_id = target_module_id(&target).to_string();
4080 timings.module_id.clone_from(&target_module_id);
4081 if self
4082 .registry
4083 .get_module_by_connection(ctx.connection_id)
4084 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4085 .is_some()
4086 {
4087 return Ok(vec![control_error_frame(
4088 &frame,
4089 "invalid_request",
4090 "module connections cannot open client routes",
4091 )?]);
4092 }
4093 debug!(
4094 connection_id = ctx.connection_id.get(),
4095 corr = frame.header.corr,
4096 module_id = %target_module_id,
4097 "handling route.open"
4098 );
4099
4100 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4106 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4107 self.observe_route_open_refusal(
4108 ctx,
4109 &target_module_id,
4110 "invalid_role_versions",
4111 error_codes::INVALID_REQUEST,
4112 );
4113 return Ok(vec![control_error_body_frame(
4114 &frame,
4115 ErrorBody {
4116 code: error_codes::INVALID_REQUEST.to_string(),
4117 message: error.to_string(),
4118 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4119 },
4120 )?]);
4121 }
4122
4123 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4124 if status.waiting_for_previous_pid.is_some() {
4125 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4126 ctx,
4127 &frame,
4128 &target_module_id,
4129 "module_warming",
4130 &status,
4131 )?]);
4132 }
4133 }
4134
4135 let Some(registration) = self
4154 .registry
4155 .get_module(&target_module_id)
4156 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4157 else {
4158 if let Some((status, warming)) =
4159 self.supervisor_status(&target_module_id, frame.header.corr)?
4160 {
4161 if status.protocol == ModuleProtocol::None {
4168 return Ok(vec![self.route_open_refusal_frame(
4169 ctx,
4170 &frame,
4171 &target_module_id,
4172 "protocol_none",
4173 error_codes::MODULE_NO_PROTOCOL,
4174 format!(
4175 "module_id '{target_module_id}' is declared protocol: none; \
4176 it speaks no subc wire and serves no routes"
4177 ),
4178 )?]);
4179 }
4180 let code = if warming {
4181 "module_warming"
4182 } else {
4183 "target_unavailable"
4184 };
4185 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4186 ctx,
4187 &frame,
4188 &target_module_id,
4189 code,
4190 &status,
4191 )?]);
4192 }
4193 if let Some(removed_ago_ms) =
4194 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4195 {
4196 return Ok(vec![self.route_open_refusal_frame(
4197 ctx,
4198 &frame,
4199 &target_module_id,
4200 "removed",
4201 error_codes::MODULE_REMOVED,
4202 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4203 )?]);
4204 }
4205 return Ok(vec![self.route_open_refusal_frame(
4206 ctx,
4207 &frame,
4208 &target_module_id,
4209 "not_registered",
4210 error_codes::UNKNOWN_MODULE,
4211 format!("module_id '{target_module_id}' is not registered"),
4212 )?]);
4213 };
4214
4215 if !registration.ready {
4219 self.counters
4220 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4221 info!(
4222 target: "control",
4223 code = error_codes::MODULE_WARMING,
4224 module_id = ?target_module_id,
4225 connection_id = ctx.connection_id.get(),
4226 reason = "declared_not_ready",
4227 "route.open refused"
4228 );
4229 self.route_outages
4232 .record_not_serving(&target_module_id, "declared_not_ready");
4233 return Ok(vec![control_error_body_frame(
4234 &frame,
4235 ErrorBody {
4236 code: error_codes::MODULE_WARMING.to_string(),
4237 message: format!(
4238 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4239 ),
4240 detail: Some(serde_json::json!({
4241 "reason": "declared_not_ready"
4242 })),
4243 },
4244 )?]);
4245 }
4246
4247 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4266 self.counters
4267 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4268 info!(
4269 target: "control",
4270 code = error_codes::MODULE_WARMING,
4271 module_id = ?target_module_id,
4272 connection_id = ctx.connection_id.get(),
4273 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4274 capability = %capability,
4275 "route.open refused"
4276 );
4277 return Ok(vec![control_error_body_frame(
4278 &frame,
4279 ErrorBody {
4280 code: error_codes::MODULE_WARMING.to_string(),
4281 message: format!(
4282 "module_id '{target_module_id}' requires capability '{capability}', \
4283 which no registered module provides; retry"
4284 ),
4285 detail: Some(serde_json::json!({
4286 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4287 "capability": capability,
4288 })),
4289 },
4290 )?]);
4291 }
4292
4293 if !target_has_required_role(&target, ®istration.manifest.provides) {
4294 return Ok(vec![self.route_open_refusal_frame(
4295 ctx,
4296 &frame,
4297 &target_module_id,
4298 "role_not_provided",
4299 "target_unavailable",
4300 format!("module_id '{target_module_id}' does not provide the requested target"),
4301 )?]);
4302 }
4303
4304 if registration.state != ChannelState::Active {
4305 return Ok(vec![self.route_open_refusal_frame(
4306 ctx,
4307 &frame,
4308 &target_module_id,
4309 "registration_not_active",
4310 "target_unavailable",
4311 format!("module_id '{target_module_id}' is not active"),
4312 )?]);
4313 }
4314
4315 if self
4316 .forwarding
4317 .module_is_draining(&target_module_id)
4318 .map_err(RouterError::Forwarding)?
4319 {
4320 return Ok(vec![self.route_open_refusal_frame(
4321 ctx,
4322 &frame,
4323 &target_module_id,
4324 "reloading",
4325 "module_reloading",
4326 format!("module_id '{target_module_id}' is reloading"),
4327 )?]);
4328 }
4329
4330 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4331 process_liveness.process_live(&target_module_id) == Some(false)
4332 }) {
4333 if process_liveness.process_replacing(&target_module_id) {
4340 return Ok(vec![self.route_open_refusal_frame(
4341 ctx,
4342 &frame,
4343 &target_module_id,
4344 "reloading",
4345 "module_reloading",
4346 format!("module_id '{target_module_id}' is reloading"),
4347 )?]);
4348 }
4349 return Ok(vec![self.route_open_refusal_frame(
4350 ctx,
4351 &frame,
4352 &target_module_id,
4353 "supervisor_not_live",
4354 "target_unavailable",
4355 format!("module_id '{target_module_id}' is not live"),
4356 )?]);
4357 }
4358
4359 if !self
4360 .forwarding
4361 .has_live_module_connection(&target_module_id)
4362 .map_err(RouterError::Forwarding)?
4363 {
4364 return Ok(vec![self.route_open_refusal_frame(
4365 ctx,
4366 &frame,
4367 &target_module_id,
4368 "no_forwarding_connection",
4369 "target_unavailable",
4370 format!("module_id '{target_module_id}' has no live forwarding connection"),
4371 )?]);
4372 }
4373
4374 if let Some(error) =
4375 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4376 {
4377 self.observe_route_open_refusal(
4378 ctx,
4379 &target_module_id,
4380 "op_not_allowed",
4381 "op_not_allowed",
4382 );
4383 return Ok(vec![error]);
4384 }
4385
4386 let principal = match self.route_open_principal(&frame, consumer_identity)? {
4387 Ok(principal) => principal,
4388 Err(error) => {
4389 self.observe_route_open_refusal(
4390 ctx,
4391 &target_module_id,
4392 "bad_consumer_identity",
4393 "bad_consumer_identity",
4394 );
4395 return Ok(vec![error]);
4396 }
4397 };
4398 timings.principal = principal_label(&principal);
4399
4400 if let Principal::Reserved {
4404 module_id: opening_module_id,
4405 } = &principal
4406 {
4407 if let Some(opening_registration) = self
4408 .registry
4409 .get_module(opening_module_id)
4410 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4411 {
4412 if let Some(capability) =
4413 denied_capability(&opening_registration.manifest, ®istration.manifest)
4414 {
4415 warn!(
4416 opening_module_id,
4417 target_module_id,
4418 capability,
4419 "refusing route.open because an attested capability deny edge matches"
4420 );
4421 return Ok(vec![self.route_open_refusal_frame(
4422 ctx,
4423 &frame,
4424 &target_module_id,
4425 "capability_deny_edge",
4426 "capability_forbidden",
4427 format!(
4428 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4429 ),
4430 )?]);
4431 }
4432 }
4433 }
4434
4435 if admission_facts.is_some() {
4436 let carrier_matches = matches!(
4437 &principal,
4438 Principal::Reserved { module_id }
4439 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4440 );
4441 if !carrier_matches {
4442 return Ok(vec![self.route_open_refusal_frame(
4443 ctx,
4444 &frame,
4445 &target_module_id,
4446 "admission_facts_carrier_not_permitted",
4447 "admission_facts_not_permitted",
4448 "admission facts may only be carried by the configured reserved module",
4449 )?]);
4450 }
4451
4452 let target_allowed = self
4453 .admission_facts_targets
4454 .as_ref()
4455 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4456 if !target_allowed {
4457 return Ok(vec![self.route_open_refusal_frame(
4458 ctx,
4459 &frame,
4460 &target_module_id,
4461 "admission_facts_target_not_listed",
4462 "admission_facts_target_not_allowed",
4463 format!(
4464 "admission facts are not permitted for target module_id '{target_module_id}'"
4465 ),
4466 )?]);
4467 }
4468
4469 }
4473
4474 let (bound_scope, mut scope_stamp) = match &scope {
4480 None => (None, None),
4481 Some(selector) => {
4482 let owner_configured = match &selector.owner {
4483 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4489 _ => false,
4490 };
4491 let mut scopes = self
4492 .scopes
4493 .write()
4494 .unwrap_or_else(|poisoned| poisoned.into_inner());
4495 let admitted = scopes
4496 .admit(&principal, &target_module_id, selector, owner_configured)
4497 .and_then(|admission| {
4498 crate::scopes::check_target_flow_support(
4499 &admission.stamp,
4500 &target_module_id,
4501 registration.manifest.capabilities.as_ref(),
4502 )?;
4503 crate::scopes::check_target_agent_run_support(
4504 &admission.stamp,
4505 &target_module_id,
4506 registration.manifest.capabilities.as_ref(),
4507 )?;
4508 let bound = scopes.reserve_bind(
4509 &admission,
4510 &principal,
4511 ctx.connection_id,
4512 selector.hold_presence,
4513 );
4514 Ok((admission, bound))
4515 });
4516 match admitted {
4517 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4518 Err(refusal) => {
4519 return Ok(vec![self.route_open_refusal_frame(
4520 ctx,
4521 &frame,
4522 &target_module_id,
4523 refusal.code,
4524 refusal.code,
4525 refusal.message,
4526 )?]);
4527 }
4528 }
4529 }
4530 };
4531
4532 timings.enter(RouteOpenPhase::ProjectRoot);
4554 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4555 Ok(project_root) => project_root,
4556 Err(err) => {
4557 return Ok(vec![control_error_frame(
4558 &frame,
4559 "invalid_project_root",
4560 err.to_string(),
4561 )?])
4562 }
4563 };
4564 timings.enter(RouteOpenPhase::Admission);
4565 identity.project_root = project_root.as_path().to_path_buf();
4566
4567 let _concurrency_guard = match self
4578 .route_bind_concurrency
4579 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4580 {
4581 Ok(guard) => guard,
4582 Err(in_flight) => {
4583 return Ok(vec![self.route_open_target_capacity_refusal(
4584 ctx,
4585 &frame,
4586 &target_module_id,
4587 in_flight,
4588 )?]);
4589 }
4590 };
4591
4592 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4595 RouteBindAdmission::Admitted { guard, probe } => {
4596 if probe {
4597 info!(
4598 module_id = %target_module_id,
4599 connection_id = ctx.connection_id.get(),
4600 "route.bind breaker half-open: admitting one probe"
4601 );
4602 }
4603 guard
4604 }
4605 RouteBindAdmission::Refused {
4606 consecutive_timeouts,
4607 retry_in,
4608 probe_in_flight,
4609 } => {
4610 return Ok(vec![self.route_open_breaker_refusal_frame(
4611 ctx,
4612 &frame,
4613 &target_module_id,
4614 consecutive_timeouts,
4615 retry_in,
4616 probe_in_flight,
4617 )?]);
4618 }
4619 };
4620
4621 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4626 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4627 timings.enter(RouteOpenPhase::EgressReservation);
4628 let pending = match self
4629 .forwarding
4630 .begin_route_bind_relay_for(
4631 ctx.connection_id,
4632 ctx.egress.clone(),
4633 response_version(&frame),
4634 frame.header.corr,
4635 &target_module_id,
4636 principal.clone(),
4637 bound_scope,
4638 Some(project_root),
4639 relay_deadline,
4640 )
4641 .await
4642 {
4643 Ok(pending) => pending,
4644 Err(err) => {
4645 return Ok(vec![self.route_open_refusal_frame(
4646 ctx,
4647 &frame,
4648 &target_module_id,
4649 "relay_reservation_failed",
4650 forwarding_error_code(&err),
4651 err.to_string(),
4652 )?])
4653 }
4654 };
4655 timings.enter(RouteOpenPhase::Admission);
4656 let crate::forwarding::PendingRouteBindRelay {
4657 endpoint,
4658 module_sink,
4659 negotiated_ver,
4660 client_channel,
4661 client_epoch,
4662 module_channel,
4663 module_epoch,
4664 corr: relay_corr,
4665 receiver,
4666 } = pending;
4667 let mut reservation =
4668 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4669
4670 let relay_registration = self
4675 .registry
4676 .get_module_by_connection(endpoint.connection_id)
4677 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4678 let relay_capabilities = relay_registration
4679 .as_ref()
4680 .and_then(|registration| registration.manifest.capabilities.as_ref());
4681 if let Some(selector) = &scope {
4682 if let Err(refusal) = self
4685 .scopes
4686 .read()
4687 .unwrap_or_else(|poisoned| poisoned.into_inner())
4688 .check_bind_proof(&principal, selector)
4689 {
4690 reservation.release_and_disarm();
4691 return Ok(vec![self.route_open_refusal_frame(
4692 ctx,
4693 &frame,
4694 &target_module_id,
4695 refusal.code,
4696 refusal.code,
4697 refusal.message,
4698 )?]);
4699 }
4700 }
4701 if let Some(stamp) = scope_stamp
4702 .as_ref()
4703 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4704 {
4705 if let Err(refusal) = crate::scopes::check_target_flow_support(
4706 stamp,
4707 &target_module_id,
4708 relay_capabilities,
4709 )
4710 .and_then(|()| {
4711 crate::scopes::check_target_agent_run_support(
4712 stamp,
4713 &target_module_id,
4714 relay_capabilities,
4715 )
4716 }) {
4717 reservation.release_and_disarm();
4718 return Ok(vec![self.route_open_refusal_frame(
4719 ctx,
4720 &frame,
4721 &target_module_id,
4722 refusal.code,
4723 refusal.code,
4724 refusal.message,
4725 )?]);
4726 }
4727 }
4728
4729 if !relay_capabilities.is_some_and(|caps| {
4732 caps.provides
4733 .iter()
4734 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
4735 }) {
4736 if let Some(stamp) = scope_stamp.as_mut() {
4737 stamp.attributes.on_behalf_of_agent = None;
4738 }
4739 }
4740 let call_mode = if relay_capabilities
4741 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4742 {
4743 Some(derive_call_mode(scope_stamp.as_ref()))
4744 } else {
4745 if let Some(stamp) = scope_stamp.as_mut() {
4746 stamp.expires_at_ms = None;
4747 }
4748 None
4749 };
4750
4751 debug!(
4752 connection_id = ctx.connection_id.get(),
4753 client_channel,
4754 client_epoch,
4755 module_channel,
4756 module_epoch,
4757 "reserved route handle pair"
4758 );
4759 let principal_label = principal_label(&principal);
4762 let relay = ModuleControlRequest::RouteBind {
4763 route_channel: module_channel,
4764 epoch: module_epoch,
4765 target,
4766 identity,
4767 principal: Some(principal),
4768 consumer_capabilities,
4769 role_versions,
4770 admission_facts,
4771 scope: scope_stamp,
4772 call_mode,
4773 };
4774 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4775 RouterError::backend(
4776 0,
4777 frame.header.corr,
4778 format!("failed to encode route.bind request: {err}"),
4779 )
4780 })?;
4781 let relay_frame = Frame::build_with_version(
4782 negotiated_ver,
4783 FrameType::Request,
4784 control_flags(),
4785 0,
4786 0,
4787 relay_corr,
4788 relay_body,
4789 )
4790 .map_err(RouterError::FrameBuild)?;
4791
4792 timings.enter(RouteOpenPhase::BindSend);
4793 let sent = module_sink.send(relay_frame).await;
4794 timings.enter(RouteOpenPhase::Admission);
4795 if let Err(err) = sent {
4796 reservation.release_and_disarm();
4797 return Ok(vec![self.route_open_refusal_frame(
4798 ctx,
4799 &frame,
4800 &target_module_id,
4801 "relay_send_failed",
4802 "target_unavailable",
4803 err.to_string(),
4804 )?]);
4805 }
4806
4807 #[cfg(test)]
4808 if let Some(hold) = &self.route_bind_relay_sent_hold {
4809 hold.wait().await;
4813 hold.wait().await;
4814 }
4815
4816 if self
4821 .forwarding
4822 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4823 .map_err(RouterError::Forwarding)?
4824 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4825 {
4826 self.send_abandoned_route_bind_goodbye(
4827 &module_sink,
4828 negotiated_ver,
4829 module_channel,
4830 module_epoch,
4831 );
4832 }
4833
4834 timings.enter(RouteOpenPhase::BindAckWait);
4835 let outcome = timeout_at(relay_deadline, receiver).await;
4836 timings.enter(RouteOpenPhase::Admission);
4837 match outcome {
4838 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4839 reservation.disarm();
4840 if breaker.record_accepted() {
4841 info!(
4842 module_id = %target_module_id,
4843 "route.bind breaker closed: the probe was accepted"
4844 );
4845 }
4846 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
4847 Ok(Vec::new())
4848 }
4849 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4850 reservation.release_and_disarm();
4851 breaker.record_inconclusive();
4855 let scope_code = match body.code.as_str() {
4859 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4860 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4861 _ => None,
4862 };
4863 if let Some(code) = scope_code {
4864 self.observe_route_open_refusal(
4865 ctx,
4866 &target_module_id,
4867 "scope_changed_before_commit",
4868 code,
4869 );
4870 return Ok(vec![control_error_body_frame(&frame, body)?]);
4871 }
4872 self.counters
4873 .increment_route_open_refused("module_rejected");
4874 info!(
4875 target: "control",
4876 code = "module_rejected",
4877 module_code = ?body.code,
4878 module_id = ?target_module_id,
4879 connection_id = ctx.connection_id.get(),
4880 "route.open refused"
4881 );
4882 Ok(vec![control_error_body_frame(&frame, body)?])
4883 }
4884 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
4885 reservation.release_and_disarm();
4886 breaker.record_inconclusive();
4887 tracing::warn!(
4894 module_id = %target_module_id,
4895 "route.bind relay abandoned: {message}"
4896 );
4897 Ok(vec![self.route_open_refusal_frame(
4898 ctx,
4899 &frame,
4900 &target_module_id,
4901 "relay_abandoned",
4902 "target_unavailable",
4903 message,
4904 )?])
4905 }
4906 Ok(Err(_)) => {
4907 reservation.release_and_disarm();
4908 breaker.record_inconclusive();
4909 Ok(vec![self.route_open_refusal_frame(
4910 ctx,
4911 &frame,
4912 &target_module_id,
4913 "relay_waiter_canceled",
4914 "target_unavailable",
4915 "route.bind relay waiter was canceled before the module responded",
4916 )?])
4917 }
4918 Err(_) => {
4919 reservation.release_and_disarm();
4920 if let Some(opened) = breaker.record_timeout(
4924 self.route_bind_breaker_threshold,
4925 self.route_bind_breaker_cooldown,
4926 ) {
4927 warn!(
4928 module_id = %target_module_id,
4929 consecutive_timeouts = opened.consecutive_timeouts,
4930 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
4931 reopened_after_probe = opened.reopened_after_probe,
4932 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
4933 );
4934 }
4935 tracing::warn!(
4943 module_id = %target_module_id,
4944 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
4945 "route.bind relay timed out: module did not ack within budget"
4946 );
4947 Ok(vec![self.route_open_refusal_frame(
4948 ctx,
4949 &frame,
4950 &target_module_id,
4951 "relay_timed_out",
4952 "module_timeout",
4953 format!(
4954 "module_id '{target_module_id}' did not answer route.bind within {:?}",
4955 route_bind_relay_timeout
4956 ),
4957 )?])
4958 }
4959 }
4960 }
4961
4962 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4963 let response = ClientControlResponse::SupervisorSpawnSnapshot {
4964 snapshot: self.supervisor.spawn_snapshot(),
4965 };
4966 Ok(vec![control_response_body_frame(
4967 &frame,
4968 &response,
4969 "ClientControlResponse::SupervisorSpawnSnapshot",
4970 )?])
4971 }
4972
4973 fn handle_supervisor_spawn_subscribe(
4974 &self,
4975 ctx: &RouteCtx,
4976 frame: Frame,
4977 since: Option<SpawnCursor>,
4978 ) -> Result<Vec<Frame>, RouterError> {
4979 match self.supervisor.subscribe_spawns(
4980 ctx.connection_id,
4981 frame.header.corr,
4982 response_version(&frame),
4983 since,
4984 ctx.egress.clone(),
4985 ) {
4986 Ok(()) => Ok(Vec::new()),
4987 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
4988 Ok(vec![control_error_body_frame(
4989 &frame,
4990 ErrorBody {
4991 code: "spawn_cursor_incarnation_mismatch".to_string(),
4992 message: "spawn cursor belongs to a different daemon incarnation"
4993 .to_string(),
4994 detail: Some(serde_json::json!({
4995 "current_daemon_incarnation": current
4996 })),
4997 },
4998 )?])
4999 }
5000 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5001 &frame,
5002 ErrorBody {
5003 code: "spawn_cursor_too_old".to_string(),
5004 message: "spawn cursor predates the retained event ring".to_string(),
5005 detail: Some(serde_json::json!({
5006 "oldest_retained_cursor": oldest
5007 })),
5008 },
5009 )?]),
5010 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5011 0,
5012 frame.header.corr,
5013 format!("failed to open supervisor spawn subscription: {error}"),
5014 )),
5015 }
5016 }
5017
5018 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5019 let generation = self
5020 .registry
5021 .generation()
5022 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5023 let mut modules = Vec::new();
5024 for module in self.supervisor.list() {
5025 let status = module.status_for_control("list").map_err(|err| {
5026 RouterError::backend(
5027 0,
5028 frame.header.corr,
5029 format!("failed to read supervisor status: {err}"),
5030 )
5031 })?;
5032 let (configured, _) = module.configuration().map_err(|err| {
5033 RouterError::backend(
5034 0,
5035 frame.header.corr,
5036 format!("failed to read module configuration: {err}"),
5037 )
5038 })?;
5039 let image = module.running_image_agreement().await;
5041 let resources = Some(module.child_resource_usage());
5044 let mut verdict =
5045 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5046 verdict.spec = Some(launch_spec_agreement(
5047 &configured,
5048 status.process_alive,
5049 status.launch_spec_digest.as_ref(),
5050 ));
5051 let pending_reload = Some(verdict);
5052 modules.push(
5055 SupervisorEntry::new(
5056 status.module_id,
5057 status.state.to_string(),
5058 status.enabled,
5059 status.live,
5060 status.health.status,
5061 )
5062 .with_launch_nonce_env(Some(!cfg!(unix)))
5063 .with_launch_nonce_source(status.launch_nonce_source)
5064 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5065 .with_protocol(status.protocol)
5066 .with_pending_reload(pending_reload)
5067 .with_last_probe_ms(status.health.last_probe_ms)
5068 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5069 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5070 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5071 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5072 .with_restart_count(Some(status.restart_count))
5073 .with_max_restarts(Some(status.max_restarts))
5074 .with_lifetime_restarts(Some(status.lifetime_restarts))
5075 .with_spawn_generation(Some(status.spawn_generation))
5076 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5077 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5078 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5079 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5080 .with_resources(resources),
5081 );
5082 }
5083 let response = ClientControlResponse::SupervisorList {
5084 generation,
5085 modules,
5086 };
5087 Ok(vec![control_response_body_frame(
5088 &frame,
5089 &response,
5090 "ClientControlResponse::SupervisorList",
5091 )?])
5092 }
5093
5094 fn handle_supervisor_stderr_tail(
5095 &self,
5096 frame: Frame,
5097 module_id: String,
5098 max_lines: Option<u32>,
5099 max_bytes: Option<u32>,
5100 ) -> Result<Vec<Frame>, RouterError> {
5101 let Some(module) = self.supervisor.get(&module_id) else {
5102 return Ok(vec![control_error_frame(
5103 &frame,
5104 "unknown_module",
5105 format!("module_id '{module_id}' is not supervised"),
5106 )?]);
5107 };
5108
5109 let snapshot = module.stderr_tail(
5110 max_lines.map(|value| value as usize),
5111 max_bytes.map(|value| value as usize),
5112 );
5113
5114 let response = ClientControlResponse::SupervisorStderrTail {
5115 module_id,
5116 tail: StderrTail {
5117 capture: match snapshot.capture {
5118 CaptureState::Captured => StderrCaptureState::Captured,
5119 CaptureState::Incomplete { reason } => {
5120 StderrCaptureState::Incomplete { reason }
5121 }
5122 CaptureState::NotCaptured { reason } => {
5123 StderrCaptureState::NotCaptured { reason }
5124 }
5125 },
5126 entries: snapshot
5127 .entries
5128 .into_iter()
5129 .map(|entry| match entry {
5130 TailEntry::Line {
5131 text,
5132 truncated,
5133 at_ms,
5134 } => StderrTailEntry::Line {
5135 text,
5136 truncated,
5137 at_ms,
5138 },
5139 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5140 })
5141 .collect(),
5142 dropped_lines: snapshot.dropped_lines,
5143 },
5144 };
5145 Ok(vec![control_response_body_frame(
5146 &frame,
5147 &response,
5148 "ClientControlResponse::SupervisorStderrTail",
5149 )?])
5150 }
5151
5152 async fn handle_supervisor_terminals(
5153 &self,
5154 frame: Frame,
5155 module_id: String,
5156 ) -> Result<Vec<Frame>, RouterError> {
5157 let Some(module) = self.supervisor.get(&module_id) else {
5158 return Ok(vec![control_error_frame(
5159 &frame,
5160 "unknown_module",
5161 format!("module_id '{module_id}' is not supervised"),
5162 )?]);
5163 };
5164
5165 let terminals = module
5168 .read_durable_terminal_history()
5169 .await
5170 .map_err(|error| {
5171 RouterError::backend(
5172 0,
5173 frame.header.corr,
5174 format!("failed to read terminal history: {error}"),
5175 )
5176 })?;
5177 let response = ClientControlResponse::SupervisorTerminals {
5178 module_id,
5179 terminals,
5180 };
5181 Ok(vec![control_response_body_frame(
5182 &frame,
5183 &response,
5184 "ClientControlResponse::SupervisorTerminals",
5185 )?])
5186 }
5187
5188 fn handle_supervisor_routes(
5189 &self,
5190 frame: Frame,
5191 module_id: Option<String>,
5192 ) -> Result<Vec<Frame>, RouterError> {
5193 let modules = self
5194 .forwarding
5195 .route_census(module_id.as_deref())
5196 .map_err(RouterError::Forwarding)?
5197 .into_iter()
5198 .map(|(module_id, routes)| SupervisorRouteModule {
5199 module_id,
5200 routes: routes
5201 .into_iter()
5202 .map(|route| SupervisorRoute {
5203 consumer: match route.principal {
5204 Principal::Reserved { module_id } => {
5205 SupervisorRouteConsumer::Reserved { module_id }
5206 }
5207 Principal::Direct | Principal::Unverified => {
5208 SupervisorRouteConsumer::Direct {
5209 connection_id: route.goodbye_target.connection_id.get(),
5210 }
5211 }
5212 },
5213 age_ms: Instant::now()
5214 .saturating_duration_since(route.bound_at)
5215 .as_millis()
5216 .try_into()
5217 .unwrap_or(u64::MAX),
5218 draining: route.draining,
5219 drain_reason: route.drain_reason,
5220 })
5221 .collect(),
5222 })
5223 .collect();
5224 let response = ClientControlResponse::SupervisorRoutes { modules };
5225 Ok(vec![control_response_body_frame(
5226 &frame,
5227 &response,
5228 "ClientControlResponse::SupervisorRoutes",
5229 )?])
5230 }
5231
5232 async fn handle_supervisor_provenance(
5233 &self,
5234 frame: Frame,
5235 module_id: Option<String>,
5236 ) -> Result<Vec<Frame>, RouterError> {
5237 let mut selected = if let Some(module_id) = module_id {
5238 let Some(module) = self.supervisor.get(&module_id) else {
5239 return Ok(vec![control_error_frame(
5240 &frame,
5241 "unknown_module",
5242 format!("module_id '{module_id}' is not supervised"),
5243 )?]);
5244 };
5245 vec![module]
5246 } else {
5247 self.supervisor.list()
5248 };
5249
5250 let mut modules = Vec::with_capacity(selected.len());
5251 for module in selected.drain(..) {
5252 let (status, observed_image) = module
5253 .status_and_running_image_agreement()
5254 .await
5255 .map_err(|err| {
5256 RouterError::backend(
5257 0,
5258 frame.header.corr,
5259 format!("failed to read supervisor status: {err}"),
5260 )
5261 })?;
5262 let module_declared = self
5263 .registry
5264 .get_module(&status.module_id)
5265 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5266 .and_then(|registration| registration.manifest.provenance)
5267 .map(|build| ModuleDeclaredProvenance::Reported { build })
5268 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5269 #[cfg(test)]
5270 let running_image = match &self.provenance_probe_override {
5271 Some(result) => result.clone(),
5272 None => observed_image,
5273 };
5274 #[cfg(not(test))]
5275 let running_image = observed_image;
5276 modules.push(SupervisorModuleProvenance {
5277 module_id: status.module_id,
5278 module_declared,
5279 daemon_observed: SupervisorObservedProcess {
5280 launch_nonce_source: status.launch_nonce_source,
5281 pid: status.pid,
5282 spawned_at_ms: status.spawned_at_ms,
5283 spawned_from: status.spawned_from,
5284 running_image,
5285 },
5286 });
5287 }
5288 let daemon = SupervisorDaemonProvenance {
5289 daemon_build: self.daemon_provenance.build.clone(),
5290 daemon_observed: DaemonObservedProcess {
5291 pid: self.daemon_provenance.pid,
5292 started_at_ms: self
5293 .daemon_provenance
5294 .start_clock
5295 .map(|clock| clock.started_at_ms())
5296 .or(self.daemon_provenance.started_at_ms),
5297 running_image: self
5298 .daemon_provenance
5299 .probe
5300 .observe(
5301 self.daemon_provenance.pid,
5302 self.daemon_provenance.executable_path.as_deref(),
5303 self.daemon_provenance.executable_identity.clone(),
5304 self.daemon_provenance.process_start_time,
5305 )
5306 .await,
5307 },
5308 };
5309 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5310 Ok(vec![control_response_body_frame(
5311 &frame,
5312 &response,
5313 "ClientControlResponse::SupervisorProvenance",
5314 )?])
5315 }
5316
5317 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5318 self.refresh_capability_requirements();
5319 let generation = self
5320 .registry
5321 .generation()
5322 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5323 let modules = self
5324 .supervisor
5325 .list()
5326 .into_iter()
5327 .map(|module| {
5328 let status = module.status_for_control("health").map_err(|err| {
5329 RouterError::backend(
5330 0,
5331 frame.header.corr,
5332 format!("failed to read supervisor health: {err}"),
5333 )
5334 })?;
5335 let module_id = status.module_id;
5336 let capability_detail = self
5337 .capability_evaluator
5338 .required_problem_detail(&module_id);
5339 Ok(SupervisorHealthEntry {
5340 module_id,
5341 status: status.health.status,
5342 detail: append_capability_problem_detail(
5343 status.health.detail,
5344 capability_detail,
5345 ),
5346 metrics: status.health.metrics,
5347 consecutive_failures: status.health.consecutive_failures,
5348 late_answer_count: status.health.late_answer_count,
5349 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5350 last_action: status.health.last_action,
5351 last_action_ms: status.health.last_action_ms,
5352 last_probe_ms: status.health.last_probe_ms,
5353 })
5354 })
5355 .collect::<Result<Vec<_>, RouterError>>()?;
5356 let response = ClientControlResponse::SupervisorHealth {
5357 generation,
5358 modules,
5359 };
5360 Ok(vec![control_response_body_frame(
5361 &frame,
5362 &response,
5363 "ClientControlResponse::SupervisorHealth",
5364 )?])
5365 }
5366
5367 async fn handle_supervisor_restart(
5368 &self,
5369 frame: Frame,
5370 module_id: String,
5371 drain_timeout_ms: Option<u64>,
5372 requester: Principal,
5373 ) -> Result<Vec<Frame>, RouterError> {
5374 let operation_lock = self.supervisor.operation_lock();
5375 let _operation_guard = operation_lock.lock().await;
5376 let Some(module) = self.supervisor.get(&module_id) else {
5377 return Ok(vec![control_error_frame(
5378 &frame,
5379 "unknown_module",
5380 format!("module_id '{module_id}' is not supervised"),
5381 )?]);
5382 };
5383
5384 self.route_outages.mark_operator_action(&module_id);
5385 if let Err(err) = module
5386 .restart_requested(drain_timeout_ms, Some(requester))
5387 .await
5388 {
5389 self.route_outages
5390 .operator_action_ended_unrefused(&module_id);
5391 let (code, message) = match err {
5392 crate::supervise::SuperviseError::Disabled { .. } => {
5393 ("module_disabled", err.to_string())
5394 }
5395 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5396 ("swap_in_progress", err.to_string())
5397 }
5398 _ => (
5399 "target_unavailable",
5400 format!("failed to restart module_id '{module_id}': {err}"),
5401 ),
5402 };
5403 return Ok(vec![control_error_frame(&frame, code, message)?]);
5404 }
5405
5406 let response = ClientControlResponse::SupervisorAck {
5407 module_id,
5408 applied: true,
5409 };
5410 Ok(vec![control_response_body_frame(
5411 &frame,
5412 &response,
5413 "ClientControlResponse::SupervisorAck",
5414 )?])
5415 }
5416
5417 async fn handle_supervisor_swap(
5421 &self,
5422 frame: Frame,
5423 module_id: String,
5424 ready_timeout_ms: Option<u64>,
5425 requester: Principal,
5426 ) -> Result<Vec<Frame>, RouterError> {
5427 let module = {
5434 let operation_lock = self.supervisor.operation_lock();
5435 let _operation_guard = operation_lock.lock().await;
5436 self.supervisor.get(&module_id)
5437 };
5438 let Some(module) = module else {
5439 return Ok(vec![control_error_frame(
5440 &frame,
5441 "unknown_module",
5442 format!("module_id '{module_id}' is not supervised"),
5443 )?]);
5444 };
5445
5446 self.route_outages.mark_operator_action(&module_id);
5447 if let Err(err) = module
5448 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5449 .await
5450 {
5451 self.route_outages
5452 .operator_action_ended_unrefused(&module_id);
5453 use crate::supervise::SuperviseError;
5454 let message = err.to_string();
5455 let error = match err {
5456 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5457 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5458 code: "swap_refused".to_string(),
5459 message,
5460 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5461 },
5462 SuperviseError::SwapFailed {
5463 arm,
5464 candidate_exit,
5465 ..
5466 } => ErrorBody {
5467 code: "swap_failed".to_string(),
5468 message,
5469 detail: Some(serde_json::json!({
5470 "arm": arm.as_str(),
5471 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5472 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5473 })),
5474 },
5475 _ => ErrorBody::new(
5476 "target_unavailable",
5477 format!("failed to swap module_id '{module_id}': {message}"),
5478 ),
5479 };
5480 return Ok(vec![control_error_body_frame(&frame, error)?]);
5481 }
5482 self.route_outages
5486 .operator_action_ended_unrefused(&module_id);
5487
5488 let response = ClientControlResponse::SupervisorAck {
5489 module_id,
5490 applied: true,
5491 };
5492 Ok(vec![control_response_body_frame(
5493 &frame,
5494 &response,
5495 "ClientControlResponse::SupervisorAck",
5496 )?])
5497 }
5498
5499 async fn handle_supervisor_reload(
5500 &self,
5501 frame: Frame,
5502 module_id: String,
5503 requester: Principal,
5504 ) -> Result<Vec<Frame>, RouterError> {
5505 let operation_lock = self.supervisor.operation_lock();
5506 let _operation_guard = operation_lock.lock().await;
5507 let Some(module) = self.supervisor.get(&module_id) else {
5508 return Ok(vec![control_error_frame(
5509 &frame,
5510 "unknown_module",
5511 format!("module_id '{module_id}' is not supervised"),
5512 )?]);
5513 };
5514
5515 self.route_outages.mark_operator_action(&module_id);
5516 if let Err(err) = module.reload_requested(Some(requester)).await {
5517 self.route_outages
5518 .operator_action_ended_unrefused(&module_id);
5519 let (code, message) = match err {
5520 crate::supervise::SuperviseError::Disabled { .. } => {
5521 ("module_disabled", err.to_string())
5522 }
5523 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5524 ("swap_in_progress", err.to_string())
5525 }
5526 _ => (
5527 "reload_failed",
5528 format!("failed to reload module_id '{module_id}': {err}"),
5529 ),
5530 };
5531 return Ok(vec![control_error_frame(&frame, code, message)?]);
5532 }
5533
5534 let response = ClientControlResponse::SupervisorAck {
5535 module_id,
5536 applied: true,
5537 };
5538 Ok(vec![control_response_body_frame(
5539 &frame,
5540 &response,
5541 "ClientControlResponse::SupervisorAck",
5542 )?])
5543 }
5544
5545 async fn handle_supervisor_rescan(
5546 &self,
5547 frame: Frame,
5548 preview: bool,
5549 requester: Principal,
5550 ) -> Result<Vec<Frame>, RouterError> {
5551 let Some(context) = self.rescan.clone() else {
5552 return Ok(vec![control_error_frame(
5553 &frame,
5554 "rescan_unavailable",
5555 "the daemon was not started with a reloadable config path".to_string(),
5556 )?]);
5557 };
5558
5559 let operation_lock = self.supervisor.operation_lock();
5560 let _operation_guard = operation_lock.lock().await;
5561 let loaded = match crate::daemon_config::load_with_activity(&context.config_path) {
5562 Ok(config) => config,
5563 Err(err) => {
5564 return Ok(vec![control_error_frame(
5565 &frame,
5566 "invalid_daemon_config",
5567 format!("supervisor rescan rejected daemon config: {err}"),
5568 )?])
5569 }
5570 };
5571 let Some((config, activity_config)) = loaded else {
5579 return Ok(vec![control_error_frame(
5580 &frame,
5581 "invalid_daemon_config",
5582 format!(
5583 "daemon config not found at {}; refusing to rescan (an absent config would \
5584 retire every supervised module)",
5585 context.config_path.display()
5586 ),
5587 )?]);
5588 };
5589 let (
5590 configured_port,
5591 storage_config,
5592 admission_facts_carrier_module_id,
5593 admission_facts_targets,
5594 scope_authority_owners,
5595 privileged_operator_prompt,
5596 modules,
5597 reserved_capabilities,
5598 ) = (
5599 config.port,
5600 config.storage,
5601 config.admission_facts_carrier_module_id,
5602 config.admission_facts_targets,
5603 config.scope_authority_owners,
5604 config.privileged_operator_prompt,
5605 config.modules,
5606 config.reserved_capabilities,
5607 );
5608
5609 let mut restart_required = Vec::new();
5618 for section in RestartRequiredSection::ALL {
5619 let changed = match section {
5620 RestartRequiredSection::Port => configured_port != context.configured_port,
5621 RestartRequiredSection::Storage => storage_config != context.storage_config,
5622 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5623 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5624 }
5625 RestartRequiredSection::AdmissionFactsTargets => {
5626 admission_facts_targets != context.admission_facts_targets
5627 }
5628 RestartRequiredSection::ScopeAuthorityOwners => {
5629 scope_authority_owners != context.scope_authority_owners
5630 }
5631 RestartRequiredSection::PrivilegedOperatorPrompt => {
5632 privileged_operator_prompt != context.privileged_operator_prompt
5633 }
5634 };
5635 if changed {
5636 restart_required.push(section.label().to_string());
5637 }
5638 }
5639 if !restart_required.is_empty() {
5640 warn!(
5641 config_path = %context.config_path.display(),
5642 sections = %restart_required.join(", "),
5643 "daemon config changed outside the modules section; restart the daemon to apply those changes"
5644 );
5645 }
5646
5647 for configured in &modules {
5648 if let Err(err) = validate_spec(&configured.module_spec()) {
5649 return Ok(vec![control_error_frame(
5650 &frame,
5651 "invalid_daemon_config",
5652 format!("supervisor rescan rejected daemon config: {err}"),
5653 )?]);
5654 }
5655 }
5656
5657 let configured_capabilities = modules
5658 .iter()
5659 .map(|module| (module.module_id.clone(), module.enabled))
5660 .collect::<Vec<_>>();
5661 let preview_capability_warnings = if preview {
5662 let (_, registrations) = self.runtime_capability_snapshot()?;
5663 let current_modules = self
5664 .supervisor
5665 .list()
5666 .into_iter()
5667 .map(|module| module.module_id().to_string())
5668 .collect::<BTreeSet<_>>();
5669 let resulting_modules = configured_capabilities.clone();
5670 let removed = current_modules
5671 .into_iter()
5672 .filter(|module_id| {
5673 !resulting_modules
5674 .iter()
5675 .any(|(configured_id, _)| configured_id == module_id)
5676 })
5677 .collect::<Vec<_>>();
5678 self.capability_evaluator.preview_removal_warnings(
5679 resulting_modules,
5680 &removed,
5681 ®istrations,
5682 )
5683 } else {
5684 Vec::new()
5685 };
5686 let result = match self
5687 .reconcile_supervised_modules(&context.supervisor, modules, preview, requester.clone())
5688 .await
5689 {
5690 Ok(result) => result,
5691 Err(message) => {
5692 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5693 }
5694 };
5695 if !preview {
5696 if let Some(ledger) = &self.activity {
5697 ledger.module_rescanned(&result.added, &result.removed, requester);
5698 }
5699 if let Err(code) = self.configure_activity(activity_config).await {
5700 warn!(
5701 code,
5702 "activity retention could not be applied during rescan"
5703 );
5704 }
5705 self.capability_evaluator
5706 .configure(configured_capabilities, reserved_capabilities);
5707 self.capability_evaluator.wake_deadline_loop();
5708 self.refresh_capability_requirements();
5709 }
5710 let mut result = result;
5711 result.restart_required = restart_required;
5712 result.capability_warnings = preview_capability_warnings;
5713 let response = ClientControlResponse::SupervisorRescan { result };
5714 Ok(vec![control_response_body_frame(
5715 &frame,
5716 &response,
5717 "ClientControlResponse::SupervisorRescan",
5718 )?])
5719 }
5720
5721 async fn handle_supervisor_release_reserved(
5722 &self,
5723 frame: Frame,
5724 module_id: String,
5725 ) -> Result<Vec<Frame>, RouterError> {
5726 let Some(context) = self.rescan.clone() else {
5727 return Ok(vec![control_error_frame(
5728 &frame,
5729 "release_unavailable",
5730 "reserved-id release requires a daemon started with a reloadable config path",
5731 )?]);
5732 };
5733 let operation_lock = self.supervisor.operation_lock();
5734 let _operation_guard = operation_lock.lock().await;
5735 let loaded = match crate::daemon_config::load(&context.config_path) {
5736 Ok(Some(config)) => config,
5737 Ok(None) => {
5738 return Ok(vec![control_error_frame(
5739 &frame,
5740 "invalid_daemon_config",
5741 format!(
5742 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5743 context.config_path.display()
5744 ),
5745 )?])
5746 }
5747 Err(err) => {
5748 return Ok(vec![control_error_frame(
5749 &frame,
5750 "invalid_daemon_config",
5751 format!("unable to verify reserved-id release against daemon config: {err}"),
5752 )?])
5753 }
5754 };
5755 if loaded
5756 .modules
5757 .iter()
5758 .any(|configured| configured.module_id == module_id)
5759 {
5760 return Ok(vec![control_error_frame(
5761 &frame,
5762 "reserved_module_configured",
5763 format!(
5764 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5765 ),
5766 )?]);
5767 }
5768 if !self.supervisor.release_retained_reserved_gate(&module_id) {
5769 return Ok(vec![control_error_frame(
5770 &frame,
5771 "reserved_gate_not_retained",
5772 format!(
5773 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
5774 ),
5775 )?]);
5776 }
5777
5778 let response = ClientControlResponse::SupervisorAck {
5779 module_id,
5780 applied: true,
5781 };
5782 Ok(vec![control_response_body_frame(
5783 &frame,
5784 &response,
5785 "ClientControlResponse::SupervisorAck",
5786 )?])
5787 }
5788
5789 async fn reconcile_supervised_modules(
5798 &self,
5799 supervisor: &Supervisor,
5800 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
5801 preview: bool,
5802 requester: Principal,
5803 ) -> Result<SupervisorRescanResult, String> {
5804 let mut current = BTreeMap::new();
5805 for module in self.supervisor.list() {
5806 let (spec, health) = module.configuration().map_err(|err| {
5807 format!(
5808 "failed to read configuration for module_id '{}': {err}",
5809 module.module_id()
5810 )
5811 })?;
5812 let enabled = module
5813 .status()
5814 .map_err(|err| {
5815 format!(
5816 "failed to read status for module_id '{}': {err}",
5817 module.module_id()
5818 )
5819 })?
5820 .enabled;
5821 current.insert(
5822 module.module_id().to_string(),
5823 (module, spec, health, enabled),
5824 );
5825 }
5826 let configured = configured_modules
5827 .into_iter()
5828 .map(|module| (module.module_id.clone(), module))
5829 .collect::<BTreeMap<_, _>>();
5830
5831 let added = configured
5832 .keys()
5833 .filter(|module_id| !current.contains_key(*module_id))
5834 .cloned()
5835 .collect::<Vec<_>>();
5836 let removed = current
5837 .keys()
5838 .filter(|module_id| !configured.contains_key(*module_id))
5839 .cloned()
5840 .collect::<Vec<_>>();
5841 let mut changed_pending_reload = Vec::new();
5842 let mut configuration_changes = BTreeSet::new();
5843 let mut enabled_changes = BTreeSet::new();
5844 let mut unchanged = 0_u32;
5845
5846 for (module_id, configured_module) in &configured {
5847 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5848 else {
5849 continue;
5850 };
5851 let launch_changed = *current_spec != configured_module.module_spec();
5855 let configuration_changed =
5856 launch_changed || *current_health != configured_module.health;
5857 let enabled_changed = *current_enabled != configured_module.enabled;
5858 if configuration_changed {
5859 configuration_changes.insert(module_id.clone());
5860 }
5861 if launch_changed {
5862 changed_pending_reload.push(module_id.clone());
5863 }
5864 if enabled_changed {
5865 enabled_changes.insert(module_id.clone());
5866 }
5867 if !configuration_changed && !enabled_changed {
5868 unchanged = unchanged.saturating_add(1);
5869 }
5870 }
5871
5872 if preview {
5878 return Ok(SupervisorRescanResult {
5879 added,
5880 removed,
5881 changed_pending_reload,
5882 enabled_changes: enabled_changes.iter().cloned().collect(),
5883 unchanged,
5884 preview: true,
5885 restart_required: Vec::new(),
5889 capability_warnings: Vec::new(),
5890 });
5891 }
5892
5893 for module_id in &removed {
5894 let module = ¤t
5895 .get(module_id)
5896 .expect("removed module came from current supervisor state")
5897 .0;
5898 module
5899 .retire_requested(Some(requester.clone()))
5900 .await
5901 .map_err(|err| {
5902 format!("failed to retire module_id '{module_id}' during rescan: {err}")
5903 })?;
5904 self.supervisor.record_rescan_removal(module_id);
5938 self.supervisor.retire(module_id);
5939 self.route_outages.forget(module_id);
5940 }
5941
5942 for module_id in configured.keys() {
5943 let Some((module, _, _, _)) = current.get(module_id) else {
5944 continue;
5945 };
5946 let configured_module = configured
5947 .get(module_id)
5948 .expect("configured module id came from configured map");
5949 if configuration_changes.contains(module_id) {
5950 module
5951 .update_configuration(
5952 configured_module.module_spec(),
5953 configured_module.health.clone(),
5954 configured_module.drain_timeout_ms,
5955 )
5956 .await
5957 .map_err(|err| {
5958 format!(
5959 "failed to update module_id '{module_id}' configuration during rescan: {err}"
5960 )
5961 })?;
5962 }
5963 if enabled_changes.contains(module_id) {
5964 self.route_outages.mark_operator_action(module_id);
5967 module
5968 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
5969 .await
5970 .map_err(|err| {
5971 self.route_outages.operator_action_ended_unrefused(module_id);
5972 format!(
5973 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
5974 configured_module.enabled
5975 )
5976 })?;
5977 }
5978 }
5979
5980 for module_id in &added {
5981 let configured_module = configured
5982 .get(module_id)
5983 .expect("added module id came from configured map");
5984 supervisor
5985 .supervise_configured_requested(
5986 configured_module.module_spec(),
5987 configured_module.enabled,
5988 configured_module.health.clone(),
5989 configured_module.drain_timeout_ms,
5990 configured_module.restart,
5991 Some(requester.clone()),
5992 )
5993 .map_err(|err| {
5994 format!("failed to add module_id '{module_id}' during rescan: {err}")
5995 })?;
5996 }
5997
5998 Ok(SupervisorRescanResult {
5999 added,
6000 removed,
6001 changed_pending_reload,
6002 enabled_changes: enabled_changes.iter().cloned().collect(),
6003 unchanged,
6004 preview: false,
6005 restart_required: Vec::new(),
6008 capability_warnings: Vec::new(),
6009 })
6010 }
6011
6012 async fn handle_supervisor_set_enabled(
6013 &self,
6014 frame: Frame,
6015 module_id: String,
6016 enabled: bool,
6017 requester: Principal,
6018 ) -> Result<Vec<Frame>, RouterError> {
6019 let operation_lock = self.supervisor.operation_lock();
6020 let _operation_guard = operation_lock.lock().await;
6021 let Some(module) = self.supervisor.get(&module_id) else {
6022 return Ok(vec![control_error_frame(
6023 &frame,
6024 "unknown_module",
6025 format!("module_id '{module_id}' is not supervised"),
6026 )?]);
6027 };
6028
6029 self.route_outages.mark_operator_action(&module_id);
6032 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6033 Ok(applied) => applied,
6034 Err(err) => {
6035 self.route_outages
6036 .operator_action_ended_unrefused(&module_id);
6037 return Ok(vec![control_error_frame(
6038 &frame,
6039 "target_unavailable",
6040 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6041 )?]);
6042 }
6043 };
6044 if !applied {
6045 self.route_outages
6048 .operator_action_ended_unrefused(&module_id);
6049 }
6050
6051 self.capability_evaluator.wake_deadline_loop();
6052 self.refresh_capability_requirements();
6053 let response = ClientControlResponse::SupervisorAck { module_id, applied };
6054 Ok(vec![control_response_body_frame(
6055 &frame,
6056 &response,
6057 "ClientControlResponse::SupervisorAck",
6058 )?])
6059 }
6060
6061 async fn handle_supervisor_health_probe(
6062 &self,
6063 frame: Frame,
6064 module_id: String,
6065 ) -> Result<Vec<Frame>, RouterError> {
6066 self.refresh_capability_requirements();
6067 let Some(registration) = self
6068 .registry
6069 .get_module(&module_id)
6070 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6071 else {
6072 return Ok(vec![control_error_frame(
6073 &frame,
6074 "unknown_module",
6075 format!("module_id '{module_id}' is not registered"),
6076 )?]);
6077 };
6078
6079 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6092 {
6093 return Ok(vec![control_error_frame(
6094 &frame,
6095 "health_not_advertised",
6096 format!("module_id '{module_id}' did not advertise health.check"),
6097 )?]);
6098 }
6099
6100 let deadline = Instant::now() + self.health_probe_timeout;
6101 let pending = match self.forwarding.begin_module_control_rpc_for(
6102 &module_id,
6103 MODULE_CONTROL_OP_HEALTH_CHECK,
6104 deadline,
6105 ) {
6106 Ok(pending) => pending,
6107 Err(err) => {
6108 return Ok(vec![control_error_frame(
6109 &frame,
6110 forwarding_error_code(&err),
6111 err.to_string(),
6112 )?])
6113 }
6114 };
6115
6116 let PendingModuleControlRpc {
6117 endpoint,
6118 module_sink,
6119 negotiated_ver,
6120 corr: probe_corr,
6121 receiver,
6122 } = pending;
6123 let mut guard =
6124 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6125 let probe_body =
6126 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6127 RouterError::backend(
6128 0,
6129 frame.header.corr,
6130 format!("failed to encode health.check request: {err}"),
6131 )
6132 })?;
6133 let probe_frame = Frame::build_with_version(
6134 negotiated_ver,
6135 FrameType::Request,
6136 control_flags(),
6137 0,
6138 0,
6139 probe_corr,
6140 probe_body,
6141 )
6142 .map_err(RouterError::FrameBuild)?;
6143
6144 match timeout_at(deadline, module_sink.send(probe_frame)).await {
6147 Ok(Ok(())) => {}
6148 Ok(Err(err)) => {
6149 return Ok(vec![control_error_frame(
6150 &frame,
6151 "target_unavailable",
6152 err.to_string(),
6153 )?]);
6154 }
6155 Err(_) => {
6156 return Ok(vec![control_error_frame(
6157 &frame,
6158 "module_timeout",
6159 format!(
6160 "module_id '{module_id}' did not answer health.check within {:?}",
6161 self.health_probe_timeout
6162 ),
6163 )?]);
6164 }
6165 }
6166
6167 match timeout_at(deadline, receiver).await {
6168 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6169 guard.disarm();
6170 let Some(report) = response.health_report() else {
6171 return Ok(vec![control_error_frame(
6172 &frame,
6173 "invalid_control_body",
6174 "health.check RPC returned a non-health response",
6175 )?]);
6176 };
6177 let HealthReport {
6182 status,
6183 detail,
6184 metrics,
6185 } = report;
6186 let capability_detail = self
6187 .capability_evaluator
6188 .required_problem_detail(&module_id);
6189 let response = ClientControlResponse::SupervisorHealthProbe {
6190 module_id,
6191 status,
6192 detail: append_capability_problem_detail(detail, capability_detail),
6193 metrics,
6194 };
6195 Ok(vec![control_response_body_frame(
6196 &frame,
6197 &response,
6198 "ClientControlResponse::SupervisorHealthProbe",
6199 )?])
6200 }
6201 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6202 guard.disarm();
6203 Ok(vec![control_error_body_frame(&frame, body)?])
6204 }
6205 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6206 guard.disarm();
6207 Ok(vec![control_error_frame(
6208 &frame,
6209 "target_unavailable",
6210 message,
6211 )?])
6212 }
6213 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6214 guard.disarm();
6215 Ok(vec![control_error_frame(
6216 &frame,
6217 "invalid_control_body",
6218 message,
6219 )?])
6220 }
6221 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6222 guard.disarm();
6223 Ok(vec![control_error_frame(
6224 &frame,
6225 "invalid_control_body",
6226 format!("expected module-control op '{expected}', got '{actual}'"),
6227 )?])
6228 }
6229 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6230 guard.disarm();
6231 Ok(vec![control_error_frame(
6232 &frame,
6233 "module_timeout",
6234 format!(
6235 "module_id '{module_id}' answered health.check after {:?}",
6236 self.health_probe_timeout
6237 ),
6238 )?])
6239 }
6240 Ok(Err(_)) => Ok(vec![control_error_frame(
6241 &frame,
6242 "target_unavailable",
6243 "health.check waiter was canceled before the module responded",
6244 )?]),
6245 Err(_) => Ok(vec![control_error_frame(
6246 &frame,
6247 "module_timeout",
6248 format!(
6249 "module_id '{module_id}' did not answer health.check within {:?}",
6250 self.health_probe_timeout
6251 ),
6252 )?]),
6253 }
6254 }
6255
6256 fn supervisor_status(
6257 &self,
6258 module_id: &str,
6259 corr: u64,
6260 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6261 self.supervisor
6262 .get(module_id)
6263 .map(|module| {
6264 let warming = module.is_warming_for_control("status").map_err(|err| {
6265 RouterError::backend(
6266 0,
6267 corr,
6268 format!(
6269 "failed to read supervisor warming state for module_id '{module_id}': {err}"
6270 ),
6271 )
6272 })?;
6273 module.status_for_control("status").map_err(|err| {
6274 RouterError::backend(
6275 0,
6276 corr,
6277 format!(
6278 "failed to read supervisor status for module_id '{module_id}': {err}"
6279 ),
6280 )
6281 }).map(|status| (status, warming))
6282 })
6283 .transpose()
6284 }
6285
6286 fn guard_module_control_op(
6287 &self,
6288 frame: &Frame,
6289 module_id: &str,
6290 op: &str,
6291 ) -> Result<Option<Frame>, RouterError> {
6292 if self.module_grants_op(module_id, op, frame.header.corr)? {
6293 return Ok(None);
6294 }
6295
6296 Ok(Some(control_error_frame(
6297 frame,
6298 "op_not_allowed",
6299 format!("module_id '{module_id}' did not grant control op '{op}'"),
6300 )?))
6301 }
6302
6303 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6304 let Some(registration) = self
6305 .registry
6306 .get_module(module_id)
6307 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6308 else {
6309 return Ok(false);
6310 };
6311 Ok(module_registration_grants_op(®istration.control_ops, op))
6312 }
6313
6314 fn handle_status_update(
6315 &self,
6316 endpoint: ModuleEndpointId,
6317 frame: Frame,
6318 ) -> Result<Vec<Frame>, RouterError> {
6319 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6320 Ok(update) => update,
6321 Err(err) => {
6322 if is_known_module_push_op(&frame.body) {
6327 return Ok(vec![control_error_frame(
6328 &frame,
6329 "invalid_control_body",
6330 format!("malformed module control push body: {err}"),
6331 )?]);
6332 }
6333 return Ok(Vec::new());
6334 }
6335 };
6336
6337 match update {
6338 ModuleControlPush::RouteStatus {
6339 route_channel,
6340 route_epoch,
6341 status,
6342 } => {
6343 self.forwarding
6344 .cache_status(endpoint, route_channel, route_epoch, status)
6345 .map_err(RouterError::Forwarding)?;
6346 }
6347 }
6348 Ok(Vec::new())
6349 }
6350
6351 fn handle_route_poll(
6352 &self,
6353 ctx: &RouteCtx,
6354 frame: Frame,
6355 route_channel: u16,
6356 route_epoch: u32,
6357 kind: PollKind,
6358 ) -> Result<Vec<Frame>, RouterError> {
6359 let snapshot = self
6360 .forwarding
6361 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6362 .map_err(RouterError::Forwarding)?;
6363 let response = match (kind, snapshot) {
6364 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6365 ClientControlResponse::RoutePoll {
6366 route_channel,
6367 route_epoch,
6368 status,
6369 live: None,
6370 }
6371 }
6372 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6373 route_channel,
6374 route_epoch,
6375 status: None,
6376 live: None,
6377 },
6378 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6379 let live = self
6405 .process_liveness
6406 .as_ref()
6407 .and_then(|source| source.process_live(&module_id))
6408 .unwrap_or(true);
6409 ClientControlResponse::RoutePoll {
6410 route_channel,
6411 route_epoch,
6412 status: None,
6413 live: Some(live),
6414 }
6415 }
6416 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6417 route_channel,
6418 route_epoch,
6419 status: None,
6420 live: Some(false),
6421 },
6422 };
6423
6424 Ok(vec![control_response_body_frame(
6425 &frame,
6426 &response,
6427 "ClientControlResponse::RoutePoll",
6428 )?])
6429 }
6430
6431 pub(crate) fn observe_module_control_completion(
6432 &self,
6433 completion: ModuleControlRpcCompletion,
6434 ) -> bool {
6435 match completion {
6436 ModuleControlRpcCompletion::Unknown => false,
6437 ModuleControlRpcCompletion::Settled => true,
6438 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6439 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6440 info!(
6441 module_id = %module_id,
6442 latency_ms,
6443 "late health.check answer proves the module is alive"
6444 );
6445 match self
6446 .supervisor
6447 .record_late_health_answer(&module_id, latency_ms)
6448 {
6449 Ok(true) => {}
6450 Ok(false) => debug!(
6451 module_id = %module_id,
6452 latency_ms,
6453 "late health.check answer has no active supervisor snapshot"
6454 ),
6455 Err(err) => warn!(
6456 module_id = %module_id,
6457 latency_ms,
6458 error = %err,
6459 "failed to record late health.check answer"
6460 ),
6461 }
6462 true
6463 }
6464 }
6465 }
6466
6467 fn refuse_to_end_module_connection_for_a_client(
6488 &self,
6489 module_connection_id: ConnectionId,
6490 corr: u64,
6491 err: ForwardingError,
6492 ) -> Result<(), RouterError> {
6493 if let ForwardingError::ConnectionClosing { connection_id } = err {
6494 if connection_id != module_connection_id {
6495 warn!(
6496 module_connection_id = module_connection_id.get(),
6497 client_connection_id = connection_id.get(),
6498 corr,
6499 "dropping a route.bind response for a closing client; the module connection keeps serving"
6500 );
6501 return Ok(());
6502 }
6503 }
6504 Err(RouterError::Forwarding(err))
6505 }
6506
6507 fn handle_module_relay_response(
6508 &self,
6509 connection_id: ConnectionId,
6510 frame: Frame,
6511 ) -> Result<Vec<Frame>, RouterError> {
6512 let mut secondary_error = None;
6513 let outcome = match frame.header.ty {
6514 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6515 Ok(probe) if probe.op == "route.bind" => {
6516 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6517 Ok(ModuleControlResponse::RouteBindAck {}) => {
6518 RouteBindRelayOutcome::Accepted
6519 }
6520 Ok(other) => {
6521 let message =
6522 format!("route.bind response carried unexpected body: {other:?}");
6523 secondary_error = Some(control_error_frame(
6524 &frame,
6525 "invalid_control_body",
6526 message.clone(),
6527 )?);
6528 RouteBindRelayOutcome::ModuleGone(message)
6529 }
6530 Err(err) => {
6531 let message = format!("malformed route.bind response body: {err}");
6532 secondary_error = Some(control_error_frame(
6533 &frame,
6534 "invalid_control_body",
6535 message.clone(),
6536 )?);
6537 RouteBindRelayOutcome::ModuleGone(message)
6538 }
6539 }
6540 }
6541 Ok(probe) => {
6542 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6543 {
6544 Ok(response) => ModuleControlRpcOutcome::Response(response),
6545 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6546 "malformed {} response body: {err}",
6547 probe.op
6548 )),
6549 };
6550 let completion = self
6551 .forwarding
6552 .complete_module_control_rpc(
6553 connection_id,
6554 frame.header.corr,
6555 Some(&probe.op),
6556 outcome,
6557 )
6558 .map_err(RouterError::Forwarding)?;
6559 if !self.observe_module_control_completion(completion) {
6560 debug!(
6561 connection_id = connection_id.get(),
6562 corr = frame.header.corr,
6563 op = %probe.op,
6564 "dropping late or unknown module-control RPC response"
6565 );
6566 }
6567 return Ok(Vec::new());
6568 }
6569 Err(err) => {
6570 if let Some(expected_op) = self
6571 .forwarding
6572 .pending_module_control_op(connection_id, frame.header.corr)
6573 .map_err(RouterError::Forwarding)?
6574 {
6575 let completion = self
6576 .forwarding
6577 .complete_module_control_rpc(
6578 connection_id,
6579 frame.header.corr,
6580 None,
6581 ModuleControlRpcOutcome::MalformedResponse(format!(
6582 "malformed {expected_op} response body: {err}"
6583 )),
6584 )
6585 .map_err(RouterError::Forwarding)?;
6586 if !self.observe_module_control_completion(completion) {
6587 debug!(
6588 connection_id = connection_id.get(),
6589 corr = frame.header.corr,
6590 "dropping late malformed module-control RPC response"
6591 );
6592 }
6593 return Ok(Vec::new());
6594 }
6595 let message = format!("malformed route.bind response body: {err}");
6596 secondary_error = Some(control_error_frame(
6597 &frame,
6598 "invalid_control_body",
6599 message.clone(),
6600 )?);
6601 RouteBindRelayOutcome::ModuleGone(message)
6602 }
6603 },
6604 FrameType::Error => {
6605 if self
6606 .forwarding
6607 .pending_module_control_op(connection_id, frame.header.corr)
6608 .map_err(RouterError::Forwarding)?
6609 .is_some()
6610 {
6611 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6612 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
6613 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6614 "malformed module-control ERROR body: {err}"
6615 )),
6616 };
6617 let completion = self
6618 .forwarding
6619 .complete_module_control_rpc(
6620 connection_id,
6621 frame.header.corr,
6622 None,
6623 outcome,
6624 )
6625 .map_err(RouterError::Forwarding)?;
6626 if !self.observe_module_control_completion(completion) {
6627 debug!(
6628 connection_id = connection_id.get(),
6629 corr = frame.header.corr,
6630 "dropping late or unknown module-control RPC error"
6631 );
6632 }
6633 return Ok(Vec::new());
6634 }
6635 match serde_json::from_slice::<ErrorBody>(&frame.body) {
6636 Ok(body) => RouteBindRelayOutcome::Rejected(body),
6637 Err(err) => {
6638 let message = format!("malformed route.bind ERROR body: {err}");
6639 secondary_error = Some(control_error_frame(
6640 &frame,
6641 "invalid_control_body",
6642 message.clone(),
6643 )?);
6644 RouteBindRelayOutcome::ModuleGone(message)
6645 }
6646 }
6647 }
6648 ty => {
6649 return Ok(vec![control_error_frame(
6650 &frame,
6651 "unsupported_control_frame",
6652 format!("unsupported module channel-0 frame {ty:?}"),
6653 )?])
6654 }
6655 };
6656
6657 let settled =
6658 self.forwarding
6659 .complete_pending_relay(connection_id, frame.header.corr, outcome);
6660 let completion = match settled {
6661 Ok(completion) => completion,
6662 Err(err) => {
6663 self.refuse_to_end_module_connection_for_a_client(
6664 connection_id,
6665 frame.header.corr,
6666 err,
6667 )?;
6668 return Ok(secondary_error.into_iter().collect());
6669 }
6670 };
6671 if let Some(target) = completion.abandoned.as_ref() {
6672 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6673 }
6674 if !completion.settled {
6675 debug!(
6676 connection_id = connection_id.get(),
6677 corr = frame.header.corr,
6678 frame_type = ?frame.header.ty,
6679 "dropping late or unknown route.bind relay response"
6680 );
6681 }
6682 Ok(secondary_error.into_iter().collect())
6683 }
6684
6685 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6686 debug!(connection_id = connection_id.get(), "handling GOODBYE");
6687 self.cleanup_connection_with_end_reason(
6691 connection_id,
6692 RegistrationEndReason::ExplicitGoodbye,
6693 )
6694 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6695 Ok(Vec::new())
6696 }
6697}
6698
6699impl Default for ControlHandler {
6700 fn default() -> Self {
6701 Self::new(Arc::new(Registry::default()))
6702 }
6703}
6704
6705impl crate::supervise::SwapPromotionObserver for ControlHandler {
6706 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6707 self.apply_registration_capabilities(registration);
6708 }
6709}
6710
6711fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6712 CapabilityRequirementStatus {
6713 consumer: status.consumer,
6714 capability: status.capability,
6715 need: match status.need {
6716 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6717 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6718 },
6719 verdict: status.verdict.as_str().to_string(),
6720 episode_seq: status.episode_seq,
6721 config_satisfiable: status.config_satisfiable,
6722 runtime_available: status.runtime_available,
6723 detail: status.detail,
6724 }
6725}
6726
6727fn append_capability_problem_detail(
6728 detail: Option<String>,
6729 capability_detail: Option<String>,
6730) -> Option<String> {
6731 match (detail, capability_detail) {
6732 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
6733 (Some(detail), None) => Some(detail),
6734 (None, Some(capability_detail)) => Some(capability_detail),
6735 (None, None) => None,
6736 }
6737}
6738
6739fn subc_ops() -> Vec<String> {
6740 SUBC_CONTROL_OPS
6741 .iter()
6742 .map(|op| (*op).to_string())
6743 .collect()
6744}
6745
6746fn module_subc_ops() -> Vec<String> {
6747 SUBC_CONTROL_OPS
6748 .iter()
6749 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
6750 .map(|op| (*op).to_string())
6751 .collect()
6752}
6753
6754#[cfg(test)]
6755fn module_baseline_control_ops() -> Vec<String> {
6756 MODULE_BASELINE_CONTROL_OPS
6757 .iter()
6758 .map(|op| (*op).to_string())
6759 .collect()
6760}
6761
6762fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
6763 let mut seen = HashSet::new();
6764 let mut effective = Vec::new();
6765 for op in MODULE_BASELINE_CONTROL_OPS {
6766 if seen.insert((*op).to_string()) {
6767 effective.push((*op).to_string());
6768 }
6769 }
6770 for op in declared.unwrap_or_default() {
6771 if seen.insert(op.clone()) {
6772 effective.push(op);
6773 }
6774 }
6775 effective
6776}
6777
6778fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
6779 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
6780}
6781
6782fn target_module_id(target: &RouteTarget) -> &str {
6783 match target {
6784 RouteTarget::ToolProvider { module_id }
6785 | RouteTarget::ManagementSurface { module_id }
6786 | RouteTarget::InternalService { module_id, .. } => module_id,
6787 }
6788}
6789
6790fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
6791 roles.iter().any(|role| match (target, role) {
6792 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
6793 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
6794 (
6795 RouteTarget::InternalService { service_id, .. },
6796 ProviderRole::InternalService {
6797 service_id: provided,
6798 ..
6799 },
6800 ) => service_id == provided,
6801 _ => false,
6802 })
6803}
6804
6805fn is_routable_role(role: &ProviderRole) -> bool {
6806 matches!(
6807 role,
6808 ProviderRole::ToolProvider { .. }
6809 | ProviderRole::ManagementSurface { .. }
6810 | ProviderRole::InternalService { .. }
6811 )
6812}
6813
6814#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6815enum ControlRequestBodyError {
6816 UnknownOp,
6817 InvalidBody,
6818}
6819
6820#[derive(Debug, Deserialize)]
6821struct ControlOpProbe {
6822 op: String,
6823}
6824
6825const MODULE_PUSH_OPS: &[&str] = &["route.status"];
6828
6829fn is_known_module_push_op(body: &[u8]) -> bool {
6830 serde_json::from_slice::<ControlOpProbe>(body)
6831 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
6832 .unwrap_or(false)
6833}
6834
6835fn is_known_module_request_op(body: &[u8]) -> bool {
6836 serde_json::from_slice::<ControlOpProbe>(body)
6837 .map(|probe| is_module_to_subc_op(&probe.op))
6838 .unwrap_or(false)
6839}
6840
6841fn is_module_to_subc_op(op: &str) -> bool {
6842 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
6843}
6844
6845fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
6846 debug!(
6847 op = %op,
6848 connection_id = connection_id.get(),
6849 corr,
6850 "control dispatch"
6851 );
6852}
6853
6854fn log_slow_control_dispatch(
6855 dispatch_started_at: Option<StdInstant>,
6856 op: &'static str,
6857 connection_id: ConnectionId,
6858 corr: u64,
6859 route_open_timings: Option<&RouteOpenTimings>,
6860) {
6861 let Some(dispatch_started_at) = dispatch_started_at else {
6862 return;
6863 };
6864 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
6866 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
6867 if let Some(timings) = route_open_timings {
6868 warn!(
6869 op = %op,
6870 connection_id = connection_id.get(),
6871 corr,
6872 elapsed_ms = elapsed.as_millis() as u64,
6873 module_id = ?timings.module_id,
6874 principal = %timings.principal,
6875 admission_ms = timings.admission.as_millis() as u64,
6876 project_root_ms = timings.project_root.as_millis() as u64,
6877 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
6878 bind_send_ms = timings.bind_send.as_millis() as u64,
6879 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
6880 "slow control dispatch"
6881 );
6882 return;
6883 }
6884 warn!(
6885 op = %op,
6886 connection_id = connection_id.get(),
6887 corr,
6888 elapsed_ms = elapsed.as_millis() as u64,
6889 "slow control dispatch"
6890 );
6891 }
6892}
6893
6894#[cfg(test)]
6896type TestCallerLookup =
6897 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
6898
6899fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
6901 match request {
6902 ClientControlRequest::SupervisorRestart { module_id, .. } => {
6903 Some(PrivilegedOperation::Restart { module_id })
6904 }
6905 ClientControlRequest::SupervisorSwap { module_id, .. } => {
6906 Some(PrivilegedOperation::Swap { module_id })
6907 }
6908 ClientControlRequest::SupervisorReload { module_id } => {
6909 Some(PrivilegedOperation::Reload { module_id })
6910 }
6911 ClientControlRequest::SupervisorRescan { preview } => {
6912 if *preview {
6913 None
6914 } else {
6915 Some(PrivilegedOperation::Rescan)
6916 }
6917 }
6918 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
6919 Some(PrivilegedOperation::SetEnabled {
6920 module_id,
6921 enabled: *enabled,
6922 })
6923 }
6924 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
6925 Some(PrivilegedOperation::ReleaseReserved { module_id })
6926 }
6927 ClientControlRequest::ServerDescribe {}
6928 | ClientControlRequest::CatalogList { .. }
6929 | ClientControlRequest::RouteOpen { .. }
6930 | ClientControlRequest::RoutePoll { .. }
6931 | ClientControlRequest::SupervisorList {}
6932 | ClientControlRequest::SupervisorHealth {}
6933 | ClientControlRequest::SupervisorHealthProbe { .. }
6934 | ClientControlRequest::SupervisorRoutes { .. }
6935 | ClientControlRequest::SupervisorProvenance { .. }
6936 | ClientControlRequest::SupervisorStderrTail { .. }
6937 | ClientControlRequest::SupervisorTerminals { .. }
6938 | ClientControlRequest::SupervisorSpawnSnapshot {}
6939 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
6940 }
6941}
6942
6943fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
6944 match request {
6945 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
6946 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
6947 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
6948 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
6949 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
6950 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
6951 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
6952 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
6953 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
6954 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
6955 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
6956 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
6957 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
6958 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
6959 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
6960 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
6961 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
6962 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
6963 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
6964 }
6965}
6966
6967fn principal_label(principal: &Principal) -> String {
6968 match principal {
6969 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
6970 Principal::Direct => "direct".to_string(),
6971 other => format!("{other:?}"),
6972 }
6973}
6974
6975fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
6976 match request {
6977 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
6978 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
6979 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
6980 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
6981 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
6982 }
6983}
6984
6985fn parse_client_control_request(
6986 body: &[u8],
6987) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
6988 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
6989 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6990 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
6991 ControlRequestBodyError::InvalidBody
6992 }
6993 Ok(_) => ControlRequestBodyError::UnknownOp,
6994 Err(_) => ControlRequestBodyError::InvalidBody,
6995 };
6996 (err, classification)
6997 })
6998}
6999
7000fn parse_module_control_request_from_module(
7001 body: &[u8],
7002) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7003 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7004 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7005 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7006 Ok(_) => ControlRequestBodyError::UnknownOp,
7007 Err(_) => ControlRequestBodyError::InvalidBody,
7008 };
7009 (err, classification)
7010 })
7011}
7012
7013#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7014enum ProviderRoleKind {
7015 ToolProvider,
7016 PipelineStage,
7017 ManagementSurface,
7018 InternalService,
7019}
7020
7021fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7022 match role {
7023 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7024 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7025 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7026 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7027 }
7028}
7029
7030fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7031 roles.iter().map(provider_role_kind).collect()
7032}
7033
7034const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7037
7038#[derive(Debug, Default, PartialEq, Eq)]
7040struct ScopeOutcomeCounts {
7041 created: usize,
7042 replaced: usize,
7043 updated: usize,
7044 unchanged: usize,
7045 refused: usize,
7046}
7047
7048impl ScopeOutcomeCounts {
7049 fn of(results: &[ScopeRecordResult]) -> Self {
7050 let mut counts = Self::default();
7051 for result in results {
7052 let slot = match result.outcome {
7053 ScopeRecordOutcome::Created => &mut counts.created,
7054 ScopeRecordOutcome::Replaced => &mut counts.replaced,
7055 ScopeRecordOutcome::Updated => &mut counts.updated,
7056 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7057 ScopeRecordOutcome::Refused => &mut counts.refused,
7058 };
7059 *slot += 1;
7060 }
7061 counts
7062 }
7063}
7064
7065#[cfg(test)]
7066mod scope_outcome_count_tests {
7067 use super::*;
7068
7069 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7070 ScopeRecordResult {
7071 scope_ref: "r".to_string(),
7072 scope_epoch: 1,
7073 outcome,
7074 code: None,
7075 message: None,
7076 version: None,
7077 parent_state: None,
7078 }
7079 }
7080
7081 #[test]
7084 fn every_outcome_is_counted_in_its_own_field() {
7085 let results = [
7086 result(ScopeRecordOutcome::Created),
7087 result(ScopeRecordOutcome::Created),
7088 result(ScopeRecordOutcome::Replaced),
7089 result(ScopeRecordOutcome::Updated),
7090 result(ScopeRecordOutcome::Unchanged),
7091 result(ScopeRecordOutcome::Refused),
7092 result(ScopeRecordOutcome::Refused),
7093 result(ScopeRecordOutcome::Refused),
7094 ];
7095 assert_eq!(
7096 ScopeOutcomeCounts::of(&results),
7097 ScopeOutcomeCounts {
7098 created: 2,
7099 replaced: 1,
7100 updated: 1,
7101 unchanged: 1,
7102 refused: 3,
7103 }
7104 );
7105 }
7106}
7107
7108fn capability_census_trigger(
7112 old: Option<&CapabilityDeclarations>,
7113 new: Option<&CapabilityDeclarations>,
7114) -> bool {
7115 let old_provides = old
7116 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7117 .unwrap_or_default();
7118 let old_denies = old
7119 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7120 .unwrap_or_default();
7121 let new = new.cloned().unwrap_or(CapabilityDeclarations {
7122 provides: Vec::new(),
7123 requires: Vec::new(),
7124 must_never_reach: Vec::new(),
7125 });
7126
7127 new.provides
7128 .iter()
7129 .any(|capability| !old_provides.contains(capability))
7130 || new
7131 .must_never_reach
7132 .iter()
7133 .any(|capability| !old_denies.contains(capability))
7134}
7135
7136fn denied_capability<'a>(
7139 opening_manifest: &'a ModuleManifest,
7140 target_manifest: &ModuleManifest,
7141) -> Option<&'a str> {
7142 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7143 let target_capabilities = target_manifest.capabilities.as_ref()?;
7144 opening_capabilities
7145 .must_never_reach
7146 .iter()
7147 .find(|denied| {
7148 target_capabilities
7149 .provides
7150 .iter()
7151 .any(|provided| provided == *denied)
7152 })
7153 .map(String::as_str)
7154}
7155
7156fn catalog_update_frozen_field_message(
7157 registered: &ModuleManifest,
7158 provides: &[ProviderRole],
7159) -> Option<String> {
7160 let old_has_provides = !registered.provides.is_empty();
7161 let new_has_provides = !provides.is_empty();
7162 if old_has_provides != new_has_provides {
7163 return Some(format!(
7164 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7165 registered.module_id
7166 ));
7167 }
7168
7169 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
7170 return Some(format!(
7171 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7172 registered.module_id
7173 ));
7174 }
7175
7176 let registered_concurrency = manifest_concurrency(registered);
7177 let mut candidate = registered.clone();
7178 candidate.provides = provides.to_vec();
7179 let candidate_concurrency = manifest_concurrency(&candidate);
7180 if candidate_concurrency != registered_concurrency {
7181 return Some(format!(
7182 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7183 registered.module_id, registered_concurrency, candidate_concurrency
7184 ));
7185 }
7186
7187 None
7190}
7191
7192fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7193 manifest.provides.iter().any(is_routable_role)
7194}
7195
7196fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7202 manifest
7203 .provides
7204 .iter()
7205 .find_map(|provider| match provider {
7206 ProviderRole::ToolProvider { concurrency, .. }
7207 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7208 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7209 })
7210 .unwrap_or(Concurrency::ModuleManaged)
7211}
7212
7213fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7220 let has_management_surface = manifest
7221 .provides
7222 .iter()
7223 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7224 if !has_management_surface {
7225 return false;
7226 }
7227 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7228 return false;
7229 };
7230 let Some(provides) = raw
7231 .get("manifest")
7232 .and_then(|manifest| manifest.get("provides"))
7233 .and_then(serde_json::Value::as_array)
7234 else {
7235 return false;
7236 };
7237 provides.iter().any(|role| {
7242 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7243 && role.get("concurrency").is_none()
7244 })
7245}
7246
7247fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7248 if peer_version != PROTOCOL_VERSION {
7249 return Err(format!(
7250 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7251 ));
7252 }
7253 Ok(PROTOCOL_VERSION)
7254}
7255
7256fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7257 Frame::build_with_version(
7258 response_version(frame),
7259 FrameType::Pong,
7260 frame.header.flags,
7261 0,
7262 0,
7263 frame.header.corr,
7264 Vec::new(),
7265 )
7266 .map_err(RouterError::FrameBuild)
7267}
7268
7269pub(crate) fn control_error_frame(
7270 frame: &Frame,
7271 code: &'static str,
7272 message: impl Into<String>,
7273) -> Result<Frame, RouterError> {
7274 control_error_body_frame(
7275 frame,
7276 ErrorBody {
7277 code: code.to_string(),
7278 message: message.into(),
7279 detail: None,
7280 },
7281 )
7282}
7283
7284fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7285 let body = serde_json::to_vec(&error).map_err(|err| {
7286 RouterError::backend(
7287 0,
7288 frame.header.corr,
7289 format!("failed to encode control ERROR: {err}"),
7290 )
7291 })?;
7292
7293 Frame::build_with_version(
7294 response_version(frame),
7295 FrameType::Error,
7296 control_flags(),
7297 0,
7298 0,
7299 frame.header.corr,
7300 body,
7301 )
7302 .map_err(RouterError::FrameBuild)
7303}
7304
7305fn control_response_body_frame<T: Serialize>(
7306 frame: &Frame,
7307 reply: &T,
7308 label: &'static str,
7309) -> Result<Frame, RouterError> {
7310 let body = serde_json::to_vec(reply).map_err(|err| {
7311 RouterError::backend(
7312 0,
7313 frame.header.corr,
7314 format!("failed to encode {label}: {err}"),
7315 )
7316 })?;
7317
7318 Frame::build_with_version(
7319 response_version(frame),
7320 FrameType::Response,
7321 control_flags(),
7322 0,
7323 0,
7324 frame.header.corr,
7325 body,
7326 )
7327 .map_err(RouterError::FrameBuild)
7328}
7329
7330fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7360 match err {
7361 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7362 ForwardingError::NoModuleConnection => "target_unavailable",
7363 ForwardingError::ModuleReloading { .. } => "module_reloading",
7364 ForwardingError::ClientRouteChannelExhausted { .. }
7365 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7366 ForwardingError::StaleModuleEndpoint
7367 | ForwardingError::UnknownReservation { .. }
7368 | ForwardingError::ConnectionClosing { .. }
7369 | ForwardingError::ClientEgressClosed { .. }
7370 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7371 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7374 ForwardingError::RelayCorrelationExhausted
7375 | ForwardingError::RouteOpenBuild(_)
7376 | ForwardingError::Poisoned => "forwarding_error",
7377 }
7378}
7379
7380fn response_version(frame: &Frame) -> u8 {
7381 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7382 frame.header.ver
7383 } else {
7384 PROTOCOL_VERSION
7385 }
7386}
7387
7388fn control_flags() -> Flags {
7389 Flags::new(false, Priority::Passive, false)
7390}
7391
7392fn send_goodbye_target_best_effort(
7397 counters: &DaemonCounters,
7398 target: &GoodbyeTarget,
7399 context: &'static str,
7400) {
7401 let Ok(frame) = Frame::build_with_version(
7402 target.negotiated_ver,
7403 FrameType::Goodbye,
7404 control_flags(),
7405 target.channel,
7406 target.epoch,
7407 0,
7408 Vec::new(),
7409 ) else {
7410 return;
7411 };
7412 crate::forwarding::send_module_route_goodbye(
7413 counters,
7414 &target.sink,
7415 frame,
7416 target.module_id.as_deref(),
7417 context,
7418 );
7419}
7420
7421pub(crate) fn send_route_control_pushes(
7422 forwarding: &ForwardingTable,
7423 routes: Vec<EndpointRoute>,
7424 push: ClientControlPush,
7425) {
7426 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7427 for route in routes {
7428 let target = route.goodbye_target;
7429 if let Some((existing, channels)) = targets
7430 .iter_mut()
7431 .find(|(existing, _)| existing.connection_id == target.connection_id)
7432 {
7433 debug_assert_eq!(
7434 existing.negotiated_ver, target.negotiated_ver,
7435 "one connection cannot negotiate multiple frame versions"
7436 );
7437 if !channels.contains(&target.channel) {
7438 channels.push(target.channel);
7439 }
7440 continue;
7441 }
7442 let channel = target.channel;
7443 targets.push((target, vec![channel]));
7444 }
7445 for (target, mut channels) in targets {
7446 channels.sort_unstable();
7447 let mut push = push.clone();
7448 match &mut push {
7449 ClientControlPush::RouteClosing {
7450 channels: covered, ..
7451 }
7452 | ClientControlPush::RouteClosed {
7453 channels: covered, ..
7454 } => *covered = channels,
7455 }
7456 let body = match serde_json::to_vec(&push) {
7457 Ok(body) => body,
7458 Err(err) => {
7459 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7460 continue;
7461 }
7462 };
7463 let frame = match Frame::build_with_version(
7464 target.negotiated_ver,
7465 FrameType::Push,
7466 control_flags(),
7467 0,
7468 0,
7469 0,
7470 body.clone(),
7471 ) {
7472 Ok(frame) => frame,
7473 Err(err) => {
7474 warn!(
7475 route_channel = target.channel,
7476 error = %err,
7477 "failed to build route lifecycle control PUSH frame"
7478 );
7479 continue;
7480 }
7481 };
7482 if let Err(err) = target.sink.try_send(frame) {
7483 if target.close_on_delivery_failure() {
7484 warn!(
7485 target_connection_id = target.connection_id.get(),
7486 route_channel = target.channel,
7487 error = %err,
7488 "route lifecycle control PUSH was not delivered to client; closing target connection"
7489 );
7490 let _ = forwarding.escalate_client_delivery_failure(
7491 target.connection_id,
7492 target.channel,
7493 target.epoch,
7494 CloseReason::new(
7495 "route_lifecycle_push_delivery_failed",
7496 format!(
7497 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7498 target.channel
7499 ),
7500 ),
7501 crate::forwarding::UndeliveredFrame {
7502 module_id: target.module_id.as_deref(),
7503 sink: &target.sink,
7504 },
7505 );
7506 }
7507 }
7508 }
7509}
7510
7511#[cfg(test)]
7512mod tests {
7513 mod activity_read {
7514 include!("activity_read_tests.rs");
7515 }
7516
7517 mod lifecycle_activity {
7518 include!("control_lifecycle_tests.rs");
7519 }
7520 #[cfg(unix)]
7521 #[tokio::test]
7522 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
7523 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
7524 let path = dir.join("subc.jsonc");
7525 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
7526 "program":"/bin/sleep","args":["60"],"protocol":"none",
7527 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
7528 }}}).to_string()).unwrap();
7529 let mut configured = crate::daemon_config::load(&path)
7530 .unwrap()
7531 .unwrap()
7532 .modules
7533 .pop()
7534 .unwrap();
7535 let registry = std::sync::Arc::new(crate::Registry::default());
7536 let handle = crate::SupervisorHandle::new();
7537 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
7538 .with_handle(handle.clone());
7539 let module = supervisor
7540 .supervise_configured_with_health(
7541 configured.module_spec(),
7542 true,
7543 configured.health.clone(),
7544 None,
7545 configured.restart,
7546 )
7547 .unwrap();
7548 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
7549 let before = module.status().unwrap().pid;
7550 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
7551 configured.health.cadence = std::time::Duration::from_secs(3600);
7552 let health_only = handler
7553 .reconcile_supervised_modules(
7554 &supervisor,
7555 vec![configured.clone()],
7556 false,
7557 Principal::Direct,
7558 )
7559 .await
7560 .unwrap();
7561 assert!(
7562 health_only.changed_pending_reload.is_empty(),
7563 "health policy is already applied live"
7564 );
7565 assert_eq!(module.status().unwrap().pid, before);
7566 assert_eq!(
7567 module.configuration().unwrap().1.http,
7568 configured.health.http
7569 );
7570 configured.args = vec!["61".into()];
7571 let launch = handler
7572 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
7573 .await
7574 .unwrap();
7575 assert_eq!(launch.changed_pending_reload, ["stock"]);
7576 assert_eq!(
7577 module.status().unwrap().pid,
7578 before,
7579 "a launch edit is stored until reload"
7580 );
7581 module.drain().await.unwrap();
7582 }
7583 use cortexkit_test_support::ScratchDir;
7584 use std::{
7585 collections::BTreeMap,
7586 fmt,
7587 path::PathBuf,
7588 sync::{Arc, Mutex},
7589 time::Duration,
7590 };
7591
7592 use serde_json::{json, Value};
7593 use subc_protocol::{
7594 manifest::{
7595 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
7596 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
7597 },
7598 session::HealthStatus,
7599 FrameType,
7600 };
7601
7602 use super::*;
7603 use crate::{
7604 forwarding::{DataRoute, DataRouteState},
7605 registry::ChannelState,
7606 router::FrameSink,
7607 stderr_tail::DEFAULT_MAX_LINE_BYTES,
7608 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
7609 RouteCtx, Router,
7610 };
7611 use tokio::{
7612 sync::mpsc,
7613 time::{sleep, Instant},
7614 };
7615 use tracing::{
7616 field::{Field, Visit},
7617 Event, Subscriber,
7618 };
7619 use tracing_subscriber::{layer::Context, prelude::*, Layer};
7620
7621 fn fake_aft_stub_path() -> PathBuf {
7640 let mut path = std::env::current_exe().expect("current_exe available in tests");
7641 path.pop(); path.pop(); path.push(if cfg!(windows) {
7644 "fake-aft-stub.exe"
7645 } else {
7646 "fake-aft-stub"
7647 });
7648 assert!(
7649 path.exists(),
7650 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
7651 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
7652 path.display()
7653 );
7654 path
7655 }
7656
7657 fn client_retries(code: &str) -> bool {
7662 subc_protocol::error_codes::is_retryable_route_open(code)
7663 }
7664
7665 #[test]
7676 fn retryability_of_forwarding_codes_matches_the_failure() {
7677 let transient = [
7680 ForwardingError::NoModuleConnection,
7681 ForwardingError::ModuleReloading {
7682 module_id: "m".into(),
7683 },
7684 ForwardingError::StaleModuleEndpoint,
7685 ForwardingError::UnknownReservation {
7686 client_channel: 1,
7687 module_channel: 1,
7688 },
7689 ForwardingError::ConnectionClosing {
7690 connection_id: ConnectionId::new(1),
7691 },
7692 ForwardingError::ClientEgressClosed {
7693 connection_id: ConnectionId::new(1),
7694 },
7695 ForwardingError::ModuleEgressUnavailable {
7696 connection_id: ConnectionId::new(1),
7697 },
7698 ];
7699 for err in transient {
7700 let code = forwarding_error_code(&err);
7701 assert!(
7702 client_retries(code),
7703 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
7704 );
7705 }
7706
7707 let permanent = [
7712 ForwardingError::ConnectionRoleConflict {
7713 connection_id: ConnectionId::new(1),
7714 },
7715 ForwardingError::ClientRouteChannelExhausted {
7716 connection_id: ConnectionId::new(1),
7717 },
7718 ForwardingError::ModuleRouteChannelExhausted {
7719 endpoint: ModuleEndpointId {
7720 connection_id: ConnectionId::new(1),
7721 generation: 1,
7722 },
7723 },
7724 ForwardingError::RelayCorrelationExhausted,
7725 ForwardingError::RouteOpenBuild("x".into()),
7726 ForwardingError::Poisoned,
7727 ];
7728 for err in permanent {
7729 let code = forwarding_error_code(&err);
7730 assert!(
7731 !client_retries(code),
7732 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
7733 );
7734 }
7735 }
7736
7737 #[tokio::test]
7765 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
7766 let handler = ControlHandler::default();
7767 let frame =
7768 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
7769
7770 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
7774 assert_eq!(
7775 stamped,
7776 Principal::Direct,
7777 "a caller that proved nothing must not be stamped as a supervised module"
7778 );
7779
7780 let forged = handler
7785 .route_open_principal(
7786 &frame,
7787 Some(ConsumerIdentity {
7788 module_id: "aft".to_string(),
7789 launch_nonce: "not-a-real-nonce".to_string(),
7790 }),
7791 )
7792 .unwrap();
7793 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
7794 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
7795 }
7796
7797 #[test]
7814 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
7815 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
7816 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
7817 let ClientControlRequest::RouteOpen {
7818 consumer_identity, ..
7819 } = parsed
7820 else {
7821 panic!("route.open body must parse as RouteOpen");
7822 };
7823 assert_eq!(
7824 consumer_identity,
7825 Some(ConsumerIdentity {
7826 module_id: "aft".to_string(),
7827 launch_nonce: "n".to_string(),
7828 }),
7829 "the wire field name must reach the value route_open_principal reads"
7830 );
7831 }
7832
7833 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
7834 ModuleManifest::builder(module_id, "0.1.0")
7835 .protocol_ver(protocol_ver)
7836 .provides(vec![ProviderRole::ToolProvider {
7837 tools: vec![Tool {
7838 name: "read".to_string(),
7839 description: None,
7840 execution_mode: ExecutionMode::Pure,
7841 schema: json!({"type": "object"}),
7842 }],
7843 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
7844 concurrency: Concurrency::ModuleManaged,
7845 emits_push: true,
7846 sub_supervises: true,
7847 }])
7848 .build()
7849 }
7850
7851 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
7852 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
7853 }
7854
7855 fn hello_frame_with_control_ops(
7856 module_id: &str,
7857 protocol_ver: u8,
7858 corr: u64,
7859 control_ops: Option<Vec<String>>,
7860 ) -> Frame {
7861 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
7862 }
7863
7864 fn hello_frame_with_nonce(
7865 module_id: &str,
7866 protocol_ver: u8,
7867 corr: u64,
7868 launch_nonce: Option<&str>,
7869 ) -> Frame {
7870 hello_frame_full(
7871 module_id,
7872 protocol_ver,
7873 corr,
7874 None,
7875 launch_nonce.map(ToOwned::to_owned),
7876 )
7877 }
7878
7879 fn hello_frame_full(
7880 module_id: &str,
7881 protocol_ver: u8,
7882 corr: u64,
7883 control_ops: Option<Vec<String>>,
7884 launch_nonce: Option<String>,
7885 ) -> Frame {
7886 let body = serde_json::to_vec(&ModuleHelloBody {
7887 manifest: manifest(module_id, protocol_ver),
7888 protocol_ver,
7889 control_ops,
7890 launch_nonce,
7891 })
7892 .unwrap();
7893 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7894 }
7895
7896 fn non_routable_hello_frame_with_control_ops(
7897 module_id: &str,
7898 corr: u64,
7899 control_ops: Option<Vec<String>>,
7900 ) -> Frame {
7901 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
7902 manifest.provides.clear();
7903 let body = serde_json::to_vec(&ModuleHelloBody {
7904 manifest,
7905 protocol_ver: PROTOCOL_VERSION,
7906 control_ops,
7907 launch_nonce: None,
7908 })
7909 .unwrap();
7910 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7911 }
7912
7913 fn capability_grammar_hello_frame(
7914 capabilities: Value,
7915 runtime_computed: Option<Value>,
7916 corr: u64,
7917 ) -> Frame {
7918 let mut body = serde_json::to_value(ModuleHelloBody {
7919 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
7920 protocol_ver: PROTOCOL_VERSION,
7921 control_ops: None,
7922 launch_nonce: None,
7923 })
7924 .expect("HELLO body serializes");
7925 body["manifest"]["capabilities"] = capabilities;
7926 if let Some(runtime_computed) = runtime_computed {
7927 body["runtime_computed"] = runtime_computed;
7928 }
7929 Frame::build(
7930 FrameType::Hello,
7931 control_flags(),
7932 0,
7933 0,
7934 corr,
7935 serde_json::to_vec(&body).expect("HELLO body reserializes"),
7936 )
7937 .expect("HELLO frame builds")
7938 }
7939
7940 fn channel_request(channel: u16, corr: u64) -> Frame {
7941 Frame::build(
7942 FrameType::Request,
7943 Flags::new(true, Priority::Interactive, false),
7944 channel,
7945 0,
7946 corr,
7947 b"opaque".to_vec(),
7948 )
7949 .unwrap()
7950 }
7951
7952 fn route_ctx(
7953 connection_id: ConnectionId,
7954 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
7955 let (tx, rx) = mpsc::channel(8);
7956 (
7957 RouteCtx {
7958 connection_id,
7959 egress: FrameSink::new(tx),
7960 },
7961 rx,
7962 )
7963 }
7964
7965 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
7966 serde_json::from_slice(&frame.body).unwrap()
7967 }
7968
7969 async fn hello_via_sink(
7974 handler: &ControlHandler,
7975 ctx: &RouteCtx,
7976 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7977 hello: Frame,
7978 ) -> Frame {
7979 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
7980 assert!(
7981 replies.is_empty(),
7982 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
7983 );
7984 let ack = rx
7985 .try_recv()
7986 .expect("HELLO_ACK is queued on the module sink")
7987 .frame;
7988 assert_eq!(ack.header.ty, FrameType::HelloAck);
7989 ack
7990 }
7991
7992 fn parse_error(frame: &Frame) -> Value {
7993 serde_json::from_slice(&frame.body).unwrap()
7994 }
7995
7996 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
7997 serde_json::from_slice(&frame.body).unwrap()
7998 }
7999
8000 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8001 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8002 route_channel,
8003 route_epoch: 0,
8004 kind,
8005 })
8006 .unwrap();
8007 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8008 }
8009
8010 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8011 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8012 module_id: module_id.to_string(),
8013 })
8014 .unwrap();
8015 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8016 }
8017
8018 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8019 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8020 }
8021
8022 fn route_open_frame_with_consumer_capabilities(
8023 corr: u64,
8024 module_id: &str,
8025 project_root: ScratchDir,
8026 consumer_capabilities: Option<Vec<String>>,
8027 ) -> Frame {
8028 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8029 target: RouteTarget::ToolProvider {
8030 module_id: module_id.to_string(),
8031 },
8032 identity: BindIdentity::new(
8033 project_root.path().to_path_buf(),
8034 "unit".to_string(),
8035 "session".to_string(),
8036 ),
8037 consumer_identity: None,
8038 consumer_capabilities,
8039 role_versions: None,
8040 admission_facts: None,
8041 scope: None,
8042 })
8043 .unwrap();
8044 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8045 }
8046
8047 fn route_open_frame_with_role_versions(
8048 corr: u64,
8049 module_id: &str,
8050 project_root: ScratchDir,
8051 role_versions: Option<BTreeMap<String, String>>,
8052 ) -> Frame {
8053 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8054 target: RouteTarget::ToolProvider {
8055 module_id: module_id.to_string(),
8056 },
8057 identity: BindIdentity::new(
8058 project_root.path().to_path_buf(),
8059 "unit".to_string(),
8060 format!("session-{corr}"),
8061 ),
8062 consumer_identity: None,
8063 consumer_capabilities: None,
8064 role_versions,
8065 admission_facts: None,
8066 scope: None,
8067 })
8068 .unwrap();
8069 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8070 }
8071
8072 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8073 entries
8074 .iter()
8075 .map(|(role, version)| (role.to_string(), version.to_string()))
8076 .collect()
8077 }
8078
8079 fn route_open_frame_with_admission_facts(
8080 corr: u64,
8081 module_id: &str,
8082 project_root: ScratchDir,
8083 consumer_identity: Option<subc_control::ConsumerIdentity>,
8084 facts: Option<Value>,
8085 ) -> Frame {
8086 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8087 target: RouteTarget::ToolProvider {
8088 module_id: module_id.to_string(),
8089 },
8090 identity: BindIdentity::new(
8091 project_root.path().to_path_buf(),
8092 "unit".to_string(),
8093 format!("session-{corr}"),
8094 ),
8095 consumer_identity,
8096 consumer_capabilities: None,
8097 role_versions: None,
8098 admission_facts: facts,
8099 scope: None,
8100 })
8101 .unwrap();
8102 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8103 }
8104
8105 #[derive(Clone, Default)]
8106 struct EventCapture {
8107 events: Arc<Mutex<Vec<CapturedEvent>>>,
8108 }
8109
8110 #[derive(Clone, Debug)]
8111 struct CapturedEvent {
8112 target: String,
8113 level: tracing::Level,
8114 fields: BTreeMap<String, String>,
8115 }
8116
8117 impl EventCapture {
8118 fn events(&self) -> Vec<CapturedEvent> {
8119 self.events.lock().unwrap().clone()
8120 }
8121 }
8122
8123 impl<S> Layer<S> for EventCapture
8124 where
8125 S: Subscriber,
8126 {
8127 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8128 let mut visitor = EventFieldVisitor::default();
8129 event.record(&mut visitor);
8130 self.events.lock().unwrap().push(CapturedEvent {
8131 target: event.metadata().target().to_string(),
8132 level: *event.metadata().level(),
8133 fields: visitor.fields,
8134 });
8135 }
8136 }
8137
8138 #[derive(Default)]
8139 struct EventFieldVisitor {
8140 fields: BTreeMap<String, String>,
8141 }
8142
8143 impl Visit for EventFieldVisitor {
8144 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8145 self.fields
8146 .insert(field.name().to_string(), format!("{value:?}"));
8147 }
8148 }
8149
8150 fn health_response(corr: u64, status: HealthStatus) -> Frame {
8151 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8152 status,
8153 detail: Some("warming".to_string()),
8154 metrics: Some(json!({"queue_depth": 3})),
8155 })
8156 .unwrap();
8157 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8158 }
8159
8160 fn route_bind_ack(corr: u64) -> Frame {
8161 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8162 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8163 }
8164
8165 fn unique_project_root(label: &str) -> ScratchDir {
8166 ScratchDir::new(label)
8167 }
8168
8169 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8170 match parse_route_poll(frame) {
8171 ClientControlResponse::RoutePoll {
8172 status: None,
8173 live: Some(live),
8174 ..
8175 } => assert_eq!(live, expected_live),
8176 other => panic!("unexpected route.poll response: {other:?}"),
8177 }
8178 }
8179
8180 fn bind_liveness_route(
8181 registry: &Registry,
8182 forwarding: &ForwardingTable,
8183 module_id: &str,
8184 ) -> (RouteCtx, u16, u32) {
8185 let module_connection = ConnectionId::new(101);
8186 let client_connection = ConnectionId::new(202);
8187 let registration = registry
8188 .register_with_control_ops(
8189 manifest(module_id, PROTOCOL_VERSION),
8190 PROTOCOL_VERSION,
8191 module_connection,
8192 module_baseline_control_ops(),
8193 )
8194 .unwrap();
8195 let (module_tx, _module_rx) = mpsc::channel(8);
8196 let endpoint = forwarding
8197 .register_module_connection(
8198 module_connection,
8199 module_id.to_string(),
8200 PROTOCOL_VERSION,
8201 manifest_concurrency(®istration.manifest),
8202 FrameSink::new(module_tx),
8203 )
8204 .unwrap();
8205 let (client_ctx, _client_rx) = route_ctx(client_connection);
8206 let pending = forwarding
8207 .begin_route_bind_relay_for_test(
8208 client_connection,
8209 client_ctx.egress.clone(),
8210 1,
8211 module_id,
8212 )
8213 .unwrap();
8214 assert_eq!(pending.endpoint, endpoint);
8215 let route_channel = pending.client_channel;
8216 let route_epoch = pending.client_epoch;
8217 forwarding
8218 .complete_pending_relay(
8219 module_connection,
8220 pending.corr,
8221 RouteBindRelayOutcome::Accepted,
8222 )
8223 .unwrap();
8224 (client_ctx, route_channel, route_epoch)
8225 }
8226
8227 struct FakeProcessLiveness {
8228 live: Option<bool>,
8229 }
8230
8231 impl ModuleProcessLiveness for FakeProcessLiveness {
8232 fn process_live(&self, _module_id: &str) -> Option<bool> {
8233 self.live
8234 }
8235 }
8236
8237 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8238 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8239 {
8240 let registry = Arc::new(Registry::default());
8241 let supervisor_handle = SupervisorHandle::new();
8242 let supervisor = Supervisor::new_for_test(
8243 Arc::clone(®istry),
8244 RestartPolicy::new(1, Duration::from_millis(10)),
8245 )
8246 .with_handle(supervisor_handle.clone());
8247 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8248 let module = supervisor
8249 .spawn(ModuleSpec {
8250 module_id: "stderr-tail-wire".to_string(),
8251 program: fake_aft_stub_path(),
8252 args: Vec::new(),
8253 env: vec![
8254 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8255 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8256 ],
8257 reserved: false,
8258 reserved_prefixes: Vec::new(),
8259 protocol: ModuleProtocol::Subc,
8260 overlap: Default::default(),
8261 })
8262 .unwrap();
8263
8264 let deadline = Instant::now() + Duration::from_secs(5);
8265 loop {
8266 let tail = module.stderr_tail(None, None);
8267 if tail
8268 .entries
8269 .iter()
8270 .any(|entry| matches!(entry, TailEntry::ProcessStart))
8271 && tail.entries.iter().any(|entry| {
8272 matches!(
8273 entry,
8274 TailEntry::Line {
8275 truncated: true,
8276 ..
8277 }
8278 )
8279 })
8280 {
8281 break;
8282 }
8283 assert!(
8284 Instant::now() < deadline,
8285 "module did not produce a truncated line and restart boundary: {tail:?}"
8286 );
8287 sleep(Duration::from_millis(10)).await;
8288 }
8289
8290 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8291 let request = ClientControlRequest::SupervisorStderrTail {
8292 module_id: "stderr-tail-wire".to_string(),
8293 max_lines: None,
8294 max_bytes: None,
8295 };
8296 let frame = Frame::build(
8297 FrameType::Request,
8298 control_flags(),
8299 0,
8300 0,
8301 1,
8302 serde_json::to_vec(&request).unwrap(),
8303 )
8304 .unwrap();
8305 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8306 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8307 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8308 serde_json::from_slice(&responses[0].body).unwrap()
8309 else {
8310 panic!("expected supervisor.stderr_tail response");
8311 };
8312
8313 assert!(
8314 tail.entries
8315 .iter()
8316 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8317 "the control response lost the restart boundary"
8318 );
8319 let Some(StderrTailEntry::Line {
8320 text,
8321 truncated,
8322 at_ms,
8323 }) = tail.entries.iter().find(|entry| {
8324 matches!(
8325 entry,
8326 StderrTailEntry::Line {
8327 truncated: true,
8328 ..
8329 }
8330 )
8331 })
8332 else {
8333 panic!("the control response lost the truncated line");
8334 };
8335 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8336 assert!(*truncated);
8337 assert!(
8338 at_ms.is_some(),
8339 "the control response lost the line's capture time"
8340 );
8341 }
8342
8343 #[tokio::test(flavor = "current_thread")]
8348 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8349 let dir = ScratchDir::new("terminals-off-worker");
8350 let journal_path = dir.join("terminals.jsonl");
8351 let registry = Arc::new(Registry::default());
8352 let supervisor_handle = SupervisorHandle::new();
8353 let supervisor =
8354 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8355 .with_handle(supervisor_handle.clone())
8356 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8357 let module = supervisor
8358 .spawn(ModuleSpec {
8359 module_id: "terminal-off-worker".to_string(),
8360 program: fake_aft_stub_path(),
8361 args: Vec::new(),
8362 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8363 reserved: false,
8364 reserved_prefixes: Vec::new(),
8365 protocol: ModuleProtocol::Subc,
8366 overlap: Default::default(),
8367 })
8368 .unwrap();
8369 let deadline = Instant::now() + Duration::from_secs(5);
8370 while module.terminal_history().entries.len() != 2 {
8371 assert!(Instant::now() < deadline, "module did not record two exits");
8372 sleep(Duration::from_millis(10)).await;
8373 }
8374
8375 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8376 let handler =
8377 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
8378 let frame = Frame::build(
8379 FrameType::Request,
8380 control_flags(),
8381 0,
8382 0,
8383 1,
8384 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8385 module_id: "terminal-off-worker".to_string(),
8386 })
8387 .unwrap(),
8388 )
8389 .unwrap();
8390 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8391 let spawned_at = std::time::Instant::now();
8392 let read = tokio::spawn({
8393 let handler = Arc::clone(&handler);
8394 async move { handler.handle_control_frame(&ctx, frame).await }
8395 });
8396 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8399 .await
8400 .unwrap()
8401 .expect("the history read reached its pause");
8402 let elapsed = spawned_at.elapsed();
8403 assert!(
8404 elapsed < Duration::from_secs(2) && !read.is_finished(),
8405 "this task could not run while the history read was paused \
8406 (resumed after {elapsed:?}, read finished: {})",
8407 read.is_finished()
8408 );
8409
8410 drop(release);
8411 let responses = read.await.unwrap().unwrap();
8412 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8413 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8414 panic!("expected supervisor.terminals response");
8415 };
8416 assert_eq!(terminals.entries.len(), 2);
8417 assert_eq!(terminals.journal_skipped_lines, 0);
8418 assert_eq!(terminals.journal_read_errors, 0);
8419 }
8420
8421 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8422 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8423 let registry = Arc::new(Registry::default());
8424 let supervisor_handle = SupervisorHandle::new();
8425 let supervisor =
8426 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8427 .with_handle(supervisor_handle.clone());
8428 let module = supervisor
8429 .spawn(ModuleSpec {
8430 module_id: "terminal-golden".to_string(),
8431 program: fake_aft_stub_path(),
8432 args: Vec::new(),
8433 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8434 reserved: false,
8435 reserved_prefixes: Vec::new(),
8436 protocol: ModuleProtocol::Subc,
8437 overlap: Default::default(),
8438 })
8439 .unwrap();
8440
8441 let deadline = Instant::now() + Duration::from_secs(5);
8442 while module.terminal_history().entries.len() != 2 {
8443 assert!(
8444 Instant::now() < deadline,
8445 "module did not retain two terminal exits: {:?}",
8446 module.terminal_history()
8447 );
8448 sleep(Duration::from_millis(10)).await;
8449 }
8450
8451 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8452 let request = ClientControlRequest::SupervisorTerminals {
8453 module_id: "terminal-golden".to_string(),
8454 };
8455 let frame = Frame::build(
8456 FrameType::Request,
8457 control_flags(),
8458 0,
8459 0,
8460 1,
8461 serde_json::to_vec(&request).unwrap(),
8462 )
8463 .unwrap();
8464 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8465 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8466 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8467 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8468 panic!("expected supervisor.terminals response");
8469 };
8470 assert_eq!(terminals.entries.len(), 2);
8471 assert_eq!(terminals.dropped, 0);
8472
8473 let mut rendered = serde_json::to_value(response).unwrap();
8474 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
8477 for (index, entry) in rendered["entries"]
8478 .as_array_mut()
8479 .expect("terminal response entries array")
8480 .iter_mut()
8481 .enumerate()
8482 {
8483 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
8484 }
8485
8486 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8487 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
8488 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
8489 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8490 std::fs::write(&golden_path, &serialized).unwrap();
8491 }
8492 let expected: Value =
8493 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
8494 assert_eq!(rendered, expected);
8495 }
8496
8497 #[test]
8498 fn hello_registers_manifest_and_returns_ack() {
8499 let registry = Arc::new(Registry::default());
8500 let handler = ControlHandler::new(Arc::clone(®istry));
8501 let conn = ConnectionId::new(1);
8502
8503 let responses = handler
8504 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
8505 .unwrap();
8506
8507 assert_eq!(responses.len(), 1);
8508 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8509 assert_eq!(responses[0].header.channel, 0);
8510 assert_eq!(responses[0].header.corr, 7);
8511 let ack = parse_ack(&responses[0]);
8512 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8513 assert!(ack
8514 .subc_capabilities
8515 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
8516 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
8517 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
8518 assert!(ack
8519 .subc_ops
8520 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
8521 assert!(ack
8522 .subc_ops
8523 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
8524
8525 let registration = registry.get_module("aft").unwrap().unwrap();
8526 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
8527 assert_eq!(registration.state, ChannelState::Active);
8528 assert_eq!(registration.connection_id, conn);
8529 assert_eq!(registration.control_ops, module_baseline_control_ops());
8530 }
8531
8532 #[test]
8533 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
8534 let invalid_identifiers = [
8535 ("case_change", "credentials-Provider/v1"),
8536 ("leading_zero", "credentials-provider/v01"),
8537 ("trailing_hyphen", "credentials-provider-/v1"),
8538 ("consecutive_hyphens", "credentials--provider/v1"),
8539 ("uppercase", "Credentials-provider/v1"),
8540 ("missing_v", "credentials-provider/1"),
8541 ("whitespace", "credentials provider/v1"),
8542 ("zero_version", "credentials-provider/v0"),
8543 ("out_of_range_version", "credentials-provider/v4294967296"),
8544 (
8545 "overlength_name",
8546 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
8547 ),
8548 ];
8549 let mut cases = invalid_identifiers
8550 .into_iter()
8551 .map(|(name, identifier)| {
8552 (
8553 format!("identifier_{name}"),
8554 "capabilities.provides[0]".to_string(),
8555 identifier.to_string(),
8556 json!({ "provides": [identifier] }),
8557 None,
8558 )
8559 })
8560 .collect::<Vec<_>>();
8561 cases.extend([
8562 (
8563 "unknown_need".to_string(),
8564 "capabilities.requires[0].need".to_string(),
8565 "deferred".to_string(),
8566 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
8567 None,
8568 ),
8569 (
8570 "duplicate_provides".to_string(),
8571 "capabilities.provides[1]".to_string(),
8572 "credentials-provider/v1".to_string(),
8573 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
8574 None,
8575 ),
8576 (
8577 "duplicate_must_never_reach".to_string(),
8578 "capabilities.must_never_reach[1]".to_string(),
8579 "credentials-provider/v1".to_string(),
8580 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
8581 None,
8582 ),
8583 (
8584 "duplicate_requires_same_need".to_string(),
8585 "capabilities.requires[1]".to_string(),
8586 "credentials-provider/v1".to_string(),
8587 json!({ "requires": [
8588 { "capability": "credentials-provider/v1", "need": "required" },
8589 { "capability": "credentials-provider/v1", "need": "required" }
8590 ] }),
8591 None,
8592 ),
8593 (
8594 "duplicate_requires_conflicting_need".to_string(),
8595 "capabilities.requires[1]".to_string(),
8596 "credentials-provider/v1".to_string(),
8597 json!({ "requires": [
8598 { "capability": "credentials-provider/v1", "need": "required" },
8599 { "capability": "credentials-provider/v1", "need": "optional" }
8600 ] }),
8601 None,
8602 ),
8603 (
8604 "capabilities_root_pointer".to_string(),
8605 "runtime_computed[0]".to_string(),
8606 "/capabilities".to_string(),
8607 json!({}),
8608 Some(json!(["/capabilities"])),
8609 ),
8610 (
8611 "capabilities_descendant_pointer".to_string(),
8612 "runtime_computed[0]".to_string(),
8613 "/capabilities/provides".to_string(),
8614 json!({}),
8615 Some(json!(["/capabilities/provides"])),
8616 ),
8617 (
8618 "malformed_pointer_without_leading_slash".to_string(),
8619 "runtime_computed[0]".to_string(),
8620 "capabilities".to_string(),
8621 json!({}),
8622 Some(json!(["capabilities"])),
8623 ),
8624 (
8625 "malformed_pointer_escape".to_string(),
8626 "runtime_computed[0]".to_string(),
8627 "/roles/~2/tools".to_string(),
8628 json!({}),
8629 Some(json!(["/roles/~2/tools"])),
8630 ),
8631 (
8632 "unknown_capabilities_field".to_string(),
8633 "capabilities.future".to_string(),
8634 "<array>".to_string(),
8635 json!({ "future": [] }),
8636 None,
8637 ),
8638 ]);
8639
8640 for (index, (name, field, value, capabilities, runtime_computed)) in
8641 cases.into_iter().enumerate()
8642 {
8643 let registry = Arc::new(Registry::default());
8644 let handler = ControlHandler::new(Arc::clone(®istry));
8645 let response = handler
8646 .handle_control(
8647 ConnectionId::new((index + 1) as u64),
8648 capability_grammar_hello_frame(
8649 capabilities,
8650 runtime_computed,
8651 index as u64 + 1,
8652 ),
8653 )
8654 .expect("invalid HELLO returns a refusal");
8655
8656 assert_eq!(response.len(), 1, "{name} must emit one refusal");
8657 let error = parse_error(&response[0]);
8658 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
8659 let message = error["message"]
8660 .as_str()
8661 .expect("error message is a string");
8662 assert!(
8663 message.contains(&field),
8664 "{name}: field missing from {message}"
8665 );
8666 assert!(
8667 message.contains(&value),
8668 "{name}: value missing from {message}"
8669 );
8670 assert_eq!(
8671 registry
8672 .active_registration_count()
8673 .expect("registry reads"),
8674 0,
8675 "{name}: refused HELLO must not create a catalog entry"
8676 );
8677 }
8678 }
8679
8680 #[test]
8681 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
8682 let registry = Arc::new(Registry::default());
8683 let handler = ControlHandler::new(Arc::clone(®istry));
8684 let capabilities = json!({
8685 "provides": ["credentials-provider/v1"],
8686 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
8687 "must_never_reach": ["federation-transport/v1"]
8688 });
8689 let response = handler
8690 .handle_control(
8691 ConnectionId::new(99),
8692 capability_grammar_hello_frame(
8693 capabilities.clone(),
8694 Some(json!(["/roles/0/tools"])),
8695 99,
8696 ),
8697 )
8698 .expect("valid HELLO registers");
8699 assert_eq!(response[0].header.ty, FrameType::HelloAck);
8700
8701 let request = Frame::build(
8702 FrameType::Request,
8703 control_flags(),
8704 0,
8705 0,
8706 100,
8707 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8708 .expect("catalog request serializes"),
8709 )
8710 .expect("catalog request frame builds");
8711 let response = handler
8712 .handle_catalog_list(request, None)
8713 .expect("catalog list succeeds");
8714 let ClientControlResponse::CatalogList { modules, .. } =
8715 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
8716 else {
8717 panic!("catalog request must return catalog.list");
8718 };
8719 assert_eq!(modules.len(), 1);
8720 assert_eq!(
8721 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
8722 capabilities
8723 );
8724 }
8725
8726 #[test]
8727 fn catalog_list_mirrors_management_operation_description() {
8728 let registry = Arc::new(Registry::default());
8729 let handler = ControlHandler::new(Arc::clone(®istry));
8730 let description = "List managed records and return their identifiers and metadata.";
8731 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
8732 manifest.provides = vec![ProviderRole::ManagementSurface {
8733 operations: vec![ManagementOperation {
8734 name: "records.list".to_string(),
8735 kind: ManagementOperationKind::Query,
8736 description: Some(description.to_string()),
8737 }],
8738 config_schema: json!({"type": "object"}),
8739 observability: vec![ObservabilitySurface {
8740 name: "records.stats".to_string(),
8741 kind: ObservabilityKind::Snapshot,
8742 }],
8743 identity_scope: vec![IdentityScope::Project],
8744 concurrency: Concurrency::ModuleManaged,
8745 }];
8746 registry
8747 .register_with_control_ops(
8748 manifest,
8749 PROTOCOL_VERSION,
8750 ConnectionId::new(99),
8751 Vec::new(),
8752 )
8753 .expect("described management manifest registers");
8754
8755 let request = Frame::build(
8756 FrameType::Request,
8757 control_flags(),
8758 0,
8759 0,
8760 100,
8761 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8762 .expect("catalog request serializes"),
8763 )
8764 .expect("catalog request frame builds");
8765 let response = handler
8766 .handle_catalog_list(request, None)
8767 .expect("catalog list succeeds");
8768 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8769 assert_eq!(
8770 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
8771 "catalog.list must preserve the declared operation description verbatim"
8772 );
8773 }
8774
8775 #[test]
8776 fn catalog_list_mirrors_registered_operation_delivery() {
8777 let registry = Arc::new(Registry::default());
8778 let handler = ControlHandler::new(Arc::clone(®istry));
8779 let delivery =
8780 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
8781 let mut body = serde_json::to_value(ModuleHelloBody {
8782 manifest: manifest("watch-management", PROTOCOL_VERSION),
8783 protocol_ver: PROTOCOL_VERSION,
8784 control_ops: None,
8785 launch_nonce: None,
8786 })
8787 .unwrap();
8788 body["manifest"]["provides"] = json!([{
8789 "role": "management_surface",
8790 "operations": [
8791 {"name": "status", "kind": "query"},
8792 {"name": "future", "kind": "query"},
8793 {"name": "reset", "kind": "mutate"}
8794 ],
8795 "config_schema": {}, "observability": [], "identity_scope": []
8796 }]);
8797 body["manifest"]["operation_delivery"] = delivery.clone();
8798 let hello = Frame::build(
8799 FrameType::Hello,
8800 control_flags(),
8801 0,
8802 0,
8803 99,
8804 serde_json::to_vec(&body).unwrap(),
8805 )
8806 .unwrap();
8807 let response = handler
8808 .handle_control(ConnectionId::new(99), hello)
8809 .unwrap();
8810 assert_eq!(response[0].header.ty, FrameType::HelloAck);
8811
8812 let (_, registrations) = registry.list_modules().unwrap();
8813 let registered = ®istrations[0].manifest;
8814 assert!(registered.is_watch_operation("status"));
8815 assert!(!registered.is_watch_operation("future"));
8816 assert!(!registered.is_watch_operation("reset"));
8817 assert!(!registered.is_watch_operation("missing"));
8818
8819 let request = Frame::build(
8820 FrameType::Request,
8821 control_flags(),
8822 0,
8823 0,
8824 100,
8825 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
8826 )
8827 .unwrap();
8828 let response = handler.handle_catalog_list(request, None).unwrap();
8829 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
8830 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
8831 let ClientControlResponse::CatalogList { modules, .. } =
8832 serde_json::from_slice(&response[0].body).unwrap()
8833 else {
8834 panic!("expected catalog.list");
8835 };
8836 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
8837 }
8838
8839 #[test]
8840 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
8841 let registry = Arc::new(Registry::default());
8842 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8843 [("vault".to_string(), true), ("squatter".to_string(), true)],
8844 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8845 );
8846 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
8847 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8848 provides: vec!["credentials-provider/v1".to_string()],
8849 requires: Vec::new(),
8850 must_never_reach: Vec::new(),
8851 });
8852 let frame = Frame::build(
8853 FrameType::Hello,
8854 control_flags(),
8855 0,
8856 0,
8857 77,
8858 serde_json::to_vec(&ModuleHelloBody {
8859 manifest: squatter,
8860 protocol_ver: PROTOCOL_VERSION,
8861 control_ops: None,
8862 launch_nonce: None,
8863 })
8864 .expect("HELLO serializes"),
8865 )
8866 .expect("HELLO frame builds");
8867 let response = handler
8868 .handle_control(ConnectionId::new(77), frame)
8869 .expect("reserved claim receives a typed refusal");
8870 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8871 assert_eq!(
8872 registry
8873 .active_registration_count()
8874 .expect("registry reads"),
8875 0,
8876 "a reserved capability refusal must not leave a catalog entry"
8877 );
8878 }
8879
8880 #[test]
8881 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
8882 for settlement in ["timeout", "inconclusive", "drop"] {
8883 let breakers = RouteBindBreakers::default();
8884 let RouteBindAdmission::Admitted {
8885 guard: mut old,
8886 probe: false,
8887 } = breakers.admit("prov")
8888 else {
8889 panic!("ordinary relay admitted")
8890 };
8891 let RouteBindAdmission::Admitted {
8892 guard: mut opener, ..
8893 } = breakers.admit("prov")
8894 else {
8895 panic!("second relay admitted")
8896 };
8897 assert!(
8898 !opener
8899 .record_timeout(1, Duration::ZERO)
8900 .unwrap()
8901 .reopened_after_probe
8902 );
8903 let RouteBindAdmission::Admitted {
8904 guard: mut probe,
8905 probe: true,
8906 } = breakers.admit("prov")
8907 else {
8908 panic!("one cooldown probe admitted")
8909 };
8910 match settlement {
8911 "timeout" => assert!(
8912 !old.record_timeout(1, Duration::ZERO)
8913 .unwrap()
8914 .reopened_after_probe
8915 ),
8916 "inconclusive" => old.record_inconclusive(),
8917 "drop" => drop(old),
8918 _ => unreachable!(),
8919 }
8920 assert!(
8921 matches!(
8922 breakers.admit("prov"),
8923 RouteBindAdmission::Refused {
8924 probe_in_flight: true,
8925 ..
8926 }
8927 ),
8928 "{settlement} of a pre-open relay cannot release the real probe"
8929 );
8930 assert!(
8931 probe
8932 .record_timeout(1, Duration::ZERO)
8933 .unwrap()
8934 .reopened_after_probe
8935 );
8936 assert!(matches!(
8937 breakers.admit("prov"),
8938 RouteBindAdmission::Admitted { probe: true, .. }
8939 ));
8940 }
8941 let breakers = RouteBindBreakers::default();
8942 let admit = || match breakers.admit("prov") {
8943 RouteBindAdmission::Admitted { guard, .. } => guard,
8944 _ => panic!("relay admitted"),
8945 };
8946 admit().record_timeout(1, Duration::ZERO);
8947 let mut old_probe = admit();
8948 breakers.reset_for_new_module_connection("prov");
8949 admit().record_timeout(1, Duration::ZERO);
8950 let _new_probe = admit();
8951 old_probe.record_inconclusive();
8952 assert!(matches!(
8953 breakers.admit("prov"),
8954 RouteBindAdmission::Refused {
8955 probe_in_flight: true,
8956 ..
8957 }
8958 ));
8959 }
8960
8961 #[tokio::test]
8962 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
8963 for candidate in [false, true] {
8964 let registry = Arc::new(Registry::default());
8965 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8966 [("vault".to_string(), true), ("squatter".to_string(), true)],
8967 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8968 );
8969 let conn = ConnectionId::new(77);
8970 let (ctx, mut rx) = route_ctx(conn);
8971 let initial = capability_manifest("squatter", &[], &[]);
8972 if candidate {
8973 registry
8974 .register_candidate_with_control_ops(
8975 initial.clone(),
8976 PROTOCOL_VERSION,
8977 conn,
8978 module_baseline_control_ops(),
8979 )
8980 .unwrap();
8981 handler
8982 .forwarding
8983 .register_candidate_module_connection(
8984 conn,
8985 "squatter".to_string(),
8986 PROTOCOL_VERSION,
8987 manifest_concurrency(&initial),
8988 ctx.egress.clone(),
8989 )
8990 .unwrap();
8991 } else {
8992 hello_via_sink(
8993 &handler,
8994 &ctx,
8995 &mut rx,
8996 hello_frame_with_manifest(initial.clone(), 1),
8997 )
8998 .await;
8999 }
9000 let response = handler
9001 .handle_control_frame(
9002 &ctx,
9003 catalog_update_with_capabilities_frame(
9004 2,
9005 capability_manifest("squatter", &["credentials-provider/v1"], &[])
9006 .capabilities
9007 .unwrap(),
9008 ),
9009 )
9010 .await
9011 .unwrap();
9012 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9013 assert_eq!(
9014 registry
9015 .get_module_by_connection(conn)
9016 .unwrap()
9017 .unwrap()
9018 .manifest,
9019 initial
9020 );
9021 }
9022 }
9023
9024 #[test]
9025 fn server_describe_surfaces_required_capability_verdict_fields() {
9026 let registry = Arc::new(Registry::default());
9027 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9028 [
9029 ("consumer".to_string(), true),
9030 ("provider".to_string(), false),
9031 ],
9032 BTreeMap::new(),
9033 );
9034 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9035 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9036 provides: Vec::new(),
9037 requires: vec![subc_protocol::manifest::CapabilityRequirement {
9038 capability: "credentials-provider/v1".to_string(),
9039 need: subc_protocol::manifest::CapabilityNeed::Required,
9040 }],
9041 must_never_reach: Vec::new(),
9042 });
9043 let hello = Frame::build(
9044 FrameType::Hello,
9045 control_flags(),
9046 0,
9047 0,
9048 78,
9049 serde_json::to_vec(&ModuleHelloBody {
9050 manifest: consumer,
9051 protocol_ver: PROTOCOL_VERSION,
9052 control_ops: None,
9053 launch_nonce: None,
9054 })
9055 .expect("HELLO serializes"),
9056 )
9057 .expect("HELLO frame builds");
9058 handler
9059 .handle_control(ConnectionId::new(78), hello)
9060 .expect("consumer registers");
9061 let describe = Frame::build(
9062 FrameType::Request,
9063 control_flags(),
9064 0,
9065 0,
9066 79,
9067 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
9068 .expect("request serializes"),
9069 )
9070 .expect("describe frame builds");
9071 let response = handler
9072 .handle_server_describe(describe)
9073 .expect("server.describe succeeds");
9074 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
9075 let requirement = &rendered["capability_requirements"][0];
9076 assert_eq!(requirement["consumer"], "consumer");
9077 assert_eq!(requirement["verdict"], "never_provided");
9078 assert_eq!(requirement["episode_seq"], 1);
9079 assert_eq!(requirement["config_satisfiable"], false);
9080 assert_eq!(requirement["runtime_available"], false);
9081 assert!(requirement["detail"]
9082 .as_str()
9083 .expect("detail string")
9084 .contains("credentials-provider/v1"));
9085 }
9086
9087 #[test]
9088 fn catalog_list_omits_capabilities_for_legacy_manifest() {
9089 let registry = Arc::new(Registry::default());
9090 let handler = ControlHandler::new(Arc::clone(®istry));
9091 let hello = handler
9092 .handle_control(
9093 ConnectionId::new(101),
9094 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
9095 )
9096 .expect("legacy HELLO registers");
9097 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9098
9099 let request = Frame::build(
9100 FrameType::Request,
9101 control_flags(),
9102 0,
9103 0,
9104 102,
9105 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9106 .expect("catalog request serializes"),
9107 )
9108 .expect("catalog request frame builds");
9109 let response = handler
9110 .handle_catalog_list(request, None)
9111 .expect("catalog list succeeds");
9112 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9113 assert!(
9114 body["modules"][0].get("capabilities").is_none(),
9115 "legacy manifest must retain an absent capabilities field on catalog.list"
9116 );
9117 }
9118
9119 #[test]
9120 fn hello_ack_omits_storage_when_no_storage_config() {
9121 let registry = Arc::new(Registry::default());
9122 let handler = ControlHandler::new(Arc::clone(®istry));
9123 let responses = handler
9124 .handle_control(
9125 ConnectionId::new(1),
9126 hello_frame("aft", PROTOCOL_VERSION, 7),
9127 )
9128 .unwrap();
9129 let ack = parse_ack(&responses[0]);
9130 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
9131 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
9132 }
9133
9134 #[tokio::test]
9135 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
9136 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
9137 let registry = Arc::new(Registry::default());
9138 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
9139 let responses = handler
9140 .handle_control(
9141 ConnectionId::new(1),
9142 hello_frame("aft", PROTOCOL_VERSION, 7),
9143 )
9144 .unwrap();
9145 let ack = parse_ack(&responses[0]);
9146 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
9147
9148 let described = handler
9149 .handle_control_frame(
9150 &route_ctx(ConnectionId::new(2)).0,
9151 Frame::build(
9152 FrameType::Request,
9153 control_flags(),
9154 0,
9155 0,
9156 9,
9157 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9158 )
9159 .unwrap(),
9160 )
9161 .await
9162 .unwrap();
9163 let ClientControlResponse::ServerDescribe { machine_id, .. } =
9164 serde_json::from_slice(&described[0].body).unwrap()
9165 else {
9166 panic!("server.describe answered with another shape");
9167 };
9168 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
9169 }
9170
9171 #[test]
9172 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
9173 let registry = Arc::new(Registry::default());
9176 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
9177 crate::daemon_config::StorageConfig::Sqlite {
9178 data_home: std::path::PathBuf::from("/data"),
9179 },
9180 ));
9181
9182 let responses = handler
9183 .handle_control(
9184 ConnectionId::new(1),
9185 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
9186 )
9187 .unwrap();
9188 let ack = parse_ack(&responses[0]);
9189 assert_eq!(
9190 ack.storage,
9191 Some(serde_json::json!({
9192 "module_id": "alfonso-routing",
9193 "storage_namespace": "default",
9194 "isolation": { "kind": "module" },
9195 "backend": {
9196 "backend": "sqlite",
9197 "path": "/data/cortexkit/alfonso-routing/store.db"
9198 }
9199 })),
9200 "the delivered descriptor is the module's own sqlite store path"
9201 );
9202 }
9203
9204 #[test]
9205 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
9206 let registry = Arc::new(Registry::default());
9207 let handler = ControlHandler::new(Arc::clone(®istry));
9208 let conn = ConnectionId::new(1);
9209 let responses = handler
9210 .handle_control(
9211 conn,
9212 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9213 )
9214 .unwrap();
9215 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9216 let registration = registry.get_module("aft").unwrap().unwrap();
9217 assert_eq!(registration.control_ops, module_baseline_control_ops());
9218
9219 let frame =
9220 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
9221 assert!(handler
9222 .guard_module_control_op(&frame, "aft", "route.bind")
9223 .unwrap()
9224 .is_none());
9225 let error = handler
9226 .guard_module_control_op(&frame, "aft", "test.synthetic")
9227 .unwrap()
9228 .expect("synthetic ungranted op should be rejected");
9229 assert_eq!(error.header.ty, FrameType::Error);
9230 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
9231 }
9232
9233 #[test]
9234 fn hello_control_ops_some_adds_optional_grants() {
9235 let registry = Arc::new(Registry::default());
9236 let handler = ControlHandler::new(Arc::clone(®istry));
9237 handler
9238 .handle_control(
9239 ConnectionId::new(1),
9240 hello_frame_with_control_ops(
9241 "aft",
9242 PROTOCOL_VERSION,
9243 7,
9244 Some(vec![
9245 "future.synthetic".to_string(),
9246 "route.bind".to_string(),
9247 ]),
9248 ),
9249 )
9250 .unwrap();
9251 let registration = registry.get_module("aft").unwrap().unwrap();
9252 assert_eq!(
9253 registration.control_ops,
9254 vec![
9255 "route.bind".to_string(),
9256 "route.status".to_string(),
9257 "future.synthetic".to_string(),
9258 ]
9259 );
9260 let frame =
9261 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
9262 assert!(handler
9263 .guard_module_control_op(&frame, "aft", "future.synthetic")
9264 .unwrap()
9265 .is_none());
9266 }
9267
9268 #[tokio::test]
9269 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
9270 let registry = Arc::new(Registry::default());
9271 let forwarding = Arc::new(ForwardingTable::default());
9272 let handler =
9273 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9274 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
9275 hello_via_sink(
9276 &handler,
9277 &module_ctx,
9278 &mut module_rx,
9279 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9280 )
9281 .await;
9282
9283 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
9284 let responses = handler
9285 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
9286 .await
9287 .unwrap();
9288 assert_eq!(responses.len(), 1);
9289 assert_eq!(responses[0].header.ty, FrameType::Error);
9290 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
9291 assert!(module_rx.try_recv().is_err());
9292 }
9293
9294 #[tokio::test]
9295 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
9296 let registry = Arc::new(Registry::default());
9297 let forwarding = Arc::new(ForwardingTable::default());
9298 let handler =
9299 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9300 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
9301 hello_via_sink(
9302 &handler,
9303 &module_ctx,
9304 &mut module_rx,
9305 hello_frame_with_control_ops(
9306 "aft",
9307 PROTOCOL_VERSION,
9308 7,
9309 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9310 ),
9311 )
9312 .await;
9313
9314 let project_root = unique_project_root("demux");
9315 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
9316 let route_handler = handler.clone();
9317 let route_task = tokio::spawn(async move {
9318 route_handler
9319 .handle_control_frame(
9320 &route_client_ctx,
9321 route_open_frame(100, "aft", project_root),
9322 )
9323 .await
9324 .unwrap()
9325 });
9326 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9327 .await
9328 .unwrap()
9329 .unwrap();
9330 assert!(matches!(
9331 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
9332 ModuleControlRequest::RouteBind { .. }
9333 ));
9334
9335 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
9336 let health_handler = handler.clone();
9337 let health_task = tokio::spawn(async move {
9338 health_handler
9339 .handle_control_frame(
9340 &health_client_ctx,
9341 supervisor_health_probe_frame(101, "aft"),
9342 )
9343 .await
9344 .unwrap()
9345 });
9346 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9347 .await
9348 .unwrap()
9349 .unwrap();
9350 assert_eq!(
9351 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9352 ModuleControlRequest::HealthCheck {}
9353 );
9354
9355 handler
9356 .handle_control_frame(
9357 &module_ctx,
9358 health_response(health_frame.header.corr, HealthStatus::Degraded),
9359 )
9360 .await
9361 .unwrap();
9362 let health_response = health_task.await.unwrap();
9363 assert_eq!(health_response.len(), 1);
9364 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9365 ClientControlResponse::SupervisorHealthProbe {
9366 module_id,
9367 status,
9368 detail,
9369 metrics,
9370 } => {
9371 assert_eq!(module_id, "aft");
9372 assert_eq!(status, HealthStatus::Degraded);
9373 assert_eq!(detail.as_deref(), Some("warming"));
9374 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
9375 }
9376 other => panic!("unexpected health response: {other:?}"),
9377 }
9378
9379 handler
9380 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9381 .await
9382 .unwrap();
9383 let route_response = route_task.await.unwrap();
9384 assert!(route_response.is_empty());
9385 let published = route_client_rx.recv().await.unwrap();
9386 assert!(matches!(
9387 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9388 ClientControlResponse::RouteOpen { .. }
9389 ));
9390 }
9391
9392 async fn relay_route_open(
9397 handler: &ControlHandler,
9398 client_connection: ConnectionId,
9399 client_egress: &FrameSink,
9400 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9401 corr: u64,
9402 module_id: &str,
9403 project_root_label: &str,
9404 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
9405 let ctx = RouteCtx {
9406 connection_id: client_connection,
9407 egress: client_egress.clone(),
9408 };
9409 let handler = handler.clone();
9410 let project_root = unique_project_root(project_root_label);
9411 let module_id = module_id.to_string();
9412 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
9413 let task = tokio::spawn(async move {
9414 let _guard = tracing::dispatcher::set_default(&dispatch);
9415 handler
9416 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
9417 .await
9418 .unwrap()
9419 });
9420 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
9421 .await
9422 .expect("module receives the relayed route.bind")
9423 .expect("module egress is open");
9424 (task, bind.frame)
9425 }
9426
9427 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
9428 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
9429 ModuleControlRequest::RouteBind {
9430 route_channel,
9431 epoch,
9432 ..
9433 } => (route_channel, epoch),
9434 other => panic!("expected a route.bind request, got {other:?}"),
9435 }
9436 }
9437
9438 fn published_route(frame: &Frame) -> (u16, u32) {
9439 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
9440 ClientControlResponse::RouteOpen {
9441 route_channel,
9442 route_epoch,
9443 } => (route_channel, route_epoch),
9444 other => panic!("expected a route.open response, got {other:?}"),
9445 }
9446 }
9447
9448 #[tokio::test]
9468 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
9469 let registry = Arc::new(Registry::default());
9470 let forwarding = Arc::new(ForwardingTable::default());
9471 let handler =
9472 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9473
9474 let module_connection = ConnectionId::new(30);
9475 let (module_ctx, mut module_rx) = route_ctx(module_connection);
9476 hello_via_sink(
9477 &handler,
9478 &module_ctx,
9479 &mut module_rx,
9480 hello_frame("aft", PROTOCOL_VERSION, 7),
9481 )
9482 .await;
9483
9484 let dying_client = ConnectionId::new(31);
9485 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
9486
9487 let (first_task, first_bind) = relay_route_open(
9490 &handler,
9491 dying_client,
9492 &dying_ctx.egress,
9493 &mut module_rx,
9494 100,
9495 "aft",
9496 "closing-first",
9497 )
9498 .await;
9499 handler
9500 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
9501 .await
9502 .unwrap();
9503 assert!(first_task.await.unwrap().is_empty());
9504 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
9505
9506 let (second_task, second_bind) = relay_route_open(
9508 &handler,
9509 dying_client,
9510 &dying_ctx.egress,
9511 &mut module_rx,
9512 101,
9513 "aft",
9514 "closing-second",
9515 )
9516 .await;
9517 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
9518
9519 assert!(forwarding
9522 .escalate_client_delivery_failure(
9523 dying_client,
9524 first_channel,
9525 first_epoch,
9526 CloseReason::new(
9527 "module_to_client_delivery_failed",
9528 "client egress refused a module frame",
9529 ),
9530 crate::forwarding::UndeliveredFrame {
9531 module_id: None,
9532 sink: &dying_ctx.egress,
9533 },
9534 )
9535 .unwrap());
9536 assert!(!dying_ctx.egress.is_closed());
9537
9538 let ack = handler
9540 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
9541 .await;
9542 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
9543 if module_loop_error.is_some() {
9544 handler.cleanup_connection(module_connection).unwrap();
9548 }
9549 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9553 .await
9554 .ok()
9555 .flatten();
9556
9557 assert!(
9559 registry
9560 .get_module_by_connection(module_connection)
9561 .unwrap()
9562 .is_some(),
9563 "one client's closing connection ended the shared module connection: \
9564 {module_loop_error:?}"
9565 );
9566 let cotenant = ConnectionId::new(32);
9568 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
9569 let (cotenant_task, cotenant_bind) = relay_route_open(
9570 &handler,
9571 cotenant,
9572 &cotenant_ctx.egress,
9573 &mut module_rx,
9574 102,
9575 "aft",
9576 "closing-cotenant",
9577 )
9578 .await;
9579 handler
9580 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
9581 .await
9582 .unwrap();
9583 assert!(cotenant_task.await.unwrap().is_empty());
9584 let (cotenant_channel, cotenant_epoch) =
9585 published_route(&cotenant_rx.recv().await.unwrap());
9586 assert!(matches!(
9587 forwarding
9588 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
9589 .unwrap(),
9590 DataRoute::Client(DataRouteState::Bound(_))
9591 ));
9592
9593 let goodbye = post_ack_module_frame
9596 .expect("module receives a GOODBYE for the abandoned route channel");
9597 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9598 assert_eq!(goodbye.header.channel, abandoned_channel);
9599 assert_eq!(goodbye.header.epoch, abandoned_epoch);
9600
9601 assert!(dying_rx.try_recv().is_err());
9605 let second_response = second_task.await.unwrap();
9606 assert_eq!(second_response.len(), 1);
9607 assert_eq!(
9608 parse_error(&second_response[0])["code"],
9609 "target_unavailable"
9610 );
9611 }
9612
9613 async fn catalog_snapshot(
9616 handler: &ControlHandler,
9617 client_ctx: &RouteCtx,
9618 corr: u64,
9619 ) -> (u64, Vec<(String, bool)>) {
9620 let body =
9621 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
9622 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
9623 let reply = handler
9624 .handle_control_frame(client_ctx, request)
9625 .await
9626 .unwrap()
9627 .pop()
9628 .unwrap();
9629 let ClientControlResponse::CatalogList {
9630 generation,
9631 modules,
9632 ..
9633 } = serde_json::from_slice(&reply.body).unwrap()
9634 else {
9635 panic!("not a catalog.list reply");
9636 };
9637 (
9638 generation,
9639 modules
9640 .into_iter()
9641 .map(|entry| (entry.module_id, entry.ready))
9642 .collect(),
9643 )
9644 }
9645
9646 fn hello_held_before_forwarding() -> (
9652 ControlHandler,
9653 Arc<std::sync::Barrier>,
9654 std::thread::JoinHandle<Vec<Frame>>,
9655 RouteCtx,
9656 mpsc::Receiver<crate::router::OutboundFrame>,
9657 ) {
9658 let hold = Arc::new(std::sync::Barrier::new(2));
9659 let handler = ControlHandler::with_forwarding(
9660 Arc::new(Registry::default()),
9661 Arc::new(ForwardingTable::default()),
9662 )
9663 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
9664 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9665 let hello = {
9666 let handler = handler.clone();
9667 let module_ctx = module_ctx.clone();
9668 std::thread::spawn(move || {
9669 handler
9670 .handle_hello(
9671 module_ctx.connection_id,
9672 Some(module_ctx.egress.clone()),
9673 hello_frame("aft", PROTOCOL_VERSION, 7),
9674 )
9675 .unwrap()
9676 })
9677 };
9678 (handler, hold, hello, module_ctx, module_rx)
9679 }
9680
9681 #[tokio::test]
9690 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
9691 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
9692 let client = ConnectionId::new(41);
9693 let (client_ctx, mut client_rx) = route_ctx(client);
9694
9695 hold.wait();
9696 assert!(handler.registry.get_module("aft").unwrap().is_some());
9698 assert!(!handler
9699 .forwarding
9700 .has_live_module_connection("aft")
9701 .unwrap());
9702 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
9703 assert!(
9704 listed.is_empty(),
9705 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
9706 );
9707 let refused = handler
9708 .handle_control_frame(
9709 &client_ctx,
9710 route_open_frame(301, "aft", unique_project_root("listed-window")),
9711 )
9712 .await
9713 .unwrap();
9714 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
9715 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
9716 let duplicate = handler
9717 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
9718 .await
9719 .unwrap();
9720 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9721
9722 hold.wait();
9723 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9724 let ack = module_rx.try_recv().unwrap().frame;
9725 assert_eq!(ack.header.ty, FrameType::HelloAck);
9726 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9727 assert_eq!(listed, vec![("aft".to_string(), true)]);
9728
9729 let (open_task, bind) = relay_route_open(
9730 &handler,
9731 client,
9732 &client_ctx.egress,
9733 &mut module_rx,
9734 303,
9735 "aft",
9736 "listed-after",
9737 )
9738 .await;
9739 assert!(handler
9740 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9741 .await
9742 .unwrap()
9743 .is_empty());
9744 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
9745 published_route(&client_rx.try_recv().unwrap().frame);
9746 }
9747
9748 #[tokio::test]
9755 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
9756 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
9757 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9758 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9759
9760 hold.wait();
9761 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
9762 assert!(listed.is_empty());
9763 assert_eq!(
9764 during, before,
9765 "the generation moved while the module was not listed"
9766 );
9767
9768 hold.wait();
9769 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9770 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9771 assert_eq!(listed, vec![("aft".to_string(), true)]);
9772 assert_eq!(
9773 after,
9774 before + 1,
9775 "the module appeared without exactly one generation bump"
9776 );
9777 }
9778
9779 #[tokio::test]
9784 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
9785 let registry = Arc::new(Registry::default());
9786 let handler = ControlHandler::with_forwarding(
9787 Arc::clone(®istry),
9788 Arc::new(ForwardingTable::default()),
9789 );
9790 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9791 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9792 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9793 drop(module_rx);
9794
9795 let replies = handler
9796 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
9797 .await
9798 .unwrap();
9799 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
9800 assert!(registry.get_module("aft").unwrap().is_none());
9801 assert!(registry
9802 .get_module_by_connection(module_ctx.connection_id)
9803 .unwrap()
9804 .is_none());
9805 assert_eq!(
9806 catalog_snapshot(&handler, &client_ctx, 301).await,
9807 (before, Vec::new())
9808 );
9809 }
9810
9811 async fn route_open_held_after_bind_send() -> (
9815 ControlHandler,
9816 Arc<tokio::sync::Barrier>,
9817 RouteCtx,
9818 mpsc::Receiver<crate::router::OutboundFrame>,
9819 ) {
9820 let hold = Arc::new(tokio::sync::Barrier::new(2));
9821 let handler = ControlHandler::with_forwarding(
9822 Arc::new(Registry::default()),
9823 Arc::new(ForwardingTable::default()),
9824 )
9825 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
9826 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
9827 hello_via_sink(
9828 &handler,
9829 &module_ctx,
9830 &mut module_rx,
9831 hello_frame("aft", PROTOCOL_VERSION, 7),
9832 )
9833 .await;
9834 (handler, hold, module_ctx, module_rx)
9835 }
9836
9837 #[tokio::test]
9844 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
9845 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9846 let client = ConnectionId::new(41);
9847 let (client_ctx, mut client_rx) = route_ctx(client);
9848
9849 let (open_task, bind) = relay_route_open(
9850 &handler,
9851 client,
9852 &client_ctx.egress,
9853 &mut module_rx,
9854 200,
9855 "aft",
9856 "acked-before-mark",
9857 )
9858 .await;
9859 let (module_channel, module_epoch) = route_bind_channel(&bind);
9860 hold.wait().await;
9862
9863 assert!(handler
9865 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9866 .await
9867 .unwrap()
9868 .is_empty());
9869 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
9870
9871 hold.wait().await;
9872 assert!(
9873 open_task.await.unwrap().is_empty(),
9874 "route.open answers accepted"
9875 );
9876
9877 match module_rx.try_recv() {
9880 Err(mpsc::error::TryRecvError::Empty) => {}
9881 Ok(outbound) => panic!(
9882 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
9883 outbound.frame.header.ty,
9884 outbound.frame.header.channel,
9885 outbound.frame.header.epoch
9886 ),
9887 Err(err) => panic!("module egress closed: {err}"),
9888 }
9889
9890 let router = Router::with_control_handler(Arc::new(handler.clone()));
9893 let request = Frame::build(
9894 FrameType::Request,
9895 Flags::new(true, Priority::Interactive, false),
9896 client_channel,
9897 client_epoch,
9898 201,
9899 b"first request".to_vec(),
9900 )
9901 .unwrap();
9902 router
9903 .route_for_connection(&client_ctx, request)
9904 .await
9905 .unwrap();
9906 let delivered = module_rx.try_recv().unwrap().frame;
9907 assert_eq!(delivered.header.ty, FrameType::Request);
9908 assert_eq!(
9909 (delivered.header.channel, delivered.header.epoch),
9910 (module_channel, module_epoch)
9911 );
9912 assert_eq!(delivered.body, b"first request");
9913 }
9914
9915 #[tokio::test]
9921 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
9922 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9923 let client = ConnectionId::new(42);
9924 let (client_ctx, _client_rx) = route_ctx(client);
9925
9926 let (open_task, bind) = relay_route_open(
9927 &handler,
9928 client,
9929 &client_ctx.egress,
9930 &mut module_rx,
9931 300,
9932 "aft",
9933 "released-before-mark",
9934 )
9935 .await;
9936 let (module_channel, module_epoch) = route_bind_channel(&bind);
9937 hold.wait().await;
9938
9939 handler.cleanup_connection(client).unwrap();
9940 assert!(
9941 module_rx.try_recv().is_err(),
9942 "client teardown must not GOODBYE a bind route.open is still queueing"
9943 );
9944
9945 hold.wait().await;
9946 let response = open_task.await.unwrap();
9947 assert_eq!(response.len(), 1);
9948 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9949
9950 let goodbye = module_rx
9951 .try_recv()
9952 .expect("module is told to drop the binding the daemon abandoned")
9953 .frame;
9954 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9955 assert_eq!(
9956 (goodbye.header.channel, goodbye.header.epoch),
9957 (module_channel, module_epoch)
9958 );
9959 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9960 }
9961
9962 #[tokio::test]
9967 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
9968 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9969 let client = ConnectionId::new(43);
9970 let (client_ctx, client_rx) = route_ctx(client);
9971
9972 let (open_task, bind) = relay_route_open(
9973 &handler,
9974 client,
9975 &client_ctx.egress,
9976 &mut module_rx,
9977 400,
9978 "aft",
9979 "gone-client-before-mark",
9980 )
9981 .await;
9982 let (module_channel, module_epoch) = route_bind_channel(&bind);
9983 hold.wait().await;
9984
9985 drop(client_rx);
9986 assert!(client_ctx.egress.is_closed());
9987 handler
9988 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9989 .await
9990 .unwrap();
9991
9992 hold.wait().await;
9993 let response = open_task.await.unwrap();
9994 assert_eq!(response.len(), 1);
9995 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9996
9997 let goodbye = module_rx
9998 .try_recv()
9999 .expect("module is told to drop the binding it acked")
10000 .frame;
10001 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10002 assert_eq!(
10003 (goodbye.header.channel, goodbye.header.epoch),
10004 (module_channel, module_epoch)
10005 );
10006 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10007 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
10008 }
10009
10010 #[test]
10017 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
10018 let handler = ControlHandler::default();
10019 let module_connection = ConnectionId::new(30);
10020 let client_connection = ConnectionId::new(31);
10021
10022 handler
10023 .refuse_to_end_module_connection_for_a_client(
10024 module_connection,
10025 77,
10026 ForwardingError::ConnectionClosing {
10027 connection_id: client_connection,
10028 },
10029 )
10030 .expect("a closing client must never end the module connection");
10031
10032 assert!(matches!(
10033 handler.refuse_to_end_module_connection_for_a_client(
10034 module_connection,
10035 78,
10036 ForwardingError::ConnectionClosing {
10037 connection_id: module_connection,
10038 },
10039 ),
10040 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
10041 connection_id
10042 })) if connection_id == module_connection
10043 ));
10044 assert!(matches!(
10045 handler.refuse_to_end_module_connection_for_a_client(
10046 module_connection,
10047 79,
10048 ForwardingError::Poisoned,
10049 ),
10050 Err(RouterError::Forwarding(ForwardingError::Poisoned))
10051 ));
10052 assert!(matches!(
10053 handler.refuse_to_end_module_connection_for_a_client(
10054 module_connection,
10055 80,
10056 ForwardingError::StaleModuleEndpoint,
10057 ),
10058 Err(RouterError::Forwarding(
10059 ForwardingError::StaleModuleEndpoint
10060 ))
10061 ));
10062 }
10063
10064 #[tokio::test]
10077 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
10078 let registry = Arc::new(Registry::default());
10079 let forwarding = Arc::new(ForwardingTable::default());
10080 let supervisor = SupervisorHandle::new();
10081 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10082 let handler =
10083 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10084 .with_supervisor(supervisor);
10085
10086 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10087 hello_via_sink(
10088 &handler,
10089 &target_ctx,
10090 &mut target_rx,
10091 hello_frame("target", PROTOCOL_VERSION, 1),
10092 )
10093 .await;
10094
10095 let wrong_nonce = handler
10099 .handle_control_frame(
10100 &route_ctx(ConnectionId::new(91)).0,
10101 route_open_frame_with_admission_facts(
10102 20,
10103 "target",
10104 unique_project_root("admission-facts"),
10105 Some(subc_control::ConsumerIdentity {
10106 module_id: "fed".to_string(),
10107 launch_nonce: "not-the-real-nonce".to_string(),
10108 }),
10109 None,
10110 ),
10111 )
10112 .await
10113 .unwrap();
10114 assert_eq!(
10115 parse_error(&wrong_nonce[0])["code"],
10116 "bad_consumer_identity",
10117 "a mismatched launch nonce must be refused, not stamped Reserved"
10118 );
10119
10120 let never_spawned = handler
10124 .handle_control_frame(
10125 &route_ctx(ConnectionId::new(92)).0,
10126 route_open_frame_with_admission_facts(
10127 21,
10128 "target",
10129 unique_project_root("admission-facts"),
10130 Some(subc_control::ConsumerIdentity {
10131 module_id: "never-spawned".to_string(),
10132 launch_nonce: "any-nonce".to_string(),
10133 }),
10134 None,
10135 ),
10136 )
10137 .await
10138 .unwrap();
10139 assert_eq!(
10140 parse_error(&never_spawned[0])["code"],
10141 "bad_consumer_identity",
10142 "an unspawned module_id must be refused rather than accepted for lack of a record"
10143 );
10144 }
10145
10146 #[tokio::test]
10169 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
10170 let registry = Arc::new(Registry::default());
10171 let forwarding = Arc::new(ForwardingTable::default());
10172 let supervisor = SupervisorHandle::new();
10173 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10174 let handler =
10175 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10176 .with_supervisor(supervisor);
10177
10178 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
10179 hello_via_sink(
10180 &handler,
10181 &target_ctx,
10182 &mut target_rx,
10183 hello_frame("target", PROTOCOL_VERSION, 1),
10184 )
10185 .await;
10186
10187 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
10188 let route_handler = handler.clone();
10189 let route_task = tokio::spawn(async move {
10190 route_handler
10191 .handle_control_frame(
10192 &client_ctx,
10193 route_open_frame_with_admission_facts(
10194 30,
10195 "target",
10196 unique_project_root("admission-facts"),
10197 Some(subc_control::ConsumerIdentity {
10198 module_id: "fed".to_string(),
10199 launch_nonce: "fed-nonce".to_string(),
10200 }),
10201 None,
10202 ),
10203 )
10204 .await
10205 .unwrap()
10206 });
10207
10208 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10215 .await
10216 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
10217 .expect("module control channel closed before route.bind");
10218 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10219 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10220 panic!("expected route.bind")
10221 };
10222 assert_eq!(
10223 principal,
10224 Some(Principal::Reserved {
10225 module_id: "fed".to_string()
10226 }),
10227 "a correctly attested consumer must be stamped Reserved for its own id"
10228 );
10229
10230 handler
10231 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10232 .await
10233 .unwrap();
10234 assert!(route_task.await.unwrap().is_empty());
10235 assert!(
10236 matches!(
10237 serde_json::from_slice::<ClientControlResponse>(
10238 &client_rx.recv().await.unwrap().body
10239 )
10240 .unwrap(),
10241 ClientControlResponse::RouteOpen { .. }
10242 ),
10243 "the route must actually open, not merely avoid an error"
10244 );
10245 }
10246
10247 #[tokio::test(start_paused = true)]
10248 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
10249 let registry = Arc::new(Registry::default());
10250 let forwarding = Arc::new(ForwardingTable::default());
10251 let supervisor = SupervisorHandle::new();
10252 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10253 let handler =
10254 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10255 .with_supervisor(supervisor);
10256
10257 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
10258 hello_via_sink(
10259 &handler,
10260 &target_ctx,
10261 &mut target_rx,
10262 hello_frame("target", PROTOCOL_VERSION, 1),
10263 )
10264 .await;
10265
10266 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
10267 let direct_handler = handler.clone();
10268 let direct_open = tokio::spawn(async move {
10269 direct_handler
10270 .handle_control_frame(
10271 &direct_ctx,
10272 route_open_frame(2, "target", unique_project_root("route-census-direct")),
10273 )
10274 .await
10275 .unwrap()
10276 });
10277 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10278 .await
10279 .expect("no direct route.bind within 5s")
10280 .expect("target control channel closed before direct route.bind");
10281 handler
10282 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
10283 .await
10284 .unwrap();
10285 assert!(direct_open.await.unwrap().is_empty());
10286 let _ = direct_rx.recv().await.unwrap();
10287
10288 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
10289 let reserved_handler = handler.clone();
10290 let reserved_open = tokio::spawn(async move {
10291 reserved_handler
10292 .handle_control_frame(
10293 &reserved_ctx,
10294 route_open_frame_with_admission_facts(
10295 3,
10296 "target",
10297 unique_project_root("admission-facts"),
10298 Some(ConsumerIdentity {
10299 module_id: "fed".to_string(),
10300 launch_nonce: "fed-nonce".to_string(),
10301 }),
10302 None,
10303 ),
10304 )
10305 .await
10306 .unwrap()
10307 });
10308 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10309 .await
10310 .expect("no reserved route.bind within 5s")
10311 .expect("target control channel closed before reserved route.bind");
10312 handler
10313 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
10314 .await
10315 .unwrap();
10316 assert!(reserved_open.await.unwrap().is_empty());
10317 let _ = reserved_rx.recv().await.unwrap();
10318
10319 forwarding
10320 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
10321 .unwrap();
10322 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
10323 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
10324 module_id: Some("target".to_string()),
10325 })
10326 .unwrap();
10327 let census_frame =
10328 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
10329 let response = handler
10330 .handle_control_frame(&census_ctx, census_frame)
10331 .await
10332 .unwrap()
10333 .pop()
10334 .unwrap();
10335 let actual: Value = serde_json::from_slice(&response.body).unwrap();
10336 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
10337 assert!(matches!(
10338 decoded,
10339 ClientControlResponse::SupervisorRoutes { .. }
10340 ));
10341 let routes = actual["modules"][0]["routes"].as_array().unwrap();
10342 assert_eq!(routes.len(), 2);
10343 assert!(routes.iter().all(|route| route["draining"] == true));
10344 assert!(
10347 routes.iter().all(|route| route["drain_reason"] == "reload"),
10348 "draining routes must name the drain's reason: {routes:?}"
10349 );
10350 assert!(routes.iter().any(|route| {
10351 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
10352 }));
10353 assert!(routes.iter().any(|route| {
10354 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
10355 }));
10356
10357 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10358 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
10359 if std::env::var_os("UPDATE_GOLDEN").is_some() {
10360 std::fs::write(
10361 &golden_path,
10362 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
10363 )
10364 .unwrap();
10365 }
10366 let expected: Value =
10367 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
10368 assert_eq!(actual, expected);
10369 }
10370
10371 async fn query_live_roots(
10372 handler: &ControlHandler,
10373 module_ctx: &RouteCtx,
10374 ) -> ModuleControlResponseToModule {
10375 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
10376 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
10377 let response = handler
10378 .handle_control_frame(module_ctx, frame)
10379 .await
10380 .unwrap()
10381 .pop()
10382 .unwrap();
10383 serde_json::from_slice(&response.body).unwrap()
10384 }
10385
10386 #[tokio::test(start_paused = true)]
10387 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
10388 let registry = Arc::new(Registry::default());
10389 let forwarding = Arc::new(ForwardingTable::default());
10390 let handler = ControlHandler::with_forwarding(registry, forwarding);
10391 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
10392 hello_via_sink(
10393 &handler,
10394 &target_ctx,
10395 &mut target_rx,
10396 hello_frame("target", PROTOCOL_VERSION, 1),
10397 )
10398 .await;
10399 let root = unique_project_root("live-roots-known");
10400 let path = ProjectRootId::from_path_allowing_missing(root.path())
10401 .unwrap()
10402 .as_path()
10403 .to_path_buf();
10404 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
10405 let open_handler = handler.clone();
10406 let opened = tokio::spawn(async move {
10407 open_handler
10408 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
10409 .await
10410 .unwrap()
10411 });
10412 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10413 .await
10414 .unwrap()
10415 .unwrap();
10416 handler
10417 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
10418 .await
10419 .unwrap();
10420 assert!(opened.await.unwrap().is_empty());
10421 let _ = client_rx.recv().await.unwrap();
10422
10423 let root = unique_project_root("live-roots-pending");
10424 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
10425 .unwrap()
10426 .as_path()
10427 .to_path_buf();
10428 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
10429 let open_handler = handler.clone();
10430 let pending = tokio::spawn(async move {
10431 open_handler
10432 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
10433 .await
10434 .unwrap()
10435 });
10436 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10437 .await
10438 .unwrap()
10439 .unwrap();
10440 let actual = query_live_roots(&handler, &target_ctx).await;
10441 let ModuleControlResponseToModule::LiveRoots {
10442 roots,
10443 unknown_root_bindings,
10444 total_bindings,
10445 } = actual
10446 else {
10447 panic!("expected live roots")
10448 };
10449 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
10450 assert_eq!(unknown_root_bindings, 0);
10451 assert_eq!(
10452 roots.len(),
10453 2,
10454 "root-known arm must retain each canonical root"
10455 );
10456 assert_eq!(
10457 total_bindings,
10458 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10459 );
10460 let counts = roots
10461 .iter()
10462 .map(|root| (root.project_root.clone(), root.bound, root.pending))
10463 .collect::<Vec<_>>();
10464 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
10465 expected.sort_by(|a, b| a.0.cmp(&b.0));
10466 assert_eq!(
10467 counts, expected,
10468 "roots must sort by path and count pending separately"
10469 );
10470 handler
10471 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
10472 .await
10473 .unwrap();
10474 assert!(pending.await.unwrap().is_empty());
10475 }
10476
10477 #[tokio::test(start_paused = true)]
10478 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
10479 let forwarding = Arc::new(ForwardingTable::default());
10480 let handler =
10481 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10482 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
10483 hello_via_sink(
10484 &handler,
10485 &target_ctx,
10486 &mut target_rx,
10487 hello_frame("target", PROTOCOL_VERSION, 1),
10488 )
10489 .await;
10490 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
10491 let pending = forwarding
10492 .begin_route_bind_relay_for_test(
10493 client_ctx.connection_id,
10494 client_ctx.egress.clone(),
10495 2,
10496 "target",
10497 )
10498 .unwrap();
10499 forwarding
10500 .complete_pending_relay(
10501 target_ctx.connection_id,
10502 pending.corr,
10503 RouteBindRelayOutcome::Accepted,
10504 )
10505 .unwrap();
10506 let actual = query_live_roots(&handler, &target_ctx).await;
10507 let ModuleControlResponseToModule::LiveRoots {
10508 roots,
10509 unknown_root_bindings,
10510 total_bindings,
10511 } = actual
10512 else {
10513 panic!("expected live roots")
10514 };
10515 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
10516 assert_eq!(
10517 unknown_root_bindings, 1,
10518 "unknown-root arm must not read as no bindings"
10519 );
10520 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
10521 assert_eq!(
10522 total_bindings,
10523 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10524 );
10525 }
10526
10527 #[tokio::test(start_paused = true)]
10535 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
10536 let forwarding = Arc::new(ForwardingTable::default());
10537 let handler =
10538 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10539 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
10540 let replies = handler
10541 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
10542 .await
10543 .unwrap();
10544 let queued_by_hello = module_rx.len();
10545
10546 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
10547 let open_handler = handler.clone();
10548 let open = tokio::spawn(async move {
10549 open_handler
10550 .handle_control_frame(
10551 &client_ctx,
10552 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
10553 )
10554 .await
10555 .unwrap()
10556 });
10557 let mut spins = 0;
10559 while module_rx.len() == queued_by_hello {
10560 spins += 1;
10561 assert!(spins < 10_000, "route.open never queued a route.bind");
10562 tokio::task::yield_now().await;
10563 }
10564
10565 for reply in replies {
10567 module_ctx.egress.send(reply).await.unwrap();
10568 }
10569
10570 let first = module_rx.recv().await.unwrap().frame;
10571 assert_eq!(
10572 first.header.ty,
10573 FrameType::HelloAck,
10574 "the first frame a registering module reads must be its HELLO_ACK"
10575 );
10576 assert_eq!(first.header.corr, 7);
10577 let second = module_rx.recv().await.unwrap().frame;
10578 assert_eq!(second.header.ty, FrameType::Request);
10579 assert!(
10580 matches!(
10581 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
10582 ModuleControlRequest::RouteBind { .. }
10583 ),
10584 "the route.bind follows the ack"
10585 );
10586 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
10587
10588 handler
10589 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
10590 .await
10591 .unwrap();
10592 assert!(open.await.unwrap().is_empty());
10593 let _ = client_rx.recv().await.unwrap();
10594 }
10595
10596 #[tokio::test(start_paused = true)]
10597 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
10598 let handler = ControlHandler::with_forwarding(
10599 Arc::new(Registry::default()),
10600 Arc::new(ForwardingTable::default()),
10601 );
10602 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
10603 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
10604 hello_via_sink(
10605 &handler,
10606 &first_ctx,
10607 &mut first_rx,
10608 hello_frame("first", PROTOCOL_VERSION, 1),
10609 )
10610 .await;
10611 hello_via_sink(
10612 &handler,
10613 &second_ctx,
10614 &mut second_rx,
10615 hello_frame("second", PROTOCOL_VERSION, 2),
10616 )
10617 .await;
10618 let root = unique_project_root("second-only");
10619 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
10620 let cloned = handler.clone();
10621 let open = tokio::spawn(async move {
10622 cloned
10623 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
10624 .await
10625 .unwrap()
10626 });
10627 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
10628 .await
10629 .unwrap()
10630 .unwrap();
10631 let first = query_live_roots(&handler, &first_ctx).await;
10632 let second = query_live_roots(&handler, &second_ctx).await;
10633 assert!(
10634 matches!(
10635 first,
10636 ModuleControlResponseToModule::LiveRoots {
10637 total_bindings: 0,
10638 ..
10639 }
10640 ),
10641 "cross-module scope must not expose another module's roots"
10642 );
10643 assert!(
10644 matches!(
10645 second,
10646 ModuleControlResponseToModule::LiveRoots {
10647 total_bindings: 1,
10648 ..
10649 }
10650 ),
10651 "second module must see its pending route"
10652 );
10653 handler
10654 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
10655 .await
10656 .unwrap();
10657 assert!(open.await.unwrap().is_empty());
10658 }
10659
10660 #[tokio::test(start_paused = true)]
10661 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
10662 let handler = ControlHandler::with_forwarding(
10663 Arc::new(Registry::default()),
10664 Arc::new(ForwardingTable::default()),
10665 );
10666 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
10667 hello_via_sink(
10668 &handler,
10669 &target_ctx,
10670 &mut target_rx,
10671 hello_frame("target", PROTOCOL_VERSION, 1),
10672 )
10673 .await;
10674 let actual = query_live_roots(&handler, &target_ctx).await;
10675 let ModuleControlResponseToModule::LiveRoots {
10676 roots,
10677 unknown_root_bindings,
10678 total_bindings,
10679 } = actual
10680 else {
10681 panic!("expected live roots")
10682 };
10683 assert!(roots.is_empty());
10684 assert_eq!(unknown_root_bindings, 0);
10685 assert_eq!(total_bindings, 0);
10686 assert_eq!(
10687 total_bindings,
10688 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10689 );
10690 }
10691
10692 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
10699 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10700 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
10701 let text = std::fs::read_to_string(&path)
10702 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
10703 let vectors: Vec<(String, Value)> = text
10704 .lines()
10705 .filter(|line| !line.trim().is_empty())
10706 .map(|line| {
10707 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
10708 let id = entry["corpus_id"]
10709 .as_str()
10710 .expect("every vector carries a corpus_id")
10711 .to_string();
10712 (id, entry["package"].clone())
10713 })
10714 .collect();
10715 assert_eq!(
10719 vectors.len(),
10720 3,
10721 "vendored fed corpus changed size; re-sync from subc-federation"
10722 );
10723
10724 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
10739 let richest = vectors
10740 .iter()
10741 .filter_map(|(_, package)| package.as_object())
10742 .map(|object| {
10743 object
10744 .keys()
10745 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
10746 .count()
10747 })
10748 .max()
10749 .unwrap_or(0);
10750 assert!(
10751 richest >= 2,
10752 "vendored corpus no longer carries a package with unmodelled fields, \
10753 so the relay test can no longer distinguish a verbatim relay from a lossy one"
10754 );
10755
10756 vectors
10757 }
10758
10759 #[tokio::test]
10767 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
10768 for (corpus_id, package) in fed_admission_facts_vectors() {
10769 let registry = Arc::new(Registry::default());
10770 let forwarding = Arc::new(ForwardingTable::default());
10771 let supervisor = SupervisorHandle::new();
10772 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10773 let handler =
10774 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10775 .with_supervisor(supervisor)
10776 .with_admission_facts_config(
10777 Some("fed".to_string()),
10778 Some(vec!["target".to_string()]),
10779 );
10780
10781 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10782 hello_via_sink(
10783 &handler,
10784 &target_ctx,
10785 &mut target_rx,
10786 hello_frame("target", PROTOCOL_VERSION, 1),
10787 )
10788 .await;
10789
10790 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
10791 let route_handler = handler.clone();
10792 let expected = package.clone();
10793 let route_task = tokio::spawn(async move {
10794 route_handler
10795 .handle_control_frame(
10796 &client_ctx,
10797 route_open_frame_with_admission_facts(
10798 20,
10799 "target",
10800 unique_project_root("admission-facts"),
10801 Some(subc_control::ConsumerIdentity {
10802 module_id: "fed".to_string(),
10803 launch_nonce: "fed-nonce".to_string(),
10804 }),
10805 Some(package),
10806 ),
10807 )
10808 .await
10809 .unwrap()
10810 });
10811
10812 let bind_frame = target_rx.recv().await.unwrap();
10813 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10814 let ModuleControlRequest::RouteBind {
10815 admission_facts, ..
10816 } = bind
10817 else {
10818 panic!("{corpus_id}: expected route.bind")
10819 };
10820 assert_eq!(
10821 admission_facts,
10822 Some(expected),
10823 "{corpus_id}: relay must not add, drop or reshape any field"
10824 );
10825
10826 handler
10827 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10828 .await
10829 .unwrap();
10830 route_task.await.unwrap();
10831 }
10832 }
10833
10834 #[tokio::test]
10835 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
10836 let registry = Arc::new(Registry::default());
10837 let forwarding = Arc::new(ForwardingTable::default());
10838 let supervisor = SupervisorHandle::new();
10839 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10840 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
10841 let handler =
10842 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10843 .with_supervisor(supervisor)
10844 .with_admission_facts_config(
10845 Some("fed".to_string()),
10846 Some(vec!["target".to_string()]),
10847 );
10848
10849 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
10850 hello_via_sink(
10851 &handler,
10852 &target_ctx,
10853 &mut target_rx,
10854 hello_frame("target", PROTOCOL_VERSION, 1),
10855 )
10856 .await;
10857 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
10858 hello_via_sink(
10859 &handler,
10860 &other_ctx,
10861 &mut other_rx,
10862 hello_frame("other", PROTOCOL_VERSION, 2),
10863 )
10864 .await;
10865
10866 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
10867 let expected_facts = facts.clone();
10868 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
10869 let route_handler = handler.clone();
10870 let route_task = tokio::spawn(async move {
10871 route_handler
10872 .handle_control_frame(
10873 &client_ctx,
10874 route_open_frame_with_admission_facts(
10875 10,
10876 "target",
10877 unique_project_root("admission-facts"),
10878 Some(subc_control::ConsumerIdentity {
10879 module_id: "fed".to_string(),
10880 launch_nonce: "fed-nonce".to_string(),
10881 }),
10882 Some(facts.clone()),
10883 ),
10884 )
10885 .await
10886 .unwrap()
10887 });
10888 let bind_frame = target_rx.recv().await.unwrap();
10889 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10890 let ModuleControlRequest::RouteBind {
10891 admission_facts, ..
10892 } = bind
10893 else {
10894 panic!("expected route.bind")
10895 };
10896 assert_eq!(admission_facts, Some(expected_facts));
10897 handler
10898 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10899 .await
10900 .unwrap();
10901 assert!(route_task.await.unwrap().is_empty());
10902 assert!(matches!(
10903 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
10904 .unwrap(),
10905 ClientControlResponse::RouteOpen { .. }
10906 ));
10907
10908 let direct = handler
10909 .handle_control_frame(
10910 &route_ctx(ConnectionId::new(73)).0,
10911 route_open_frame_with_admission_facts(
10912 11,
10913 "target",
10914 unique_project_root("admission-facts"),
10915 None,
10916 Some(json!({"x": 1})),
10917 ),
10918 )
10919 .await
10920 .unwrap();
10921 assert_eq!(
10922 parse_error(&direct[0])["code"],
10923 "admission_facts_not_permitted"
10924 );
10925
10926 let different_reserved = handler
10927 .handle_control_frame(
10928 &route_ctx(ConnectionId::new(77)).0,
10929 route_open_frame_with_admission_facts(
10930 15,
10931 "target",
10932 unique_project_root("admission-facts"),
10933 Some(subc_control::ConsumerIdentity {
10934 module_id: "other".to_string(),
10935 launch_nonce: "other-nonce".to_string(),
10936 }),
10937 Some(json!({"x": 1})),
10938 ),
10939 )
10940 .await
10941 .unwrap();
10942 assert_eq!(
10943 parse_error(&different_reserved[0])["code"],
10944 "admission_facts_not_permitted"
10945 );
10946
10947 let other_target = handler
10948 .handle_control_frame(
10949 &route_ctx(ConnectionId::new(74)).0,
10950 route_open_frame_with_admission_facts(
10951 12,
10952 "other",
10953 unique_project_root("admission-facts"),
10954 Some(subc_control::ConsumerIdentity {
10955 module_id: "fed".to_string(),
10956 launch_nonce: "fed-nonce".to_string(),
10957 }),
10958 Some(json!({"x": 1})),
10959 ),
10960 )
10961 .await
10962 .unwrap();
10963 assert_eq!(
10964 parse_error(&other_target[0])["code"],
10965 "admission_facts_target_not_allowed"
10966 );
10967
10968 let nonexistent = handler
10969 .handle_control_frame(
10970 &route_ctx(ConnectionId::new(75)).0,
10971 route_open_frame_with_admission_facts(
10972 13,
10973 "missing",
10974 unique_project_root("admission-facts"),
10975 None,
10976 Some(json!({"x": 1})),
10977 ),
10978 )
10979 .await
10980 .unwrap();
10981 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
10982
10983 let described = handler
10984 .handle_control_frame(
10985 &route_ctx(ConnectionId::new(76)).0,
10986 Frame::build(
10987 FrameType::Request,
10988 control_flags(),
10989 0,
10990 0,
10991 14,
10992 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10993 )
10994 .unwrap(),
10995 )
10996 .await
10997 .unwrap();
10998 let ClientControlResponse::ServerDescribe { capabilities, .. } =
10999 serde_json::from_slice(&described[0].body).unwrap()
11000 else {
11001 panic!("expected server.describe response")
11002 };
11003 assert!(capabilities
11004 .iter()
11005 .any(|cap| cap == "admission_facts_relay_v1"));
11006 }
11007
11008 #[tokio::test]
11009 async fn admission_facts_without_configured_carrier_are_rejected() {
11010 let registry = Arc::new(Registry::default());
11011 let forwarding = Arc::new(ForwardingTable::default());
11012 let handler = ControlHandler::with_forwarding(registry, forwarding);
11013 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
11014 hello_via_sink(
11015 &handler,
11016 &target_ctx,
11017 &mut target_rx,
11018 hello_frame("target", PROTOCOL_VERSION, 1),
11019 )
11020 .await;
11021
11022 let responses = handler
11023 .handle_control_frame(
11024 &route_ctx(ConnectionId::new(79)).0,
11025 route_open_frame_with_admission_facts(
11026 16,
11027 "target",
11028 unique_project_root("admission-facts"),
11029 None,
11030 Some(json!({"x": 1})),
11031 ),
11032 )
11033 .await
11034 .unwrap();
11035 assert_eq!(
11036 parse_error(&responses[0])["code"],
11037 "admission_facts_not_permitted"
11038 );
11039 }
11040
11041 #[tokio::test]
11042 async fn route_open_relays_consumer_capabilities_verbatim() {
11043 let registry = Arc::new(Registry::default());
11044 let forwarding = Arc::new(ForwardingTable::default());
11045 let handler =
11046 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11047 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
11048 hello_via_sink(
11049 &handler,
11050 &module_ctx,
11051 &mut module_rx,
11052 hello_frame("aft", PROTOCOL_VERSION, 7),
11053 )
11054 .await;
11055
11056 let expected = vec!["elicitation".to_string(), "roots".to_string()];
11057 let expected_for_request = expected.clone();
11058 let project_root = unique_project_root("consumer-capabilities-present");
11059 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
11060 let route_handler = handler.clone();
11061 let route_task = tokio::spawn(async move {
11062 route_handler
11063 .handle_control_frame(
11064 &client_ctx,
11065 route_open_frame_with_consumer_capabilities(
11066 401,
11067 "aft",
11068 project_root,
11069 Some(expected_for_request),
11070 ),
11071 )
11072 .await
11073 .unwrap()
11074 });
11075 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11076 .await
11077 .unwrap()
11078 .unwrap();
11079 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11080 let ModuleControlRequest::RouteBind {
11081 consumer_capabilities,
11082 ..
11083 } = bind
11084 else {
11085 panic!("expected route.bind request, got {bind:?}");
11086 };
11087 assert_eq!(consumer_capabilities, Some(expected.clone()));
11088
11089 handler
11090 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11091 .await
11092 .unwrap();
11093 let route_response = route_task.await.unwrap();
11094 assert!(route_response.is_empty());
11095 let published = client_rx.recv().await.unwrap();
11096 assert!(matches!(
11097 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11098 ClientControlResponse::RouteOpen { .. }
11099 ));
11100 }
11101
11102 #[tokio::test]
11103 async fn route_open_without_consumer_capabilities_relays_none() {
11104 let registry = Arc::new(Registry::default());
11105 let forwarding = Arc::new(ForwardingTable::default());
11106 let handler =
11107 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11108 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
11109 hello_via_sink(
11110 &handler,
11111 &module_ctx,
11112 &mut module_rx,
11113 hello_frame("aft", PROTOCOL_VERSION, 7),
11114 )
11115 .await;
11116
11117 let project_root = unique_project_root("consumer-capabilities-absent");
11118 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
11119 let route_handler = handler.clone();
11120 let route_task = tokio::spawn(async move {
11121 route_handler
11122 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
11123 .await
11124 .unwrap()
11125 });
11126 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11127 .await
11128 .unwrap()
11129 .unwrap();
11130 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11131 let ModuleControlRequest::RouteBind {
11132 consumer_capabilities,
11133 ..
11134 } = bind
11135 else {
11136 panic!("expected route.bind request, got {bind:?}");
11137 };
11138 assert_eq!(consumer_capabilities, None);
11139
11140 handler
11141 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11142 .await
11143 .unwrap();
11144 let route_response = route_task.await.unwrap();
11145 assert!(route_response.is_empty());
11146 let published = client_rx.recv().await.unwrap();
11147 assert!(matches!(
11148 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11149 ClientControlResponse::RouteOpen { .. }
11150 ));
11151 }
11152
11153 async fn bind_role_versions_for(
11157 sent: Option<BTreeMap<String, String>>,
11158 connection: u64,
11159 ) -> Option<BTreeMap<String, String>> {
11160 let registry = Arc::new(Registry::default());
11161 let forwarding = Arc::new(ForwardingTable::default());
11162 let handler =
11163 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11164 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
11165 hello_via_sink(
11166 &handler,
11167 &module_ctx,
11168 &mut module_rx,
11169 hello_frame("aft", PROTOCOL_VERSION, 7),
11170 )
11171 .await;
11172 let project_root = unique_project_root("role-versions");
11173 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
11174 let route_handler = handler.clone();
11175 let route_task = tokio::spawn(async move {
11176 route_handler
11177 .handle_control_frame(
11178 &client_ctx,
11179 route_open_frame_with_role_versions(403, "aft", project_root, sent),
11180 )
11181 .await
11182 .unwrap()
11183 });
11184 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11185 .await
11186 .expect("a well-formed route.open reaches the module as a bind")
11187 .unwrap();
11188 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11189 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
11190 panic!("expected route.bind request, got {bind:?}");
11191 };
11192 handler
11193 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11194 .await
11195 .unwrap();
11196 assert!(route_task.await.unwrap().is_empty());
11197 let published = client_rx.recv().await.unwrap();
11198 assert!(matches!(
11199 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11200 ClientControlResponse::RouteOpen { .. }
11201 ));
11202 role_versions
11203 }
11204
11205 #[tokio::test]
11206 async fn route_open_relays_role_versions_verbatim() {
11207 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
11208 assert_eq!(
11209 bind_role_versions_for(Some(sent.clone()), 141).await,
11210 Some(sent)
11211 );
11212 }
11213
11214 #[tokio::test]
11217 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
11218 assert_eq!(bind_role_versions_for(None, 143).await, None);
11219 assert_eq!(
11220 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
11221 None
11222 );
11223 }
11224
11225 #[tokio::test]
11226 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
11227 let registry = Arc::new(Registry::default());
11228 let forwarding = Arc::new(ForwardingTable::default());
11229 let handler =
11230 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11231 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
11232 hello_via_sink(
11233 &handler,
11234 &module_ctx,
11235 &mut module_rx,
11236 hello_frame("aft", PROTOCOL_VERSION, 7),
11237 )
11238 .await;
11239
11240 let nine: BTreeMap<String, String> = (0..9)
11241 .map(|index| (format!("role-{index}"), "v1".to_string()))
11242 .collect();
11243 for (label, malformed) in [
11244 (
11245 "invalid role name",
11246 role_versions(&[("Tool_Provider", "v1")]),
11247 ),
11248 ("invalid version", role_versions(&[("tool-provider", "v0")])),
11249 ("nine entries", nine),
11250 ] {
11251 let responses = tokio::time::timeout(
11254 Duration::from_secs(1),
11255 handler.handle_control_frame(
11256 &route_ctx(ConnectionId::new(148)).0,
11257 route_open_frame_with_role_versions(
11258 404,
11259 "aft",
11260 unique_project_root("role-versions-malformed"),
11261 Some(malformed),
11262 ),
11263 ),
11264 )
11265 .await
11266 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
11267 .unwrap();
11268 assert_eq!(responses.len(), 1, "{label}");
11269 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
11270 let error = parse_error(&responses[0]);
11271 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
11272 assert_eq!(
11273 error["detail"]["field"], "role_versions",
11274 "{label}: {error}"
11275 );
11276 assert!(
11277 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
11278 "{label}: a malformed declaration is terminal"
11279 );
11280 assert!(
11281 module_rx.try_recv().is_err(),
11282 "{label}: the module must never see a bind"
11283 );
11284 }
11285 }
11286
11287 #[tokio::test]
11291 async fn route_role_versions_capability_is_advertised() {
11292 let handler = ControlHandler::new(Arc::new(Registry::default()));
11293 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
11294 let ack = hello_via_sink(
11295 &handler,
11296 &ctx,
11297 &mut rx,
11298 hello_frame("m", PROTOCOL_VERSION, 1),
11299 )
11300 .await;
11301 let ack = parse_ack(&ack);
11302 assert!(
11303 ack.subc_capabilities
11304 .iter()
11305 .any(|c| c == "route-role-versions/v1"),
11306 "{:?}",
11307 ack.subc_capabilities
11308 );
11309
11310 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11311 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11312 let reply = handler
11313 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
11314 .await
11315 .unwrap()
11316 .pop()
11317 .unwrap();
11318 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11319 serde_json::from_slice(&reply.body).unwrap()
11320 else {
11321 panic!("not a server.describe reply");
11322 };
11323 assert!(
11324 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
11325 "{capabilities:?}"
11326 );
11327 }
11328
11329 #[tokio::test]
11333 async fn catalog_events_capability_is_advertised() {
11334 let handler = ControlHandler::new(Arc::new(Registry::default()));
11335 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11336 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11337 let reply = handler
11338 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
11339 .await
11340 .unwrap()
11341 .pop()
11342 .unwrap();
11343 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11344 serde_json::from_slice(&reply.body).unwrap()
11345 else {
11346 panic!("not a server.describe reply");
11347 };
11348 assert!(
11349 capabilities
11350 .iter()
11351 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
11352 "{capabilities:?}"
11353 );
11354 }
11355
11356 #[tokio::test]
11357 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
11358 let handler = ControlHandler::new(Arc::new(Registry::default()));
11359 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
11360 let ack = hello_via_sink(
11361 &handler,
11362 &module,
11363 &mut module_rx,
11364 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
11365 )
11366 .await;
11367 let ack = parse_ack(&ack);
11368 assert!(
11369 ack.subc_capabilities
11370 .iter()
11371 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11372 "{:?}",
11373 ack.subc_capabilities
11374 );
11375 assert!(ack
11376 .subc_capabilities
11377 .iter()
11378 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11379
11380 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
11381 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11382 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11383 let reply = handler
11384 .handle_control_frame(&client, frame)
11385 .await
11386 .unwrap()
11387 .pop()
11388 .unwrap();
11389 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11390 serde_json::from_slice(&reply.body).unwrap()
11391 else {
11392 panic!("not a server.describe reply");
11393 };
11394 assert!(
11395 capabilities
11396 .iter()
11397 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11398 "{capabilities:?}"
11399 );
11400 assert!(capabilities
11401 .iter()
11402 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11403 }
11404
11405 #[tokio::test]
11406 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
11407 let handler = ControlHandler::new(Arc::new(Registry::default()));
11408 let (module, mut rx) = route_ctx(ConnectionId::new(156));
11409 let ack = hello_via_sink(
11410 &handler,
11411 &module,
11412 &mut rx,
11413 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
11414 )
11415 .await;
11416 assert!(parse_ack(&ack)
11417 .subc_capabilities
11418 .iter()
11419 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
11420 let (client, _rx) = route_ctx(ConnectionId::new(157));
11421 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11422 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11423 let reply = handler
11424 .handle_control_frame(&client, frame)
11425 .await
11426 .unwrap()
11427 .pop()
11428 .unwrap();
11429 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11430 serde_json::from_slice(&reply.body).unwrap()
11431 else {
11432 panic!("server.describe reply")
11433 };
11434 assert!(capabilities
11435 .iter()
11436 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
11437 }
11438
11439 #[tokio::test]
11440 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
11441 let handler = ControlHandler::new(Arc::new(Registry::default()));
11442 let (module, mut rx) = route_ctx(ConnectionId::new(154));
11443 let ack = hello_via_sink(
11444 &handler,
11445 &module,
11446 &mut rx,
11447 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
11448 )
11449 .await;
11450 assert!(parse_ack(&ack)
11451 .subc_capabilities
11452 .iter()
11453 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11454 let (client, _rx) = route_ctx(ConnectionId::new(155));
11455 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11456 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11457 let reply = handler
11458 .handle_control_frame(&client, frame)
11459 .await
11460 .unwrap()
11461 .pop()
11462 .unwrap();
11463 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11464 serde_json::from_slice(&reply.body).unwrap()
11465 else {
11466 panic!("not a server.describe reply")
11467 };
11468 assert!(capabilities
11469 .iter()
11470 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11471 }
11472
11473 #[tokio::test]
11474 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
11475 let registry = Arc::new(Registry::default());
11476 let forwarding = Arc::new(ForwardingTable::default());
11477 let handler =
11478 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11479 .with_health_probe_timeout(Duration::from_secs(5));
11480 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
11481 hello_via_sink(
11482 &handler,
11483 &module_ctx,
11484 &mut module_rx,
11485 non_routable_hello_frame_with_control_ops(
11486 "mcp",
11487 300,
11488 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11489 ),
11490 )
11491 .await;
11492 assert!(registry
11493 .get_module("mcp")
11494 .unwrap()
11495 .unwrap()
11496 .manifest
11497 .provides
11498 .is_empty());
11499
11500 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
11501 let route_response = handler
11502 .handle_control_frame(
11503 &route_client_ctx,
11504 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
11505 )
11506 .await
11507 .unwrap();
11508 assert_eq!(route_response[0].header.ty, FrameType::Error);
11509 assert_eq!(
11510 parse_error(&route_response[0])["code"],
11511 "target_unavailable"
11512 );
11513 assert!(parse_error(&route_response[0])["message"]
11514 .as_str()
11515 .unwrap()
11516 .contains("does not provide the requested target"));
11517 assert!(module_rx.try_recv().is_err());
11518
11519 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
11520 let health_handler = handler.clone();
11521 let health_task = tokio::spawn(async move {
11522 health_handler
11523 .handle_control_frame(
11524 &health_client_ctx,
11525 supervisor_health_probe_frame(302, "mcp"),
11526 )
11527 .await
11528 .unwrap()
11529 });
11530 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11531 .await
11532 .unwrap()
11533 .unwrap();
11534 assert_eq!(
11535 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11536 ModuleControlRequest::HealthCheck {}
11537 );
11538 handler
11539 .handle_control_frame(
11540 &module_ctx,
11541 health_response(health_frame.header.corr, HealthStatus::Ok),
11542 )
11543 .await
11544 .unwrap();
11545 let health_response = health_task.await.unwrap();
11546 assert_eq!(health_response[0].header.ty, FrameType::Response);
11547 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11548 ClientControlResponse::SupervisorHealthProbe {
11549 module_id, status, ..
11550 } => {
11551 assert_eq!(module_id, "mcp");
11552 assert_eq!(status, HealthStatus::Ok);
11553 }
11554 other => panic!("unexpected health response: {other:?}"),
11555 }
11556
11557 forwarding
11562 .cleanup_connection(module_ctx.connection_id)
11563 .unwrap();
11564 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
11565 let cleanup_response = tokio::time::timeout(
11566 Duration::from_millis(200),
11567 handler.handle_control_frame(
11568 &cleanup_probe_ctx,
11569 supervisor_health_probe_frame(303, "mcp"),
11570 ),
11571 )
11572 .await
11573 .expect("probe should fail immediately when the control lane is gone")
11574 .unwrap();
11575 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
11576 assert_eq!(
11577 parse_error(&cleanup_response[0])["code"],
11578 "target_unavailable"
11579 );
11580 assert!(parse_error(&cleanup_response[0])["message"]
11581 .as_str()
11582 .unwrap()
11583 .contains("no module connection"));
11584
11585 handler
11586 .cleanup_connection(module_ctx.connection_id)
11587 .unwrap();
11588 }
11589
11590 #[tokio::test]
11591 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
11592 let registry = Arc::new(Registry::default());
11593 let supervisor_handle = SupervisorHandle::new();
11594 let supervisor =
11595 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11596 .with_handle(supervisor_handle.clone())
11597 .with_connection_file_path(
11598 std::env::temp_dir()
11599 .join(format!("subc-route-open-warming-{}", std::process::id())),
11600 );
11601 let module = supervisor
11602 .supervise_configured(
11603 ModuleSpec {
11604 module_id: "warming".to_string(),
11605 program: fake_aft_stub_path(),
11606 args: Vec::new(),
11607 env: Vec::new(),
11608 reserved: false,
11609 reserved_prefixes: Vec::new(),
11610 protocol: ModuleProtocol::Subc,
11611 overlap: Default::default(),
11612 },
11613 true,
11614 )
11615 .unwrap();
11616 assert_eq!(module.state().unwrap(), ModuleState::Running);
11617
11618 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11619 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
11620 let response = handler
11621 .handle_control_frame(
11622 &ctx,
11623 route_open_frame(304, "warming", unique_project_root("warming")),
11624 )
11625 .await
11626 .unwrap();
11627 module.stop().await.unwrap();
11628
11629 assert_eq!(response[0].header.ty, FrameType::Error);
11630 let error = parse_error(&response[0]);
11631 assert_eq!(error["code"], "module_warming");
11632 assert!(error["message"]
11633 .as_str()
11634 .unwrap()
11635 .contains("state=running, enabled=true, live=false"));
11636 }
11637
11638 #[test]
11639 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
11640 let handler = ControlHandler::new(Arc::new(Registry::default()));
11641 let capture = EventCapture::default();
11642 let _subscriber =
11643 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11644 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
11645 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
11646 let pending = (0..limit).collect::<Vec<_>>();
11647 let response = handler
11648 .route_open_capacity_refusal(
11649 &ctx,
11650 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
11651 "busy",
11652 pending.len(),
11653 limit,
11654 )
11655 .unwrap();
11656 assert_eq!(parse_error(&response)["code"], "target_unavailable");
11657 let event = capture
11658 .events()
11659 .into_iter()
11660 .find(|event| {
11661 event.target == "control"
11662 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
11663 })
11664 .expect("connection admission refusal event");
11665 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11666 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11667 }
11668
11669 #[test]
11670 fn route_open_target_cap_logs_admission_reason_and_capacity() {
11671 let handler = ControlHandler::new(Arc::new(Registry::default()));
11672 let capture = EventCapture::default();
11673 let _subscriber =
11674 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11675 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
11676 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
11677 let guards = (0..limit)
11678 .map(|_| {
11679 handler
11680 .route_bind_concurrency
11681 .try_admit("busy", limit)
11682 .unwrap()
11683 })
11684 .collect::<Vec<_>>();
11685 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
11686 Err(in_flight) => in_flight,
11687 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
11688 };
11689 let response = handler
11690 .route_open_target_capacity_refusal(
11691 &ctx,
11692 &route_open_frame(397, "busy", unique_project_root("target-cap")),
11693 "busy",
11694 in_flight,
11695 )
11696 .unwrap();
11697 assert_eq!(parse_error(&response)["code"], "target_unavailable");
11698 let event = capture
11699 .events()
11700 .into_iter()
11701 .find(|event| {
11702 event.target == "control"
11703 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
11704 })
11705 .expect("target admission refusal event");
11706 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11707 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11708 drop(guards);
11709 }
11710
11711 #[tokio::test(flavor = "current_thread")]
11712 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
11713 let handler = ControlHandler::new(Arc::new(Registry::default()));
11714 let capture = EventCapture::default();
11715 let _subscriber =
11716 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11717 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
11718 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11719
11720 let _ = handler
11721 .handle_client_control_request(
11722 &ctx,
11723 frame,
11724 ClientControlRequest::SupervisorSetEnabled {
11725 module_id: "broca".to_string(),
11726 enabled: false,
11727 },
11728 &mut None,
11729 )
11730 .await
11731 .unwrap();
11732
11733 let events = capture
11734 .events()
11735 .into_iter()
11736 .filter(|event| {
11737 event.target == "control"
11738 && event.fields.get("message").map(String::as_str)
11739 == Some("supervisor request received")
11740 })
11741 .collect::<Vec<_>>();
11742 assert_eq!(events.len(), 1, "one supervisor request log line");
11743 let event = &events[0];
11744 assert_eq!(event.level, tracing::Level::INFO);
11745 assert_eq!(
11746 event.fields.get("op"),
11747 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
11748 );
11749 assert_eq!(
11750 event.fields.get("module_id"),
11751 Some(&"\"broca\"".to_string())
11752 );
11753 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
11754 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
11755 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
11756 }
11757
11758 #[tokio::test(flavor = "current_thread")]
11759 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
11760 let registry = Arc::new(Registry::default());
11761 let module_connection = ConnectionId::new(303);
11762 registry
11763 .register_with_control_ops(
11764 manifest("aft", PROTOCOL_VERSION),
11765 PROTOCOL_VERSION,
11766 module_connection,
11767 module_baseline_control_ops(),
11768 )
11769 .unwrap();
11770 let handler = ControlHandler::new(registry);
11771 let capture = EventCapture::default();
11772 let _subscriber =
11773 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11774 let (ctx, _rx) = route_ctx(module_connection);
11775 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11776
11777 let _ = handler
11778 .handle_client_control_request(
11779 &ctx,
11780 frame,
11781 ClientControlRequest::SupervisorSetEnabled {
11782 module_id: "broca".to_string(),
11783 enabled: false,
11784 },
11785 &mut None,
11786 )
11787 .await
11788 .unwrap();
11789
11790 let callers = capture
11791 .events()
11792 .into_iter()
11793 .filter(|event| {
11794 event.fields.get("message").map(String::as_str)
11795 == Some("supervisor request received")
11796 })
11797 .map(|event| event.fields.get("caller").cloned())
11798 .collect::<Vec<_>>();
11799 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
11800 }
11801
11802 #[tokio::test(flavor = "current_thread")]
11803 async fn supervisor_list_does_not_log_request_received() {
11804 let handler = ControlHandler::new(Arc::new(Registry::default()));
11805 let capture = EventCapture::default();
11806 let _subscriber =
11807 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11808 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
11809 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
11810
11811 handler
11812 .handle_client_control_request(
11813 &ctx,
11814 frame,
11815 ClientControlRequest::SupervisorList {},
11816 &mut None,
11817 )
11818 .await
11819 .unwrap();
11820
11821 assert!(capture.events().into_iter().all(|event| {
11822 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11823 }));
11824 }
11825
11826 #[tokio::test(flavor = "current_thread")]
11827 async fn supervisor_rescan_preview_does_not_log_request_received() {
11828 let handler = ControlHandler::new(Arc::new(Registry::default()));
11829 let capture = EventCapture::default();
11830 let _subscriber =
11831 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11832 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
11833 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
11834
11835 handler
11836 .handle_client_control_request(
11837 &ctx,
11838 frame,
11839 ClientControlRequest::SupervisorRescan { preview: true },
11840 &mut None,
11841 )
11842 .await
11843 .unwrap();
11844
11845 assert!(capture.events().into_iter().all(|event| {
11846 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11847 }));
11848 }
11849
11850 struct ApproveOperator;
11851 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
11852 fn prompt(
11853 &self,
11854 _: &str,
11855 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
11856 ) -> crate::operator_confirm::ProviderResult {
11857 crate::operator_confirm::ProviderResult::Approved
11858 }
11859 }
11860
11861 #[tokio::test]
11862 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
11863 assert_eq!(
11864 ControlHandler::default().privileged_operator_prompt,
11865 PrivilegedOperatorPrompt::Log
11866 );
11867 for (initial, changed, running) in [
11868 ("log", "enforce", PrivilegedOperatorPrompt::Log),
11869 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
11870 ] {
11871 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
11872 let path = dir.join("subc.jsonc");
11873 std::fs::write(
11874 &path,
11875 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
11876 )
11877 .unwrap();
11878 let config = crate::daemon_config::load(&path).unwrap().unwrap();
11879 let registry = Arc::new(Registry::default());
11880 let handle = SupervisorHandle::new();
11881 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
11882 .with_handle(handle.clone());
11883 let handler = ControlHandler::new(registry)
11884 .with_supervisor(handle)
11885 .with_privileged_operator_prompt(config.privileged_operator_prompt)
11886 .with_supervisor_rescan(supervisor, &path, config.port);
11887 handler.forwarding.operator_confirms().configure(
11888 Arc::new(ApproveOperator),
11889 crate::operator_confirm::OperatorLimits::default(),
11890 );
11891 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
11892
11893 for (disk_mode, needs_restart) in [
11894 (Some(initial), false),
11895 (Some(changed), true),
11896 (Some(changed), true),
11897 (None, running == PrivilegedOperatorPrompt::Enforce),
11898 (Some(initial), false),
11899 ] {
11900 let mut doc = json!({"version":1});
11901 if let Some(mode) = disk_mode {
11902 doc["privileged_operator_prompt"] = json!(mode);
11903 }
11904 std::fs::write(&path, doc.to_string()).unwrap();
11905 for preview in [true, false] {
11906 let request = ClientControlRequest::SupervisorRescan { preview };
11907 let frame = Frame::build(
11908 FrameType::Request,
11909 control_flags(),
11910 0,
11911 0,
11912 1,
11913 serde_json::to_vec(&request).unwrap(),
11914 )
11915 .unwrap();
11916 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
11917 assert_eq!(frames.len(), 1);
11918 let ClientControlResponse::SupervisorRescan { result } =
11919 serde_json::from_slice(&frames[0].body).unwrap()
11920 else {
11921 panic!("expected a rescan response");
11922 };
11923 let expected = if needs_restart {
11924 vec!["privileged_operator_prompt"]
11925 } else {
11926 Vec::new()
11927 };
11928 assert_eq!(result.restart_required, expected);
11929 assert_eq!(handler.privileged_operator_prompt, running);
11930 assert_eq!(
11931 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
11932 running
11933 );
11934 }
11935 }
11936 }
11937 }
11938
11939 #[tokio::test]
11940 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
11941 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
11942 let path = dir.join("subc.jsonc");
11943 let registry = Arc::new(Registry::default());
11944 let handle = SupervisorHandle::new();
11945 let supervisor =
11946 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
11947 let handler = ControlHandler::new(registry)
11948 .with_supervisor(handle)
11949 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
11950 .with_supervisor_rescan(supervisor, &path, None);
11951 handler.forwarding.operator_confirms().configure(
11952 Arc::new(ApproveOperator),
11953 crate::operator_confirm::OperatorLimits::default(),
11954 );
11955 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
11956 for preview in [true, false] {
11957 std::fs::write(
11958 &path,
11959 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
11960 )
11961 .unwrap();
11962 let request = ClientControlRequest::SupervisorRescan { preview };
11963 let frame = Frame::build(
11964 FrameType::Request,
11965 control_flags(),
11966 0,
11967 0,
11968 1,
11969 serde_json::to_vec(&request).unwrap(),
11970 )
11971 .unwrap();
11972 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
11973 assert_eq!(frames.len(), 1);
11974 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
11975 assert_eq!(
11976 handler.privileged_operator_prompt,
11977 PrivilegedOperatorPrompt::Enforce
11978 );
11979 }
11980 }
11981
11982 #[tokio::test]
11987 async fn route_open_refusal_names_the_check_that_refused() {
11988 let handler = ControlHandler::new(Arc::new(Registry::default()));
11989 let capture = EventCapture::default();
11990 let _subscriber =
11991 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11992 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
11993 let response = handler
11994 .handle_control_frame(
11995 &ctx,
11996 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
11997 )
11998 .await
11999 .unwrap();
12000
12001 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12002 let event = capture
12003 .events()
12004 .into_iter()
12005 .find(|event| {
12006 event.target == "control"
12007 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12008 })
12009 .expect("route.open refusal event");
12010 assert_eq!(
12011 event.fields.get("reason"),
12012 Some(&"\"not_registered\"".to_string())
12013 );
12014 }
12015
12016 #[tokio::test]
12017 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
12018 let registry = Arc::new(Registry::default());
12019 let supervisor_handle = SupervisorHandle::new();
12020 let supervisor =
12021 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12022 .with_handle(supervisor_handle.clone())
12023 .with_connection_file_path(std::env::temp_dir().join(format!(
12024 "subc-route-open-refusal-info-{}",
12025 std::process::id()
12026 )));
12027 let module = supervisor
12028 .supervise_configured(
12029 ModuleSpec {
12030 module_id: "warming".to_string(),
12031 program: fake_aft_stub_path(),
12032 args: Vec::new(),
12033 env: Vec::new(),
12034 reserved: false,
12035 reserved_prefixes: Vec::new(),
12036 protocol: ModuleProtocol::Subc,
12037 overlap: Default::default(),
12038 },
12039 true,
12040 )
12041 .unwrap();
12042 assert_eq!(module.state().unwrap(), ModuleState::Running);
12043
12044 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12045 assert!(handler
12046 .counters()
12047 .snapshot()
12048 .get("route_open_refused_by_code")
12049 .is_none());
12050 let capture = EventCapture::default();
12051 let _subscriber =
12052 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12053 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
12054 let response = handler
12055 .handle_control_frame(
12056 &ctx,
12057 route_open_frame(394, "warming", unique_project_root("refusal-info")),
12058 )
12059 .await
12060 .unwrap();
12061 module.stop().await.unwrap();
12062
12063 assert_eq!(parse_error(&response[0])["code"], "module_warming");
12064 let event = capture
12065 .events()
12066 .into_iter()
12067 .find(|event| {
12068 event.target == "control"
12069 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
12070 })
12071 .expect("route.open refusal event");
12072 assert_eq!(
12073 event.fields.get("module_id"),
12074 Some(&"\"warming\"".to_string())
12075 );
12076 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
12077 assert_eq!(
12078 event.fields.get("reason"),
12079 Some(&"\"supervised_not_registered\"".to_string())
12080 );
12081 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
12082 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
12083 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
12084 assert_eq!(
12085 handler.counters().snapshot()["route_open_refused_by_code"],
12086 json!({ "module_warming": 1 })
12087 );
12088 }
12089
12090 const OUTAGE_START: &str = "route.open refusing module: not serving";
12091 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
12092
12093 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
12094 capture
12095 .events()
12096 .into_iter()
12097 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
12098 .collect()
12099 }
12100
12101 fn supervise_stub(
12102 registry: &Arc<Registry>,
12103 module_id: &str,
12104 enabled: bool,
12105 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
12106 let supervisor_handle = SupervisorHandle::new();
12107 let supervisor =
12108 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
12109 .with_handle(supervisor_handle.clone())
12110 .with_connection_file_path(std::env::temp_dir().join(format!(
12111 "subc-route-outage-{module_id}-{}",
12112 std::process::id()
12113 )));
12114 let module = supervisor
12115 .supervise_configured(
12116 ModuleSpec {
12117 module_id: module_id.to_string(),
12118 program: fake_aft_stub_path(),
12119 args: Vec::new(),
12120 env: Vec::new(),
12121 reserved: false,
12122 reserved_prefixes: Vec::new(),
12123 protocol: ModuleProtocol::Subc,
12124 overlap: Default::default(),
12125 },
12126 enabled,
12127 )
12128 .unwrap();
12129 (supervisor_handle, module)
12130 }
12131
12132 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
12133 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
12134 module_id: module_id.to_string(),
12135 drain_timeout_ms: Some(50),
12136 })
12137 .unwrap();
12138 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
12139 }
12140
12141 #[test]
12145 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
12146 let registry = Arc::new(Registry::default());
12147 let forwarding = Arc::new(ForwardingTable::default());
12148 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12149 let second = ControlHandler::with_forwarding(registry, forwarding);
12150 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
12151 }
12152
12153 #[tokio::test(flavor = "current_thread")]
12157 async fn route_open_unknown_module_refusals_add_no_outage_state() {
12158 let handler = ControlHandler::new(Arc::new(Registry::default()));
12159 let capture = EventCapture::default();
12160 let _subscriber =
12161 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12162 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
12163 for corr in 0..8 {
12164 let response = handler
12165 .handle_control_frame(
12166 &ctx,
12167 route_open_frame(
12168 380 + corr,
12169 &format!("nobody-{corr}"),
12170 unique_project_root("outage-unknown"),
12171 ),
12172 )
12173 .await
12174 .unwrap();
12175 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12176 }
12177
12178 assert_eq!(handler.route_outages.tracked_module_count(), 0);
12179 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
12180 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
12181 }
12182
12183 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12188 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
12189 let registry = Arc::new(Registry::default());
12190 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
12191 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12192 let capture = EventCapture::default();
12193 let _subscriber =
12194 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12195 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
12196 handler.route_outages.record_accepted("outage-restart");
12199
12200 let response = handler
12201 .handle_control_frame(
12202 &ctx,
12203 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
12204 )
12205 .await
12206 .unwrap();
12207 assert_eq!(response[0].header.ty, FrameType::Error);
12208 let starts = outage_lines(&capture, OUTAGE_START);
12209 assert_eq!(starts.len(), 1, "{starts:?}");
12210 assert_eq!(starts[0].level, tracing::Level::WARN);
12211 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
12212 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
12213 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
12214 handler.route_outages.record_accepted("outage-restart");
12215 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
12216
12217 let restart = handler
12218 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
12219 .await
12220 .unwrap();
12221 assert_eq!(
12222 restart[0].header.ty,
12223 FrameType::Response,
12224 "{:?}",
12225 parse_error(&restart[0])
12226 );
12227 handler
12228 .handle_control_frame(
12229 &ctx,
12230 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
12231 )
12232 .await
12233 .unwrap();
12234 module.stop().await.unwrap();
12235
12236 let starts = outage_lines(&capture, OUTAGE_START);
12237 assert_eq!(starts.len(), 2, "{starts:?}");
12238 assert_eq!(starts[1].level, tracing::Level::INFO);
12239 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
12240 }
12241
12242 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12246 async fn failed_operator_restart_leaves_no_operator_mark() {
12247 let registry = Arc::new(Registry::default());
12248 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
12249 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12250 let capture = EventCapture::default();
12251 let _subscriber =
12252 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12253 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
12254 handler.route_outages.record_accepted("outage-disabled");
12255
12256 let restart = handler
12257 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
12258 .await
12259 .unwrap();
12260 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
12261 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
12262
12263 handler
12264 .handle_control_frame(
12265 &ctx,
12266 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
12267 )
12268 .await
12269 .unwrap();
12270 let starts = outage_lines(&capture, OUTAGE_START);
12271 assert_eq!(starts.len(), 1, "{starts:?}");
12272 assert_eq!(starts[0].level, tracing::Level::WARN);
12273 }
12274
12275 #[tokio::test(flavor = "current_thread")]
12276 async fn route_open_unknown_module_escapes_target_module_id() {
12277 let handler = ControlHandler::new(Arc::new(Registry::default()));
12278 let capture = EventCapture::default();
12279 let _subscriber =
12280 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12281 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
12282 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
12283 let response = handler
12284 .handle_control_frame(
12285 &ctx,
12286 route_open_frame(
12287 395,
12288 hostile_module_id,
12289 unique_project_root("hostile-target-module-id"),
12290 ),
12291 )
12292 .await
12293 .unwrap();
12294
12295 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12296 let event = capture
12297 .events()
12298 .into_iter()
12299 .find(|event| {
12300 event.target == "control"
12301 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12302 })
12303 .expect("route.open unknown-module refusal event");
12304 let logged = event.fields.get("module_id").expect("module_id field");
12305 assert!(!logged.bytes().any(|byte| byte < 0x20));
12306 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12307 }
12308
12309 #[tokio::test(flavor = "current_thread")]
12310 async fn route_open_module_rejection_uses_daemon_counter_key() {
12311 let registry = Arc::new(Registry::default());
12312 let forwarding = Arc::new(ForwardingTable::default());
12313 let handler =
12314 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12315 let module_connection = ConnectionId::new(95);
12316 let (module_ctx, mut module_rx) = route_ctx(module_connection);
12317 hello_via_sink(
12318 &handler,
12319 &module_ctx,
12320 &mut module_rx,
12321 hello_frame("aft", PROTOCOL_VERSION, 395),
12322 )
12323 .await;
12324
12325 let client_connection = ConnectionId::new(96);
12326 let (client_ctx, _client_rx) = route_ctx(client_connection);
12327 let capture = EventCapture::default();
12328 let _subscriber =
12329 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12330 let (route_task, bind) = relay_route_open(
12331 &handler,
12332 client_connection,
12333 &client_ctx.egress,
12334 &mut module_rx,
12335 396,
12336 "aft",
12337 "hostile-module-code",
12338 )
12339 .await;
12340 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
12341 let rejection = Frame::build(
12342 FrameType::Error,
12343 control_flags(),
12344 0,
12345 0,
12346 bind.header.corr,
12347 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
12348 )
12349 .unwrap();
12350 handler
12351 .handle_control_frame(&module_ctx, rejection)
12352 .await
12353 .unwrap();
12354
12355 let response = route_task.await.unwrap();
12356 assert_eq!(parse_error(&response[0])["code"], hostile_code);
12357 let counters = handler.counters().snapshot();
12358 assert_eq!(
12359 counters["route_open_refused_by_code"],
12360 json!({ "module_rejected": 1 })
12361 );
12362 assert!(counters["route_open_refused_by_code"]
12363 .get(hostile_code)
12364 .is_none());
12365
12366 let event = capture
12367 .events()
12368 .into_iter()
12369 .find(|event| {
12370 event.target == "control"
12371 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
12372 })
12373 .expect("route.open module-rejection refusal event");
12374 let logged = event.fields.get("module_code").expect("module_code field");
12375 assert!(!logged.bytes().any(|byte| byte < 0x20));
12376 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12377 }
12378
12379 #[tokio::test]
12380 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
12381 let registry = Arc::new(Registry::default());
12382 let supervisor_handle = SupervisorHandle::new();
12383 let missing_program = std::env::temp_dir().join(format!(
12384 "subc-route-open-missing-program-{}",
12385 std::process::id()
12386 ));
12387 let supervisor =
12388 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12389 .with_handle(supervisor_handle.clone());
12390 let module = supervisor
12391 .supervise_configured(
12392 ModuleSpec {
12393 module_id: "failed".to_string(),
12394 program: missing_program,
12395 args: Vec::new(),
12396 env: Vec::new(),
12397 reserved: false,
12398 reserved_prefixes: Vec::new(),
12399 protocol: ModuleProtocol::Subc,
12400 overlap: Default::default(),
12401 },
12402 true,
12403 )
12404 .unwrap();
12405 assert_eq!(module.state().unwrap(), ModuleState::Failed);
12406
12407 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12408 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
12409 let response = handler
12410 .handle_control_frame(
12411 &ctx,
12412 route_open_frame(305, "failed", unique_project_root("failed")),
12413 )
12414 .await
12415 .unwrap();
12416
12417 assert_eq!(response[0].header.ty, FrameType::Error);
12418 let error = parse_error(&response[0]);
12419 assert_eq!(error["code"], "target_unavailable");
12420 assert!(error["message"]
12421 .as_str()
12422 .unwrap()
12423 .contains("state=failed, enabled=true, live=false"));
12424 }
12425
12426 #[tokio::test]
12427 async fn route_open_role_mismatch_remains_target_unavailable() {
12428 let registry = Arc::new(Registry::default());
12429 let handler = ControlHandler::new(Arc::clone(®istry));
12430 handler
12431 .handle_control(
12432 ConnectionId::new(41),
12433 non_routable_hello_frame_with_control_ops("health-only", 306, None),
12434 )
12435 .unwrap();
12436
12437 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
12438 let response = handler
12439 .handle_control_frame(
12440 &ctx,
12441 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
12442 )
12443 .await
12444 .unwrap();
12445
12446 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12447 assert!(parse_error(&response[0])["message"]
12448 .as_str()
12449 .unwrap()
12450 .contains("does not provide the requested target"));
12451 }
12452
12453 #[tokio::test]
12454 async fn route_open_inactive_registration_remains_target_unavailable() {
12455 let registry = Arc::new(Registry::default());
12456 let handler = ControlHandler::new(Arc::clone(®istry));
12457 handler
12458 .handle_control(
12459 ConnectionId::new(43),
12460 hello_frame("inactive", PROTOCOL_VERSION, 308),
12461 )
12462 .unwrap();
12463 assert!(registry
12464 .set_module_state_for_test("inactive", ChannelState::Closed)
12465 .unwrap());
12466
12467 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
12468 let response = handler
12469 .handle_control_frame(
12470 &ctx,
12471 route_open_frame(309, "inactive", unique_project_root("inactive")),
12472 )
12473 .await
12474 .unwrap();
12475
12476 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12477 assert!(parse_error(&response[0])["message"]
12478 .as_str()
12479 .unwrap()
12480 .contains("is not active"));
12481 }
12482
12483 #[tokio::test]
12484 async fn late_health_reply_is_recorded_through_the_module_response_path() {
12485 let registry = Arc::new(Registry::default());
12486 let forwarding = Arc::new(ForwardingTable::default());
12487 let supervisor_handle = SupervisorHandle::new();
12488 let supervisor =
12489 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
12490 .with_forwarding(Arc::clone(&forwarding))
12491 .with_handle(supervisor_handle.clone());
12492 let module = supervisor
12493 .supervise_configured(
12494 crate::ModuleSpec {
12495 module_id: "late-health-response".to_string(),
12496 program: PathBuf::from("disabled-module"),
12497 args: Vec::new(),
12498 env: Vec::new(),
12499 reserved: false,
12500 reserved_prefixes: Vec::new(),
12501 protocol: ModuleProtocol::Subc,
12502 overlap: Default::default(),
12503 },
12504 false,
12505 )
12506 .unwrap();
12507 let handler =
12508 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12509 .with_supervisor(supervisor_handle);
12510 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
12511 handler
12512 .handle_control_frame(
12513 &module_ctx,
12514 hello_frame_with_control_ops(
12515 "late-health-response",
12516 PROTOCOL_VERSION,
12517 7,
12518 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12519 ),
12520 )
12521 .await
12522 .unwrap();
12523 let probe_started_at = Instant::now() - Duration::from_millis(80);
12524 let pending = forwarding
12525 .begin_health_probe_rpc_for(
12526 "late-health-response",
12527 MODULE_CONTROL_OP_HEALTH_CHECK,
12528 probe_started_at,
12529 Instant::now() - Duration::from_millis(1),
12530 )
12531 .unwrap();
12532 assert!(forwarding
12533 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
12534 .unwrap());
12535
12536 let responses = handler
12537 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
12538 .await
12539 .unwrap();
12540
12541 assert!(responses.is_empty());
12542 let health = module.status().unwrap().health;
12543 assert_eq!(health.late_answer_count, 1);
12544 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
12545 }
12546
12547 #[tokio::test]
12548 async fn health_probe_timeout_and_module_death_are_typed() {
12549 let registry = Arc::new(Registry::default());
12550 let forwarding = Arc::new(ForwardingTable::default());
12551 let handler =
12552 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12553 .with_health_probe_timeout(Duration::from_millis(50));
12554 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
12555 hello_via_sink(
12556 &handler,
12557 &module_ctx,
12558 &mut module_rx,
12559 hello_frame_with_control_ops(
12560 "aft",
12561 PROTOCOL_VERSION,
12562 7,
12563 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12564 ),
12565 )
12566 .await;
12567
12568 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12569 let responses = handler
12570 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
12571 .await
12572 .unwrap();
12573 assert_eq!(responses[0].header.ty, FrameType::Error);
12574 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
12575 let _ = module_rx.try_recv();
12576
12577 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
12578 let health_handler = handler.clone();
12579 let death_task = tokio::spawn(async move {
12580 health_handler
12581 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
12582 .await
12583 .unwrap()
12584 });
12585 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12586 .await
12587 .unwrap()
12588 .unwrap();
12589 handler
12590 .cleanup_connection(module_ctx.connection_id)
12591 .unwrap();
12592 let responses = death_task.await.unwrap();
12593 assert_eq!(responses[0].header.ty, FrameType::Error);
12594 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
12595 }
12596
12597 #[test]
12598 fn hello_requires_exact_protocol_version() {
12599 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
12600 let registry = Arc::new(Registry::default());
12601 let handler = ControlHandler::new(Arc::clone(®istry));
12602 let responses = handler
12603 .handle_control(
12604 ConnectionId::new(connection),
12605 hello_frame("aft", offered, 9),
12606 )
12607 .unwrap();
12608
12609 assert_eq!(responses.len(), 1);
12610 assert_eq!(responses[0].header.ty, FrameType::Error);
12611 let error = parse_error(&responses[0]);
12612 assert_eq!(error["code"], "version_unsupported");
12613 assert!(registry.get_module("aft").unwrap().is_none());
12614 assert_eq!(registry.active_registration_count().unwrap(), 0);
12615 }
12616 }
12617
12618 #[test]
12619 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
12620 let registry = Arc::new(Registry::default());
12621 let forwarding = Arc::new(ForwardingTable::default());
12622 let handler =
12623 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12624 let module_connection = ConnectionId::new(301);
12625 let registration = registry
12626 .register_with_control_ops(
12627 manifest("aft-push", PROTOCOL_VERSION),
12628 PROTOCOL_VERSION,
12629 module_connection,
12630 module_baseline_control_ops(),
12631 )
12632 .unwrap();
12633 let (module_tx, _module_rx) = mpsc::channel(8);
12634 let endpoint = forwarding
12635 .register_module_connection(
12636 module_connection,
12637 "aft-push".to_string(),
12638 PROTOCOL_VERSION,
12639 manifest_concurrency(®istration.manifest),
12640 FrameSink::new(module_tx),
12641 )
12642 .unwrap();
12643
12644 let unknown = Frame::build(
12646 FrameType::Push,
12647 control_flags(),
12648 0,
12649 0,
12650 5,
12651 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
12652 )
12653 .unwrap();
12654 let out = handler.handle_status_update(endpoint, unknown).unwrap();
12655 assert!(
12656 out.is_empty(),
12657 "unknown push op must be ignored, got {out:?}"
12658 );
12659
12660 let malformed = Frame::build(
12662 FrameType::Push,
12663 control_flags(),
12664 0,
12665 0,
12666 6,
12667 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
12668 )
12669 .unwrap();
12670 let out = handler.handle_status_update(endpoint, malformed).unwrap();
12671 assert_eq!(out.len(), 1);
12672 assert_eq!(out[0].header.ty, FrameType::Error);
12673 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
12674 }
12675
12676 #[test]
12677 fn hello_rejected_when_connection_already_owns_client_routes() {
12678 let registry = Arc::new(Registry::default());
12679 let forwarding = Arc::new(ForwardingTable::default());
12680 let handler =
12681 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12682 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
12684 let client_connection = ConnectionId::new(202);
12685
12686 let responses = handler
12689 .handle_control(
12690 client_connection,
12691 hello_frame("aft-second", PROTOCOL_VERSION, 9),
12692 )
12693 .unwrap();
12694 assert_eq!(responses[0].header.ty, FrameType::Error);
12695 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
12696 assert!(registry.get_module("aft-second").unwrap().is_none());
12697 }
12698
12699 #[tokio::test]
12700 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
12701 let registry = Arc::new(Registry::default());
12702 let forwarding = Arc::new(ForwardingTable::default());
12703 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
12704 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
12705 hello_via_sink(
12706 &handler,
12707 &module_ctx,
12708 &mut module_rx,
12709 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
12710 )
12711 .await;
12712 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
12713 let pending = forwarding
12714 .begin_route_bind_relay_for_test(
12715 client_ctx.connection_id,
12716 client_ctx.egress.clone(),
12717 2,
12718 "alpha",
12719 )
12720 .unwrap();
12721 forwarding
12722 .complete_pending_relay(
12723 module_ctx.connection_id,
12724 pending.corr,
12725 RouteBindRelayOutcome::Accepted,
12726 )
12727 .unwrap();
12728 client_rx.try_recv().unwrap();
12729 for module_id in ["beta", "alpha"] {
12730 let replies = handler
12731 .handle_control_frame(
12732 &module_ctx,
12733 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
12734 )
12735 .await
12736 .unwrap();
12737 assert_eq!(replies.len(), 1, "second HELLO must be refused");
12738 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
12739 }
12740 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
12741 assert!(registry.get_module("beta").unwrap().is_none());
12742 assert!(matches!(
12743 forwarding
12744 .lookup_data_route(
12745 client_ctx.connection_id,
12746 pending.client_channel,
12747 pending.client_epoch,
12748 )
12749 .unwrap(),
12750 DataRoute::Client(DataRouteState::Bound(_))
12751 ));
12752 assert!(handler
12753 .hello_launch_nonces
12754 .lock()
12755 .unwrap()
12756 .presented(module_ctx.connection_id, Some("alpha-nonce")));
12757 assert!(module_rx.try_recv().is_err());
12758 }
12759
12760 #[test]
12761 fn reserved_module_hello_requires_matching_launch_nonce() {
12762 let registry = Arc::new(Registry::default());
12763 let supervisor = SupervisorHandle::new();
12764 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
12767 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12768
12769 let no_nonce = handler
12771 .handle_control(
12772 ConnectionId::new(1),
12773 hello_frame("vault", PROTOCOL_VERSION, 1),
12774 )
12775 .unwrap();
12776 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
12777 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
12778 assert!(registry.get_module("vault").unwrap().is_none());
12779
12780 let wrong = handler
12782 .handle_control(
12783 ConnectionId::new(2),
12784 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
12785 )
12786 .unwrap();
12787 assert_eq!(wrong[0].header.ty, FrameType::Error);
12788 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
12789 assert!(registry.get_module("vault").unwrap().is_none());
12790
12791 let ok = handler
12793 .handle_control(
12794 ConnectionId::new(3),
12795 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
12796 )
12797 .unwrap();
12798 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
12799 assert!(registry.get_module("vault").unwrap().is_some());
12800 }
12801
12802 #[test]
12803 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
12804 let registry = Arc::new(Registry::default());
12805 let supervisor = SupervisorHandle::new();
12806 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
12807 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
12808 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12809
12810 let squat = handler
12811 .handle_control(
12812 ConnectionId::new(1),
12813 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
12814 )
12815 .unwrap();
12816 assert_eq!(squat[0].header.ty, FrameType::Error);
12817 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
12818 assert!(parse_error(&squat[0])["message"]
12819 .as_str()
12820 .unwrap()
12821 .contains("fed:"));
12822
12823 let accepted_peer = handler
12824 .handle_control(
12825 ConnectionId::new(2),
12826 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
12827 )
12828 .unwrap();
12829 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
12830
12831 let accepted_short = handler
12832 .handle_control(
12833 ConnectionId::new(3),
12834 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
12835 )
12836 .unwrap();
12837 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
12838
12839 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
12840 let response = handler
12841 .handle_control(
12842 ConnectionId::new(conn),
12843 hello_frame(module_id, PROTOCOL_VERSION, conn),
12844 )
12845 .unwrap();
12846 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
12847 }
12848 }
12849
12850 #[test]
12851 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
12852 let registry = Arc::new(Registry::default());
12853 let supervisor = SupervisorHandle::new();
12854 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
12855 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
12856 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
12857 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12858
12859 let owner_nonce = handler
12860 .handle_control(
12861 ConnectionId::new(1),
12862 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
12863 )
12864 .unwrap();
12865 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
12866 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
12867 assert!(registry.get_module("fed:special").unwrap().is_none());
12868
12869 let exact_nonce = handler
12870 .handle_control(
12871 ConnectionId::new(2),
12872 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
12873 )
12874 .unwrap();
12875 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
12876 assert!(registry.get_module("fed:special").unwrap().is_some());
12877 }
12878
12879 #[test]
12880 fn non_reserved_module_ignores_launch_nonce() {
12881 let registry = Arc::new(Registry::default());
12882 let handler = ControlHandler::new(Arc::clone(®istry));
12885 let no_nonce = handler
12886 .handle_control(
12887 ConnectionId::new(1),
12888 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
12889 )
12890 .unwrap();
12891 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
12892 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
12893
12894 let echoed_nonce = handler
12895 .handle_control(
12896 ConnectionId::new(2),
12897 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
12898 )
12899 .unwrap();
12900 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
12901 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
12902 }
12903
12904 #[test]
12905 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
12906 let handler = ControlHandler::default();
12907 let conn = ConnectionId::new(1);
12908 let malformed = Frame::build(
12909 FrameType::Hello,
12910 control_flags(),
12911 0,
12912 0,
12913 3,
12914 b"{not json".to_vec(),
12915 )
12916 .unwrap();
12917
12918 let error = handler.handle_control(conn, malformed).unwrap();
12919 assert_eq!(error[0].header.ty, FrameType::Error);
12920 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
12921
12922 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
12923 let pong = handler.handle_control(conn, ping).unwrap();
12924 assert_eq!(pong[0].header.ty, FrameType::Pong);
12925 assert_eq!(pong[0].header.corr, 4);
12926 }
12927
12928 #[test]
12929 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
12930 let registry = Arc::new(Registry::default());
12931 let handler = ControlHandler::new(Arc::clone(®istry));
12932
12933 handler
12934 .handle_control(
12935 ConnectionId::new(1),
12936 hello_frame("aft", PROTOCOL_VERSION, 1),
12937 )
12938 .unwrap();
12939 let duplicate = handler
12940 .handle_control(
12941 ConnectionId::new(2),
12942 hello_frame("aft", PROTOCOL_VERSION, 2),
12943 )
12944 .unwrap();
12945
12946 assert_eq!(duplicate[0].header.ty, FrameType::Error);
12947 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
12948 let registration = registry.get_module("aft").unwrap().unwrap();
12949 assert_eq!(registration.connection_id, ConnectionId::new(1));
12950 }
12951
12952 #[test]
12953 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
12954 let registry = Arc::new(Registry::default());
12955 let forwarding = Arc::new(ForwardingTable::default());
12956 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
12957 let handler =
12958 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12959 .with_process_liveness(process_liveness);
12960 let (ctx, route_channel, route_epoch) =
12961 bind_liveness_route(®istry, &forwarding, "aft-dead");
12962 let responses = handler
12963 .handle_route_poll(
12964 &ctx,
12965 route_poll_frame(41, PollKind::Liveness, route_channel),
12966 route_channel,
12967 route_epoch,
12968 PollKind::Liveness,
12969 )
12970 .unwrap();
12971
12972 assert_eq!(responses.len(), 1);
12973 assert_eq!(responses[0].header.ty, FrameType::Response);
12974 assert_route_poll_liveness(&responses[0], false);
12975 }
12976
12977 #[test]
12978 fn liveness_poll_without_process_source_uses_bound_route() {
12979 let registry = Arc::new(Registry::default());
12980 let forwarding = Arc::new(ForwardingTable::default());
12981 let handler =
12982 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12983 let (ctx, route_channel, route_epoch) =
12984 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
12985 let responses = handler
12986 .handle_route_poll(
12987 &ctx,
12988 route_poll_frame(42, PollKind::Liveness, route_channel),
12989 route_channel,
12990 route_epoch,
12991 PollKind::Liveness,
12992 )
12993 .unwrap();
12994
12995 assert_route_poll_liveness(&responses[0], true);
12996 }
12997
12998 #[test]
12999 fn liveness_poll_untracked_process_source_uses_bound_route() {
13000 let registry = Arc::new(Registry::default());
13001 let forwarding = Arc::new(ForwardingTable::default());
13002 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
13003 let handler =
13004 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13005 .with_process_liveness(process_liveness);
13006 let (ctx, route_channel, route_epoch) =
13007 bind_liveness_route(®istry, &forwarding, "aft-untracked");
13008 let responses = handler
13009 .handle_route_poll(
13010 &ctx,
13011 route_poll_frame(43, PollKind::Liveness, route_channel),
13012 route_channel,
13013 route_epoch,
13014 PollKind::Liveness,
13015 )
13016 .unwrap();
13017
13018 assert_route_poll_liveness(&responses[0], true);
13019 }
13020
13021 #[tokio::test]
13022 async fn unknown_op_returns_unknown_control_op() {
13023 let handler = ControlHandler::default();
13024 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
13025 let request = Frame::build(
13026 FrameType::Request,
13027 control_flags(),
13028 0,
13029 0,
13030 55,
13031 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
13032 )
13033 .unwrap();
13034
13035 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13036
13037 assert_eq!(response.len(), 1);
13038 assert_eq!(response[0].header.ty, FrameType::Error);
13039 assert_eq!(response[0].header.corr, 55);
13040 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
13041 }
13042
13043 #[tokio::test]
13044 async fn supervisor_provenance_rejects_unknown_exact_module() {
13045 let handler = ControlHandler::default();
13046 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
13047 let request = Frame::build(
13048 FrameType::Request,
13049 control_flags(),
13050 0,
13051 0,
13052 57,
13053 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
13054 )
13055 .unwrap();
13056
13057 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13058
13059 assert_eq!(response.len(), 1);
13060 assert_eq!(response[0].header.ty, FrameType::Error);
13061 assert_eq!(response[0].header.corr, 57);
13062 let error = parse_error(&response[0]);
13063 assert_eq!(error["code"], "unknown_module");
13064 assert_eq!(error["message"], "module_id 'missing' is not supervised");
13065 }
13066
13067 #[test]
13068 fn provenance_probe_override_keeps_handler_tests_deterministic() {
13069 let expected = subc_control::RunningImageAgreement::Unavailable {
13070 reason: subc_control::RunningImageUnavailableReason::HashFailed,
13071 };
13072 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
13073 assert_eq!(handler.provenance_probe_override, Some(expected));
13074 }
13075
13076 #[test]
13077 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
13078 let verdict = reload_verdict(
13079 std::path::Path::new("/bin/new"),
13080 Some(std::path::Path::new("/bin/old")),
13081 subc_control::RunningImageAgreement::Unavailable {
13082 reason: subc_control::RunningImageUnavailableReason::HashFailed,
13083 },
13084 );
13085 assert!(matches!(
13086 verdict.path,
13087 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
13088 if configured == std::path::Path::new("/bin/new")
13089 && spawned_from == std::path::Path::new("/bin/old")
13090 ));
13091 }
13092
13093 #[test]
13094 fn launch_spec_without_running_digest_is_unavailable_not_match() {
13095 use subc_control::{
13096 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
13097 };
13098 let configured = crate::supervise::ModuleSpec {
13099 module_id: "legacy".into(),
13100 program: "/module".into(),
13101 args: Vec::new(),
13102 env: Vec::new(),
13103 reserved: false,
13104 reserved_prefixes: Vec::new(),
13105 protocol: subc_control::ModuleProtocol::Subc,
13106 overlap: Default::default(),
13107 };
13108 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
13109 assert_eq!(
13110 launch_spec_agreement(&configured, false, Some(&digest)),
13111 Agreement::Unavailable {
13112 reason: Reason::NotRunning
13113 }
13114 );
13115 assert_eq!(
13116 launch_spec_agreement(&configured, true, None),
13117 Agreement::Unavailable {
13118 reason: Reason::SpawnedSpecUnavailable
13119 }
13120 );
13121 assert_eq!(
13122 launch_spec_agreement(&configured, true, Some(&digest)),
13123 Agreement::Match
13124 );
13125 }
13126
13127 #[cfg(windows)]
13128 #[test]
13129 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
13130 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
13131 let configured = dir.join("module.exe");
13132 std::fs::write(&configured, b"executable path fixture").unwrap();
13133 let spawned = std::fs::canonicalize(&configured).unwrap();
13134 assert_ne!(
13135 configured, spawned,
13136 "fixture must exercise different path spellings"
13137 );
13138 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
13139 let lower_case = configured.to_string_lossy().to_lowercase();
13140 for spelling in [
13141 &configured,
13142 Path::new(&forward_slashes),
13143 Path::new(&lower_case),
13144 ] {
13145 let verdict = reload_verdict(
13146 spelling,
13147 Some(&spawned),
13148 subc_control::RunningImageAgreement::Unavailable {
13149 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13150 },
13151 );
13152 assert_eq!(
13153 verdict.path,
13154 ReloadPathAgreement::Match,
13155 "configured {spelling:?}, spawned {spawned:?}"
13156 );
13157 }
13158 }
13159
13160 #[cfg(windows)]
13161 #[test]
13162 fn windows_reload_path_keeps_different_existing_program_pending() {
13163 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
13164 let old = dir.join("old.exe");
13165 let new = dir.join("new.exe");
13166 std::fs::write(&old, b"same executable content").unwrap();
13167 std::fs::copy(&old, &new).unwrap();
13168 let spawned = std::fs::canonicalize(&old).unwrap();
13169 let image = subc_control::RunningImageAgreement::Unavailable {
13170 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13171 };
13172 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
13173 assert!(
13174 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
13175 if configured == new && spawned_from == spawned)
13176 );
13177 let missing = dir.join("missing.exe");
13178 assert!(matches!(
13179 reload_verdict(&missing, Some(&spawned), image).path,
13180 ReloadPathAgreement::Mismatch { .. }
13181 ));
13182 }
13183
13184 #[test]
13185 fn reload_verdict_detects_replaced_image_at_same_path() {
13186 let image = subc_control::RunningImageAgreement::Mismatch {
13187 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
13188 digest: "old".into(),
13189 },
13190 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
13191 digest: "new".into(),
13192 },
13193 };
13194 let verdict = reload_verdict(
13195 std::path::Path::new("/bin/same"),
13196 Some(std::path::Path::new("/bin/same")),
13197 image.clone(),
13198 );
13199 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13200 assert_eq!(verdict.image, image);
13201 }
13202
13203 #[test]
13204 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
13205 let image = subc_control::RunningImageAgreement::Unavailable {
13206 reason: subc_control::RunningImageUnavailableReason::NotRunning,
13207 };
13208 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
13209 assert_eq!(
13210 verdict.path,
13211 subc_control::ReloadPathAgreement::Unavailable {
13212 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
13213 }
13214 );
13215 assert_eq!(verdict.image, image);
13216
13217 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
13218 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13219 };
13220 let verdict = reload_verdict(
13221 std::path::Path::new("/bin/same"),
13222 Some(std::path::Path::new("/bin/same")),
13223 unconfirmed.clone(),
13224 );
13225 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13226 assert_eq!(verdict.image, unconfirmed);
13227 }
13228
13229 #[test]
13230 fn reload_verdict_preserves_each_image_unavailability_reason() {
13231 use subc_control::RunningImageUnavailableReason as Reason;
13232
13233 for reason in [
13234 Reason::NotRunning,
13235 Reason::UnsupportedPlatform,
13236 Reason::RunningExecutableUnreadable,
13237 Reason::SpawnedPathUnreadable,
13238 Reason::HashFailed,
13239 Reason::ProcessIdentityUnconfirmed,
13240 Reason::Unknown("future_probe_reason".to_string()),
13241 ] {
13242 let image = subc_control::RunningImageAgreement::Unavailable {
13243 reason: reason.clone(),
13244 };
13245 let verdict = reload_verdict(
13246 std::path::Path::new("/bin/same"),
13247 Some(std::path::Path::new("/bin/same")),
13248 image.clone(),
13249 );
13250 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13251 assert_eq!(verdict.image, image, "{reason:?}");
13252 }
13253 }
13254
13255 #[tokio::test]
13256 async fn malformed_control_bodies_return_invalid_control_body() {
13257 let handler = ControlHandler::default();
13258 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
13259
13260 for (corr, body) in [
13261 (56, br#"{"route_channel":1}"#.as_slice()),
13262 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
13263 (
13264 58,
13265 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
13266 ),
13267 ] {
13268 let request = Frame::build(
13269 FrameType::Request,
13270 control_flags(),
13271 0,
13272 0,
13273 corr,
13274 body.to_vec(),
13275 )
13276 .unwrap();
13277 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13278
13279 assert_eq!(response.len(), 1);
13280 assert_eq!(response[0].header.ty, FrameType::Error);
13281 assert_eq!(response[0].header.corr, corr);
13282 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
13283 }
13284 }
13285
13286 #[tokio::test]
13287 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
13288 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13289 let registry = Arc::new(Registry::default());
13290 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
13291 let router = Router::with_control_handler(Arc::clone(&control));
13292 let connection = router.begin_connection();
13293 let (ctx, mut rx) = route_ctx(connection.id());
13294
13295 router
13296 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
13297 .await
13298 .unwrap();
13299 let response = rx.recv().await.unwrap();
13300 let ack = parse_ack(&response);
13301 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13302 let channel = 1;
13303
13304 let goodbye =
13305 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
13306 router.route_for_connection(&ctx, goodbye).await.unwrap();
13307 assert!(rx.try_recv().is_err());
13308 assert!(registry.get_module("aft").unwrap().is_none());
13309
13310 router
13311 .route_for_connection(&ctx, channel_request(channel, 13))
13312 .await
13313 .unwrap();
13314 let error_frame = rx.recv().await.unwrap();
13315 assert_eq!(error_frame.header.ty, FrameType::Error);
13316 assert_eq!(error_frame.header.channel, channel);
13317 let captured = crate::router::test_log::captured_logs(&logs);
13318 assert_eq!(
13319 captured
13320 .lines()
13321 .filter(|line| {
13322 line.contains("module_id=aft")
13323 && line.contains("reason=explicit_goodbye")
13324 && line.contains("module registration ended")
13325 })
13326 .count(),
13327 1,
13328 "unexpected GOODBYE registry log: {captured}"
13329 );
13330 }
13331
13332 #[tokio::test]
13333 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
13334 let registry = Arc::new(Registry::default());
13335 let handler = ControlHandler::new(registry).with_capability_config(
13336 [("prov".to_string(), true), ("cons".to_string(), true)],
13337 BTreeMap::new(),
13338 );
13339 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
13340 register_capability_manifest(
13341 &handler,
13342 &provider_ctx,
13343 &mut provider_rx,
13344 capability_manifest("prov", &["thing/v1"], &[]),
13345 1,
13346 )
13347 .await;
13348 let mut consumer = capability_manifest("cons", &[], &[]);
13349 consumer.capabilities.as_mut().unwrap().requires.push(
13350 subc_protocol::manifest::CapabilityRequirement {
13351 capability: "thing/v1".to_string(),
13352 need: subc_protocol::manifest::CapabilityNeed::Required,
13353 },
13354 );
13355 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
13356 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
13357 assert_eq!(
13358 handler.capability_evaluator.verdict("cons", "thing/v1"),
13359 Some(CapabilityVerdict::Provided)
13360 );
13361 let (mut client_rx, _) = open_route_for_capability_test(
13362 &handler,
13363 &provider_ctx,
13364 &mut provider_rx,
13365 703,
13366 3,
13367 "prov",
13368 None,
13369 )
13370 .await;
13371 let goodbye =
13372 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13373 handler
13374 .handle_control_frame(&provider_ctx, goodbye)
13375 .await
13376 .unwrap();
13377 assert_eq!(
13378 handler.capability_evaluator.verdict("cons", "thing/v1"),
13379 Some(CapabilityVerdict::NeverProvided)
13380 );
13381 let closed = client_rx
13382 .try_recv()
13383 .expect("GOODBYE pushes route.closed before route GOODBYE");
13384 assert!(
13385 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
13386 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
13387 );
13388 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
13389 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
13390 }
13391
13392 struct RouteClosedFixture {
13393 handler: ControlHandler,
13394 #[cfg(unix)]
13395 supervisor: Supervisor,
13396 module_ctx: RouteCtx,
13397 #[cfg(unix)]
13398 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13399 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13400 route: Arc<crate::forwarding::RouteBinding>,
13401 }
13402
13403 async fn route_closed_fixture() -> RouteClosedFixture {
13404 let registry = Arc::new(Registry::default());
13405 let handler = ControlHandler::new(Arc::clone(®istry));
13406 #[cfg(unix)]
13407 let supervisor = Supervisor::new(registry, RestartPolicy::default())
13408 .with_forwarding(Arc::clone(&handler.forwarding));
13409 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
13410 register_capability_manifest(
13411 &handler,
13412 &module_ctx,
13413 &mut module_rx,
13414 capability_manifest("prov", &[], &[]),
13415 1,
13416 )
13417 .await;
13418 let (client_rx, bind) = open_route_for_capability_test(
13419 &handler,
13420 &module_ctx,
13421 &mut module_rx,
13422 702,
13423 2,
13424 "prov",
13425 None,
13426 )
13427 .await;
13428 let ModuleControlRequest::RouteBind {
13429 route_channel: channel,
13430 epoch: route_epoch,
13431 ..
13432 } = bind
13433 else {
13434 panic!("expected route.bind");
13435 };
13436 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
13437 handler
13438 .forwarding
13439 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
13440 .unwrap()
13441 else {
13442 panic!("the acknowledged route must be bound");
13443 };
13444 RouteClosedFixture {
13445 handler,
13446 #[cfg(unix)]
13447 supervisor,
13448 module_ctx,
13449 #[cfg(unix)]
13450 module_rx,
13451 client_rx,
13452 route,
13453 }
13454 }
13455
13456 impl RouteClosedFixture {
13457 #[cfg(unix)]
13458 async fn cut(&mut self) {
13459 let notices = async {
13460 let mut draining = self.module_rx.recv().await.unwrap();
13461 assert!(matches!(
13462 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
13463 &draining.body
13464 )
13465 .unwrap(),
13466 subc_protocol::session::ModuleControlCommand::Draining {
13467 reason: RouteCloseReason::Restart,
13468 ..
13469 }
13470 ));
13471 draining.flushed.take().unwrap().send(()).unwrap();
13472 let mut closing = self.client_rx.recv().await.unwrap();
13473 assert!(matches!(
13474 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
13475 ClientControlPush::RouteClosing {
13476 reason: RouteCloseReason::Restart,
13477 ..
13478 }
13479 ));
13480 closing.flushed.take().unwrap().send(()).unwrap();
13481 };
13482 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
13483 result.unwrap();
13484 }
13485
13486 fn disconnect(&mut self) -> ClientControlPush {
13487 self.handler
13488 .cleanup_connection(self.module_ctx.connection_id)
13489 .unwrap();
13490 let closed = self.client_rx.try_recv().unwrap();
13491 assert_eq!(closed.header.ty, FrameType::Push);
13492 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
13493 assert!(matches!(
13494 &push,
13495 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
13496 if channels == &[self.route.client_channel]
13497 ));
13498 assert_eq!(
13499 self.client_rx.try_recv().unwrap().header.ty,
13500 FrameType::Goodbye
13501 );
13502 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
13503 push
13504 }
13505 }
13506
13507 #[cfg(unix)]
13508 #[tokio::test(start_paused = true)]
13509 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
13510 let mut fixture = route_closed_fixture().await;
13511 fixture.cut().await;
13512 assert!(matches!(
13513 fixture.disconnect(),
13514 ClientControlPush::RouteClosed {
13515 reason: RouteCloseReason::Restart,
13516 drained: true,
13517 ..
13518 }
13519 ));
13520 }
13521
13522 #[cfg(unix)]
13523 #[tokio::test(start_paused = true)]
13524 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
13525 let mut fixture = route_closed_fixture().await;
13526 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
13527 let started = tokio::time::Instant::now();
13528 fixture.cut().await;
13529 assert!(started.elapsed() >= Duration::from_secs(2));
13530 assert_eq!(fixture.route.flow.in_flight(), 1);
13531 assert!(matches!(
13532 fixture.disconnect(),
13533 ClientControlPush::RouteClosed {
13534 reason: RouteCloseReason::Restart,
13535 drained: false,
13536 ..
13537 }
13538 ));
13539 }
13540
13541 #[tokio::test]
13542 async fn module_connection_crash_reports_crash_and_not_drained() {
13543 let mut fixture = route_closed_fixture().await;
13544 assert!(matches!(
13548 fixture.disconnect(),
13549 ClientControlPush::RouteClosed {
13550 reason: RouteCloseReason::Crash,
13551 drained: false,
13552 ..
13553 }
13554 ));
13555 }
13556
13557 #[tokio::test]
13558 async fn dropping_router_connection_releases_registration() {
13559 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13560 let registry = Arc::new(Registry::default());
13561 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
13562 let router = Router::with_control_handler(Arc::clone(&control));
13563 let connection = router.begin_connection();
13564 let connection_id = connection.id();
13565 let (ctx, mut rx) = route_ctx(connection_id);
13566
13567 router
13568 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
13569 .await
13570 .unwrap();
13571 let response = rx.recv().await.unwrap();
13572 let ack = parse_ack(&response);
13573 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13574 assert!(registry.get_module("aft").unwrap().is_some());
13575
13576 drop(connection);
13577
13578 assert!(registry.get_module("aft").unwrap().is_none());
13579 assert_eq!(registry.active_registration_count().unwrap(), 0);
13580
13581 control
13582 .cleanup_connection(ConnectionId::new(u64::MAX))
13583 .unwrap();
13584 let captured = crate::router::test_log::captured_logs(&logs);
13585 println!("captured registry lifecycle logs:\n{captured}");
13586 let events: Vec<_> = captured
13587 .lines()
13588 .filter(|line| {
13589 line.contains("module registered module_id=aft ")
13590 || line.contains("module registration ended")
13591 })
13592 .collect();
13593 assert_eq!(
13594 events.len(),
13595 2,
13596 "unexpected registry lifecycle logs: {captured}"
13597 );
13598 assert!(events[0].contains("module registered module_id=aft "));
13599 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
13600 assert!(events[1].contains(&format!(
13601 "module_id=aft connection_id={}",
13602 connection_id.get()
13603 )));
13604 assert!(events[1].contains("reason=connection_closed"));
13605 assert!(events[1].contains("module registration ended"));
13606 assert_eq!(
13607 captured
13608 .lines()
13609 .filter(|line| line.contains("module registered module_id=aft "))
13610 .count(),
13611 1,
13612 "legacy registration admission line must appear once: {captured}"
13613 );
13614 }
13615
13616 #[tokio::test]
13617 async fn hello_registration_keeps_legacy_module_registered_line_once() {
13618 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13619 let registry = Arc::new(Registry::default());
13620 let control = ControlHandler::new(Arc::clone(®istry));
13621 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
13622 hello_via_sink(
13623 &control,
13624 &ctx,
13625 &mut rx,
13626 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
13627 )
13628 .await;
13629
13630 let captured = crate::router::test_log::captured_logs(&logs);
13631 let admissions: Vec<_> = captured
13632 .lines()
13633 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
13634 .collect();
13635 assert_eq!(
13636 admissions.len(),
13637 1,
13638 "legacy admission line must remain exactly once: {captured}"
13639 );
13640 assert!(admissions[0].contains("routable_provider=true"));
13641 assert!(admissions[0].contains("connection_id=777"));
13642 }
13643
13644 fn capability_manifest(
13645 module_id: &str,
13646 provides: &[&str],
13647 must_never_reach: &[&str],
13648 ) -> ModuleManifest {
13649 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
13650 manifest.capabilities = Some(CapabilityDeclarations {
13651 provides: provides
13652 .iter()
13653 .map(|capability| (*capability).to_string())
13654 .collect(),
13655 requires: Vec::new(),
13656 must_never_reach: must_never_reach
13657 .iter()
13658 .map(|capability| (*capability).to_string())
13659 .collect(),
13660 });
13661 manifest
13662 }
13663
13664 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
13665 Frame::build(
13666 FrameType::Hello,
13667 control_flags(),
13668 0,
13669 0,
13670 corr,
13671 serde_json::to_vec(&ModuleHelloBody {
13672 protocol_ver: manifest.protocol_ver,
13673 manifest,
13674 control_ops: None,
13675 launch_nonce: None,
13676 })
13677 .expect("capability test HELLO serializes"),
13678 )
13679 .expect("capability test HELLO frame builds")
13680 }
13681
13682 fn catalog_update_with_capabilities_frame(
13683 corr: u64,
13684 capabilities: CapabilityDeclarations,
13685 ) -> Frame {
13686 Frame::build(
13687 FrameType::Request,
13688 control_flags(),
13689 0,
13690 0,
13691 corr,
13692 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
13693 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
13694 capabilities: Some(capabilities),
13695 ready: None,
13696 })
13697 .expect("capability catalog.update serializes"),
13698 )
13699 .expect("capability catalog.update frame builds")
13700 }
13701
13702 async fn register_capability_manifest(
13703 handler: &ControlHandler,
13704 ctx: &RouteCtx,
13705 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13706 manifest: ModuleManifest,
13707 corr: u64,
13708 ) {
13709 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
13710 }
13711
13712 async fn open_route_for_capability_test(
13713 handler: &ControlHandler,
13714 target_ctx: &RouteCtx,
13715 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13716 client_connection_id: u64,
13717 corr: u64,
13718 target_module_id: &str,
13719 consumer_identity: Option<ConsumerIdentity>,
13720 ) -> (
13721 mpsc::Receiver<crate::router::OutboundFrame>,
13722 ModuleControlRequest,
13723 ) {
13724 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
13725 let route_handler = handler.clone();
13726 let target_module_id = target_module_id.to_string();
13727 let route_task = tokio::spawn(async move {
13728 route_handler
13729 .handle_control_frame(
13730 &client_ctx,
13731 route_open_frame_with_admission_facts(
13732 corr,
13733 &target_module_id,
13734 unique_project_root("admission-facts"),
13735 consumer_identity,
13736 None,
13737 ),
13738 )
13739 .await
13740 .expect("capability test route.open succeeds")
13741 });
13742 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
13743 .await
13744 .expect("capability test route.open must reach route.bind")
13745 .expect("target control receiver stays open");
13746 let bind_request: ModuleControlRequest =
13747 serde_json::from_slice(&bind.body).expect("route.bind decodes");
13748 handler
13749 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
13750 .await
13751 .expect("capability test route.bind ACK succeeds");
13752 assert!(route_task.await.expect("route.open task joins").is_empty());
13753 let opened = client_rx
13754 .recv()
13755 .await
13756 .expect("successful route.open publishes a response");
13757 assert!(matches!(
13758 serde_json::from_slice::<ClientControlResponse>(&opened.body),
13759 Ok(ClientControlResponse::RouteOpen { .. })
13760 ));
13761 (client_rx, bind_request)
13762 }
13763
13764 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
13765 assert_eq!(frame.header.ty, FrameType::Push);
13766 assert_eq!(frame.header.channel, 0);
13767 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
13768 .expect("route.closed control push decodes");
13769 let ClientControlPush::RouteClosed { channels, .. } = &push else {
13770 panic!("expected route.closed");
13771 };
13772 assert_eq!(channels.len(), 1, "exactly one violating route closed");
13773 let channels = channels.clone();
13774 assert_eq!(
13775 push,
13776 ClientControlPush::RouteClosed {
13777 module_id: target_module_id.to_string(),
13778 channels,
13779 reason: RouteCloseReason::CapabilityDenied,
13780 drained: false,
13781 abandoned: 0,
13782 excluded_subscriptions: 0,
13783 terminal: Some(false),
13784 }
13785 );
13786 }
13787
13788 #[tokio::test]
13789 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
13790 let registry = Arc::new(Registry::default());
13791 let forwarding = Arc::new(ForwardingTable::default());
13792 let supervisor = SupervisorHandle::new();
13793 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13794 let handler =
13795 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13796 .with_supervisor(supervisor);
13797 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
13798 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
13799 register_capability_manifest(
13800 &handler,
13801 &target_ctx,
13802 &mut target_rx,
13803 capability_manifest("target", &["credentials-provider/v1"], &[]),
13804 1,
13805 )
13806 .await;
13807 register_capability_manifest(
13808 &handler,
13809 &opener_ctx,
13810 &mut opener_rx,
13811 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13812 2,
13813 )
13814 .await;
13815
13816 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
13817 let replies = handler
13818 .handle_control_frame(
13819 &client_ctx,
13820 route_open_frame_with_admission_facts(
13821 3,
13822 "target",
13823 unique_project_root("admission-facts"),
13824 Some(ConsumerIdentity {
13825 module_id: "opener".to_string(),
13826 launch_nonce: "opener-nonce".to_string(),
13827 }),
13828 None,
13829 ),
13830 )
13831 .await
13832 .expect("denied route.open returns a typed frame");
13833 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
13834 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13835 assert!(
13836 target_rx.try_recv().is_err(),
13837 "forbidden route.open must not relay route.bind"
13838 );
13839 }
13840
13841 #[tokio::test]
13842 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
13843 let registry = Arc::new(Registry::default());
13844 let forwarding = Arc::new(ForwardingTable::default());
13845 let supervisor = SupervisorHandle::new();
13846 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13847 let handler =
13848 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13849 .with_supervisor(supervisor);
13850 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
13851 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
13852 register_capability_manifest(
13853 &handler,
13854 &target_ctx,
13855 &mut target_rx,
13856 capability_manifest("target", &["credentials-provider/v1"], &[]),
13857 1,
13858 )
13859 .await;
13860 register_capability_manifest(
13861 &handler,
13862 &old_opener_ctx,
13863 &mut old_opener_rx,
13864 capability_manifest("opener", &[], &[]),
13865 2,
13866 )
13867 .await;
13868 let (mut client_rx, _) = open_route_for_capability_test(
13869 &handler,
13870 &target_ctx,
13871 &mut target_rx,
13872 712,
13873 3,
13874 "target",
13875 Some(ConsumerIdentity {
13876 module_id: "opener".to_string(),
13877 launch_nonce: "opener-nonce".to_string(),
13878 }),
13879 )
13880 .await;
13881 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13882
13883 handler
13884 .cleanup_connection(old_opener_ctx.connection_id)
13885 .expect("old opener registration cleans up");
13886 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
13887 register_capability_manifest(
13888 &handler,
13889 &new_opener_ctx,
13890 &mut new_opener_rx,
13891 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13892 4,
13893 )
13894 .await;
13895
13896 assert_capability_denied_push(
13897 client_rx
13898 .try_recv()
13899 .expect("HELLO deny addition must emit route.closed")
13900 .frame,
13901 "target",
13902 );
13903 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13904 assert!(matches!(
13905 target_rx.try_recv(),
13906 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
13907 ));
13908 }
13909
13910 #[tokio::test]
13911 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
13912 let registry = Arc::new(Registry::default());
13913 let forwarding = Arc::new(ForwardingTable::default());
13914 let supervisor = SupervisorHandle::new();
13915 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13916 let handler =
13917 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13918 .with_supervisor(supervisor);
13919 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
13920 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
13921 register_capability_manifest(
13922 &handler,
13923 &target_ctx,
13924 &mut target_rx,
13925 capability_manifest("target", &[], &[]),
13926 1,
13927 )
13928 .await;
13929 register_capability_manifest(
13930 &handler,
13931 &opener_ctx,
13932 &mut opener_rx,
13933 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13934 2,
13935 )
13936 .await;
13937 let (mut client_rx, _) = open_route_for_capability_test(
13938 &handler,
13939 &target_ctx,
13940 &mut target_rx,
13941 722,
13942 3,
13943 "target",
13944 Some(ConsumerIdentity {
13945 module_id: "opener".to_string(),
13946 launch_nonce: "opener-nonce".to_string(),
13947 }),
13948 )
13949 .await;
13950 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13951
13952 let replies = handler
13953 .handle_control_frame(
13954 &target_ctx,
13955 catalog_update_with_capabilities_frame(
13956 4,
13957 CapabilityDeclarations {
13958 provides: vec!["credentials-provider/v1".to_string()],
13959 requires: Vec::new(),
13960 must_never_reach: Vec::new(),
13961 },
13962 ),
13963 )
13964 .await
13965 .expect("claim catalog.update succeeds");
13966 assert!(matches!(
13967 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
13968 Ok(ModuleControlResponseToModule::CatalogUpdate {})
13969 ));
13970 assert_capability_denied_push(
13971 client_rx
13972 .try_recv()
13973 .expect("claim addition must emit route.closed")
13974 .frame,
13975 "target",
13976 );
13977 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13978 assert!(matches!(
13979 target_rx.try_recv(),
13980 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
13981 ));
13982 }
13983
13984 #[tokio::test]
13985 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
13986 let registry = Arc::new(Registry::default());
13987 let forwarding = Arc::new(ForwardingTable::default());
13988 let supervisor = SupervisorHandle::new();
13989 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13990 let handler =
13991 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13992 .with_supervisor(supervisor);
13993 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
13994 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
13995 register_capability_manifest(
13996 &handler,
13997 &target_ctx,
13998 &mut target_rx,
13999 capability_manifest("target", &["credentials-provider/v1"], &[]),
14000 1,
14001 )
14002 .await;
14003 register_capability_manifest(
14004 &handler,
14005 &opener_ctx,
14006 &mut opener_rx,
14007 capability_manifest("opener", &[], &[]),
14008 2,
14009 )
14010 .await;
14011 let (mut client_rx, _) = open_route_for_capability_test(
14012 &handler,
14013 &target_ctx,
14014 &mut target_rx,
14015 732,
14016 3,
14017 "target",
14018 Some(ConsumerIdentity {
14019 module_id: "opener".to_string(),
14020 launch_nonce: "opener-nonce".to_string(),
14021 }),
14022 )
14023 .await;
14024 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14025
14026 handler
14027 .handle_control_frame(
14028 &target_ctx,
14029 catalog_update_with_capabilities_frame(
14030 4,
14031 CapabilityDeclarations {
14032 provides: Vec::new(),
14033 requires: Vec::new(),
14034 must_never_reach: Vec::new(),
14035 },
14036 ),
14037 )
14038 .await
14039 .expect("claim removal catalog.update succeeds");
14040 assert_eq!(
14041 forwarding.active_binding_count().unwrap(),
14042 1,
14043 "removing an attested target claim must leave the route census unchanged"
14044 );
14045 assert!(
14046 client_rx.try_recv().is_err(),
14047 "claim removal must not emit route.closed capability_denied"
14048 );
14049 assert!(
14050 target_rx.try_recv().is_err(),
14051 "claim removal must not send the target a route GOODBYE"
14052 );
14053 }
14054
14055 #[tokio::test]
14058 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
14059 let registry = Arc::new(Registry::default());
14060 let forwarding = Arc::new(ForwardingTable::default());
14061 let supervisor = SupervisorHandle::new();
14062 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14063 let handler =
14064 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14065 .with_supervisor(supervisor);
14066 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
14067 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
14068 register_capability_manifest(
14069 &handler,
14070 &target_ctx,
14071 &mut target_rx,
14072 capability_manifest("target", &["credentials-provider/v1"], &[]),
14073 1,
14074 )
14075 .await;
14076 register_capability_manifest(
14077 &handler,
14078 &opener_ctx,
14079 &mut opener_rx,
14080 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14081 2,
14082 )
14083 .await;
14084
14085 let (_client_rx, bind) = open_route_for_capability_test(
14086 &handler,
14087 &target_ctx,
14088 &mut target_rx,
14089 742,
14090 3,
14091 "target",
14092 None,
14093 )
14094 .await;
14095 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
14096 panic!("direct scope-honesty route must bind");
14097 };
14098 assert_eq!(principal, Some(Principal::Direct));
14099 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14100 }
14101
14102 #[tokio::test]
14105 async fn must_never_reach_self_route_is_capability_forbidden() {
14106 let registry = Arc::new(Registry::default());
14107 let forwarding = Arc::new(ForwardingTable::default());
14108 let supervisor = SupervisorHandle::new();
14109 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
14110 let handler =
14111 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14112 .with_supervisor(supervisor);
14113 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
14114 register_capability_manifest(
14115 &handler,
14116 &self_ctx,
14117 &mut self_rx,
14118 capability_manifest(
14119 "self-provider",
14120 &["credentials-provider/v1"],
14121 &["credentials-provider/v1"],
14122 ),
14123 1,
14124 )
14125 .await;
14126
14127 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
14128 let replies = handler
14129 .handle_control_frame(
14130 &client_ctx,
14131 route_open_frame_with_admission_facts(
14132 2,
14133 "self-provider",
14134 unique_project_root("admission-facts"),
14135 Some(ConsumerIdentity {
14136 module_id: "self-provider".to_string(),
14137 launch_nonce: "self-nonce".to_string(),
14138 }),
14139 None,
14140 ),
14141 )
14142 .await
14143 .expect("self-route refusal returns a typed frame");
14144 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14145 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14146 assert!(
14147 self_rx.try_recv().is_err(),
14148 "self denial must not relay route.bind"
14149 );
14150 }
14151
14152 #[test]
14153 fn unsupported_channel_zero_frame_returns_error() {
14154 let handler = ControlHandler::default();
14155 let request = Frame::build(
14156 FrameType::Request,
14157 control_flags(),
14158 0,
14159 0,
14160 21,
14161 b"opaque".to_vec(),
14162 )
14163 .unwrap();
14164
14165 let response = handler
14166 .handle_control(ConnectionId::new(1), request)
14167 .unwrap();
14168
14169 assert_eq!(response[0].header.ty, FrameType::Error);
14170 assert_eq!(
14171 parse_error(&response[0])["code"],
14172 "unsupported_control_frame"
14173 );
14174 }
14175
14176 mod swap {
14181 use super::*;
14182
14183 const INCUMBENT: ConnectionId = ConnectionId::new(30);
14184 const CANDIDATE: ConnectionId = ConnectionId::new(40);
14185
14186 struct Swap {
14187 registry: Arc<Registry>,
14188 forwarding: Arc<ForwardingTable>,
14189 handler: ControlHandler,
14190 incumbent_ctx: RouteCtx,
14191 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14192 candidate_ctx: RouteCtx,
14193 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14194 }
14195
14196 async fn swap_with_incumbent() -> Swap {
14197 let registry = Arc::new(Registry::default());
14198 let forwarding = Arc::new(ForwardingTable::default());
14199 let handler =
14200 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14201 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
14202 hello_via_sink(
14203 &handler,
14204 &incumbent_ctx,
14205 &mut incumbent_rx,
14206 hello_frame("aft", PROTOCOL_VERSION, 7),
14207 )
14208 .await;
14209 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
14210 Swap {
14211 registry,
14212 forwarding,
14213 handler,
14214 incumbent_ctx,
14215 incumbent_rx,
14216 candidate_ctx,
14217 candidate_rx,
14218 }
14219 }
14220
14221 fn register_candidate(swap: &Swap, ready: Option<bool>) {
14222 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
14223 candidate_manifest.ready = ready;
14224 let registration = swap
14225 .registry
14226 .register_candidate_with_control_ops(
14227 candidate_manifest,
14228 PROTOCOL_VERSION,
14229 CANDIDATE,
14230 module_baseline_control_ops(),
14231 )
14232 .unwrap();
14233 swap.forwarding
14234 .register_candidate_module_connection(
14235 CANDIDATE,
14236 "aft".to_string(),
14237 PROTOCOL_VERSION,
14238 manifest_concurrency(®istration.manifest),
14239 swap.candidate_ctx.egress.clone(),
14240 )
14241 .unwrap();
14242 }
14243
14244 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
14245 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
14246 swap.registry.promote_candidate("aft").unwrap().unwrap();
14247 cutover.incumbent.unwrap()
14248 }
14249
14250 #[tokio::test]
14259 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
14260 let mut swap = swap_with_incumbent().await;
14261 let hold = Arc::new(std::sync::Barrier::new(2));
14262 let handler = swap
14263 .handler
14264 .clone()
14265 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
14266 handler
14267 .supervisor
14268 .open_swap("aft", "candidate-nonce".to_string());
14269 let hello = {
14270 let handler = handler.clone();
14271 let candidate_ctx = swap.candidate_ctx.clone();
14272 std::thread::spawn(move || {
14273 handler
14274 .handle_hello(
14275 candidate_ctx.connection_id,
14276 Some(candidate_ctx.egress.clone()),
14277 hello_frame_with_nonce(
14278 "aft",
14279 PROTOCOL_VERSION,
14280 8,
14281 Some("candidate-nonce"),
14282 ),
14283 )
14284 .unwrap()
14285 })
14286 };
14287
14288 hold.wait();
14289 assert!(swap
14292 .registry
14293 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
14294 .unwrap()
14295 .is_some());
14296 assert!(swap
14297 .registry
14298 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
14299 .unwrap()
14300 .is_none());
14301 assert!(swap
14302 .forwarding
14303 .module_endpoint_for_connection(CANDIDATE)
14304 .unwrap()
14305 .is_none());
14306
14307 hold.wait();
14308 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
14309 assert_eq!(
14310 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
14311 FrameType::HelloAck
14312 );
14313 let published = swap
14314 .registry
14315 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
14316 .unwrap()
14317 .expect("candidate is published once its HELLO completes");
14318 assert_eq!(published.connection_id, CANDIDATE);
14319 assert!(swap
14320 .forwarding
14321 .module_endpoint_for_connection(CANDIDATE)
14322 .unwrap()
14323 .is_some());
14324 }
14325
14326 fn keyed_total(counters: &Value, key: &str) -> u64 {
14327 counters[key]
14328 .as_object()
14329 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
14330 .unwrap_or(0)
14331 }
14332
14333 #[tokio::test]
14334 async fn replacement_logs_old_end_and_new_admission_once() {
14335 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14336 let mut swap = swap_with_incumbent().await;
14337 swap.handler
14338 .supervisor
14339 .open_swap("aft", "candidate-nonce".to_string());
14340 hello_via_sink(
14341 &swap.handler,
14342 &swap.candidate_ctx,
14343 &mut swap.candidate_rx,
14344 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
14345 )
14346 .await;
14347 cutover(&swap);
14348 swap.handler.cleanup_connection(INCUMBENT).unwrap();
14349
14350 let captured = crate::router::test_log::captured_logs(&logs);
14351 println!("captured replacement registry logs:\n{captured}");
14352 let new_admissions: Vec<_> = captured
14353 .lines()
14354 .filter(|line| {
14355 line.contains("connection_id=40")
14356 && line.contains("swap candidate registered; not routable until cutover")
14357 })
14358 .collect();
14359 assert_eq!(
14360 new_admissions.len(),
14361 1,
14362 "unexpected admission logs: {captured}"
14363 );
14364 assert!(new_admissions[0].contains("module_id=aft"));
14365 assert!(new_admissions[0].contains("connection_id=40"));
14366 assert!(
14367 new_admissions[0].contains("swap candidate registered; not routable until cutover")
14368 );
14369 assert!(new_admissions[0].contains("ready=true"));
14370
14371 let old_admissions: Vec<_> = captured
14372 .lines()
14373 .filter(|line| {
14374 line.contains("module registered module_id=aft ")
14375 && line.contains("connection_id=30")
14376 })
14377 .collect();
14378 assert_eq!(
14379 old_admissions.len(),
14380 1,
14381 "unexpected incumbent admission: {captured}"
14382 );
14383
14384 let old_ends: Vec<_> = captured
14385 .lines()
14386 .filter(|line| {
14387 line.contains("connection_id=30") && line.contains("module registration ended")
14388 })
14389 .collect();
14390 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
14391 assert!(old_ends[0].contains("module_id=aft"));
14392 assert!(old_ends[0].contains("connection_id=30"));
14393 assert!(old_ends[0].contains("reason=replaced"));
14394 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
14395 assert_eq!(
14396 captured
14397 .lines()
14398 .filter(|line| line.contains("module registration promoted"))
14399 .count(),
14400 1,
14401 "unexpected promotion logs: {captured}"
14402 );
14403 }
14404
14405 #[tokio::test]
14411 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
14412 let mut swap = swap_with_incumbent().await;
14413 let handler = swap.handler.clone();
14414
14415 let cotenant = ConnectionId::new(31);
14417 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
14418 let (cotenant_task, cotenant_bind) = relay_route_open(
14419 &handler,
14420 cotenant,
14421 &cotenant_ctx.egress,
14422 &mut swap.incumbent_rx,
14423 100,
14424 "aft",
14425 "swap-cotenant",
14426 )
14427 .await;
14428 handler
14429 .handle_control_frame(
14430 &swap.incumbent_ctx,
14431 route_bind_ack(cotenant_bind.header.corr),
14432 )
14433 .await
14434 .unwrap();
14435 assert!(cotenant_task.await.unwrap().is_empty());
14436 let (cotenant_channel, cotenant_epoch) =
14437 published_route(&cotenant_rx.recv().await.unwrap());
14438
14439 let caller = ConnectionId::new(32);
14441 let (caller_ctx, mut caller_rx) = route_ctx(caller);
14442 let (caller_task, caller_bind) = relay_route_open(
14443 &handler,
14444 caller,
14445 &caller_ctx.egress,
14446 &mut swap.incumbent_rx,
14447 101,
14448 "aft",
14449 "swap-caller",
14450 )
14451 .await;
14452 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
14453
14454 register_candidate(&swap, None);
14455 cutover(&swap);
14456
14457 let ack = handler
14459 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
14460 .await;
14461 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
14462 if module_loop_error.is_some() {
14463 handler.cleanup_connection(INCUMBENT).unwrap();
14466 }
14467
14468 assert!(
14470 cotenant_rx.try_recv().is_err(),
14471 "the co-tenant route on the incumbent was torn down by one late ack: \
14472 {module_loop_error:?}"
14473 );
14474 assert!(matches!(
14475 swap.forwarding
14476 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
14477 .unwrap(),
14478 DataRoute::Client(DataRouteState::Bound(_))
14479 ));
14480 assert_eq!(module_loop_error, None);
14481 assert!(swap
14482 .registry
14483 .get_module_by_connection(INCUMBENT)
14484 .unwrap()
14485 .is_some());
14486
14487 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
14489 .await
14490 .expect("the incumbent is told to drop the abandoned binding")
14491 .unwrap()
14492 .frame;
14493 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14494 assert_eq!(goodbye.header.channel, abandoned_channel);
14495 assert_eq!(goodbye.header.epoch, abandoned_epoch);
14496 assert!(swap.incumbent_rx.try_recv().is_err());
14497
14498 let response = caller_task.await.unwrap();
14500 assert_eq!(response.len(), 1);
14501 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14502 assert!(caller_rx.try_recv().is_err());
14503
14504 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
14508 let counters = handler.counters().snapshot();
14509 assert_eq!(
14510 keyed_total(&counters, "route_open_accepted_by_principal"),
14511 1
14512 );
14513 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
14514 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
14515 }
14516
14517 #[tokio::test]
14521 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
14522 let mut swap = swap_with_incumbent().await;
14523 register_candidate(&swap, None);
14524 let incumbent = cutover(&swap);
14525 swap.forwarding
14526 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
14527 .unwrap()
14528 .expect("the incumbent is still registered");
14529
14530 let client = ConnectionId::new(33);
14531 let (client_ctx, mut client_rx) = route_ctx(client);
14532 let route_handler = swap.handler.clone();
14533 let open_ctx = RouteCtx {
14534 connection_id: client,
14535 egress: client_ctx.egress.clone(),
14536 };
14537 let mut route_task = tokio::spawn(async move {
14538 route_handler
14539 .handle_control_frame(
14540 &open_ctx,
14541 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
14542 )
14543 .await
14544 .unwrap()
14545 });
14546 let bind = tokio::select! {
14547 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
14548 response = &mut route_task => {
14549 let response = response.unwrap();
14550 panic!(
14551 "post-cutover route.open was refused instead of relayed to the candidate: {}",
14552 parse_error(&response[0])["code"]
14553 );
14554 }
14555 };
14556 swap.handler
14557 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
14558 .await
14559 .unwrap();
14560 assert!(route_task.await.unwrap().is_empty());
14561 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
14562 match swap
14563 .forwarding
14564 .lookup_data_route(client, channel, epoch)
14565 .unwrap()
14566 {
14567 DataRoute::Client(DataRouteState::Bound(route)) => {
14568 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
14569 }
14570 other => panic!("expected a bound route on the candidate, got {other:?}"),
14571 }
14572 assert!(swap.incumbent_rx.try_recv().is_err());
14573 }
14574
14575 #[tokio::test]
14580 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
14581 let swap = swap_with_incumbent().await;
14582 register_candidate(&swap, Some(false));
14583 let update = Frame::build(
14584 FrameType::Request,
14585 control_flags(),
14586 0,
14587 0,
14588 55,
14589 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14590 provides: manifest("aft", PROTOCOL_VERSION).provides,
14591 capabilities: None,
14592 ready: Some(true),
14593 })
14594 .unwrap(),
14595 )
14596 .unwrap();
14597
14598 let replies = swap
14599 .handler
14600 .handle_control_frame(&swap.candidate_ctx, update)
14601 .await
14602 .unwrap();
14603
14604 assert_eq!(replies.len(), 1);
14605 assert_eq!(
14606 replies[0].header.ty,
14607 FrameType::Response,
14608 "candidate catalog.update was refused: {:?}",
14609 serde_json::from_slice::<Value>(&replies[0].body).ok()
14610 );
14611 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
14612 assert_eq!(
14613 swap.registry
14614 .get_module("aft")
14615 .unwrap()
14616 .unwrap()
14617 .connection_id,
14618 INCUMBENT
14619 );
14620 }
14621 }
14622
14623 mod swap_admission {
14627 use super::*;
14628
14629 const INCUMBENT_NONCE: &str = "incumbent-nonce";
14630 const CANDIDATE_NONCE: &str = "candidate-nonce";
14631
14632 fn handler_with_incumbent(
14633 module_id: &str,
14634 reserved: bool,
14635 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
14636 let registry = Arc::new(Registry::default());
14637 let supervisor = SupervisorHandle::new();
14638 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
14639 if reserved {
14640 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
14641 }
14642 let handler =
14643 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
14644 let incumbent = handler
14645 .handle_control(
14646 ConnectionId::new(1),
14647 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
14648 )
14649 .unwrap();
14650 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
14651 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
14652 (registry, supervisor, handler)
14653 }
14654
14655 #[test]
14661 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
14662 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
14663
14664 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
14665 let replies = handler
14666 .handle_control(
14667 ConnectionId::new(connection),
14668 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
14669 )
14670 .unwrap();
14671 assert_eq!(replies[0].header.ty, FrameType::Error);
14672 assert_eq!(
14673 parse_error(&replies[0])["code"],
14674 "swap_token_invalid",
14675 "nonce {nonce:?}"
14676 );
14677 }
14678 assert!(registry.get_candidate("aft").unwrap().is_none());
14679 assert_eq!(
14680 registry.get_module("aft").unwrap().unwrap().connection_id,
14681 ConnectionId::new(1)
14682 );
14683
14684 let admitted = handler
14687 .handle_control(
14688 ConnectionId::new(4),
14689 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
14690 )
14691 .unwrap();
14692 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
14693 assert_eq!(
14694 registry
14695 .get_candidate("aft")
14696 .unwrap()
14697 .unwrap()
14698 .connection_id,
14699 ConnectionId::new(4)
14700 );
14701 assert_eq!(
14702 registry.get_module("aft").unwrap().unwrap().connection_id,
14703 ConnectionId::new(1),
14704 "the candidate must not take the active slot"
14705 );
14706 let replayed = handler
14707 .handle_control(
14708 ConnectionId::new(5),
14709 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
14710 )
14711 .unwrap();
14712 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
14713
14714 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
14718 let squatter = handler
14719 .handle_control(
14720 ConnectionId::new(6),
14721 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
14722 )
14723 .unwrap();
14724 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
14725 assert!(
14726 registry.get_module("aft").unwrap().is_none(),
14727 "a squatter took the active slot of an id being swapped"
14728 );
14729 }
14730
14731 #[test]
14736 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
14737 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
14738
14739 let replies = handler
14740 .handle_control(
14741 ConnectionId::new(2),
14742 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14743 )
14744 .unwrap();
14745
14746 assert_eq!(
14747 replies[0].header.ty,
14748 FrameType::HelloAck,
14749 "reserved candidate refused: {:?}",
14750 serde_json::from_slice::<Value>(&replies[0].body).ok()
14751 );
14752 assert_eq!(
14753 registry
14754 .get_candidate("vault")
14755 .unwrap()
14756 .unwrap()
14757 .connection_id,
14758 ConnectionId::new(2)
14759 );
14760 }
14761
14762 #[test]
14765 fn without_an_open_swap_the_ordinary_gates_decide() {
14766 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
14767 supervisor.close_swap("vault");
14768
14769 let candidate = handler
14770 .handle_control(
14771 ConnectionId::new(2),
14772 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14773 )
14774 .unwrap();
14775 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
14776 let duplicate = handler
14777 .handle_control(
14778 ConnectionId::new(3),
14779 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
14780 )
14781 .unwrap();
14782 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14783 assert!(registry.get_candidate("vault").unwrap().is_none());
14784 }
14785 }
14786
14787 mod scopes {
14791 mod idle_tests {
14792 use super::*;
14793 include!("scope_idle_tests.rs");
14794 }
14795 mod route_open_timings {
14796 use super::*;
14797 include!("route_open_timing_tests.rs");
14798 }
14799 mod activity {
14800 include!("activity_scope_tests.rs");
14801 }
14802 mod delta {
14803 include!("scope_handler_delta_tests.rs");
14804 }
14805 mod call_modes {
14806 include!("scope_call_mode_tests.rs");
14807 }
14808
14809 use subc_protocol::scope::{
14810 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
14811 ScopeStatus,
14812 };
14813
14814 use super::*;
14815
14816 const OWNER: &str = "prefrontal-core";
14817
14818 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
14819 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
14820 }
14821
14822 async fn call(
14823 handler: &ControlHandler,
14824 ctx: &RouteCtx,
14825 request: &ModuleControlRequestFromModule,
14826 ) -> Frame {
14827 let body = serde_json::to_vec(request).unwrap();
14828 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
14829 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
14830 assert_eq!(replies.len(), 1, "{replies:?}");
14831 replies.pop().unwrap()
14832 }
14833
14834 async fn sync(
14835 handler: &ControlHandler,
14836 ctx: &RouteCtx,
14837 generation: u64,
14838 scopes: Vec<ScopeRecord>,
14839 ) -> Result<ModuleControlResponseToModule, String> {
14840 let reply = call(
14841 handler,
14842 ctx,
14843 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
14844 )
14845 .await;
14846 match reply.header.ty {
14847 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
14848 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
14849 }
14850 }
14851
14852 async fn describe(
14853 handler: &ControlHandler,
14854 ctx: &RouteCtx,
14855 owner: &str,
14856 scope_ref: &str,
14857 ) -> ModuleControlResponseToModule {
14858 let reply = call(
14859 handler,
14860 ctx,
14861 &ModuleControlRequestFromModule::ScopeDescribe {
14862 owner: Principal::Reserved {
14863 module_id: owner.to_string(),
14864 },
14865 scope_ref: scope_ref.to_string(),
14866 },
14867 )
14868 .await;
14869 assert_eq!(
14870 reply.header.ty,
14871 FrameType::Response,
14872 "{:?}",
14873 parse_error(&reply)
14874 );
14875 serde_json::from_slice(&reply.body).unwrap()
14876 }
14877
14878 #[tokio::test]
14879 async fn scope_describe_never_echoes_bind_proof_or_hash() {
14880 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
14881 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
14882 let mut rig = rig().await;
14883 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
14884 .await;
14885 let _bound = rig
14886 .bound(
14887 Some(AFT),
14888 PLEXUS,
14889 Some(
14890 rig.selector("s", Some(1))
14891 .with_bind_proof(Some(PROOF.into())),
14892 ),
14893 )
14894 .await;
14895 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
14896 let output = serde_json::to_string(&reply).unwrap();
14897 assert!(output.contains("\"status\":\"live\""));
14898 assert!(!output.contains(PROOF));
14899 assert!(!output.contains(HASH));
14900 }
14901
14902 async fn module(
14904 handler: &ControlHandler,
14905 connection: u64,
14906 module_id: &str,
14907 nonce: Option<&str>,
14908 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
14909 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
14910 hello_via_sink(
14911 handler,
14912 &ctx,
14913 &mut rx,
14914 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
14915 )
14916 .await;
14917 (ctx, rx)
14918 }
14919
14920 #[tokio::test]
14923 async fn a_client_connection_cannot_sync_or_describe() {
14924 let handler = ControlHandler::new(Arc::new(Registry::default()));
14925 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
14926 for request in [
14927 ModuleControlRequestFromModule::ScopeSync {
14928 generation: 1,
14929 scopes: vec![head("s", 1)],
14930 },
14931 ModuleControlRequestFromModule::ScopeDescribe {
14932 owner: Principal::Direct,
14933 scope_ref: "s".to_string(),
14934 },
14935 ] {
14936 let reply = call(&handler, &ctx, &request).await;
14937 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
14938 }
14939 assert!(
14940 !handler
14941 .scopes
14942 .read()
14943 .unwrap()
14944 .describe(
14945 &Principal::Reserved {
14946 module_id: OWNER.to_string()
14947 },
14948 "s"
14949 )
14950 .owner_synced
14951 );
14952 }
14953
14954 #[tokio::test]
14957 async fn a_module_without_a_supervised_launch_cannot_sync() {
14958 let handler = ControlHandler::new(Arc::new(Registry::default()));
14959 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
14960 assert_eq!(
14961 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
14962 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14963 );
14964 }
14965
14966 #[tokio::test]
14967 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
14968 let supervisor = SupervisorHandle::new();
14969 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14970 let handler = ControlHandler::new(Arc::new(Registry::default()))
14971 .with_supervisor(supervisor.clone());
14972 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
14973 sync(&handler, &incumbent, 1, vec![head("s", 1)])
14974 .await
14975 .expect("the current launch syncs");
14976
14977 supervisor.open_swap(OWNER, "n2".to_string());
14980 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
14981 assert_eq!(
14982 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
14983 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14984 );
14985 sync(&handler, &incumbent, 2, vec![head("s", 1)])
14986 .await
14987 .expect("the serving owner syncs during the swap");
14988
14989 supervisor.close_swap(OWNER);
14992 assert_eq!(
14993 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
14994 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14995 );
14996 sync(&handler, &incumbent, 3, vec![head("s", 1)])
14997 .await
14998 .expect("the serving owner syncs after the rollback");
14999 handler.cleanup_connection(candidate.connection_id).unwrap();
15000
15001 supervisor.open_swap(OWNER, "n3".to_string());
15006 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
15007 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
15008 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
15009 .await
15010 .expect("the promoted launch takes authority");
15011 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
15012 panic!("unexpected reply {reply:?}");
15013 };
15014 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
15015 assert_eq!(
15016 sync(&handler, &incumbent, 4, Vec::new()).await,
15017 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15018 );
15019 }
15020
15021 #[tokio::test]
15024 async fn closing_the_authority_connection_frees_sync_authority() {
15025 let supervisor = SupervisorHandle::new();
15026 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15027 let handler = ControlHandler::new(Arc::new(Registry::default()))
15028 .with_supervisor(supervisor.clone());
15029 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15030 sync(&handler, &first, 10, vec![head("s", 1)])
15031 .await
15032 .unwrap();
15033 handler.cleanup_connection(first.connection_id).unwrap();
15034
15035 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
15036 sync(&handler, &second, 1, vec![head("s", 1)])
15037 .await
15038 .expect("the next connection takes the released authority");
15039 }
15040
15041 #[tokio::test]
15042 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
15043 let supervisor = SupervisorHandle::new();
15044 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15045 let handler =
15046 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
15047 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
15048 sync(&handler, &first, 10, vec![head("s", 1)])
15049 .await
15050 .unwrap();
15051 handler
15052 .handle_control_frame(
15053 &first,
15054 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
15055 )
15056 .await
15057 .unwrap();
15058 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
15059 sync(&handler, &second, 1, vec![head("s", 1)])
15060 .await
15061 .expect("GOODBYE releases authority even if the old socket remains open");
15062 }
15063
15064 #[tokio::test]
15065 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
15066 let registry = Arc::new(Registry::default());
15067 let supervisor_handle = SupervisorHandle::new();
15068 let supervisor =
15069 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
15070 .with_handle(supervisor_handle.clone())
15071 .with_daemon_incarnation("incarnation-7".to_string());
15072 supervisor
15075 .supervise_configured(
15076 ModuleSpec {
15077 module_id: OWNER.to_string(),
15078 program: PathBuf::from("/nonexistent/prefrontal-core"),
15079 args: Vec::new(),
15080 env: Vec::new(),
15081 reserved: false,
15082 reserved_prefixes: Vec::new(),
15083 protocol: ModuleProtocol::Subc,
15084 overlap: Default::default(),
15085 },
15086 false,
15087 )
15088 .unwrap();
15089 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
15090 let handler =
15091 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15092 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
15093
15094 let ModuleControlResponseToModule::ScopeDescribe {
15096 status,
15097 daemon_incarnation,
15098 owner_synced,
15099 owner_configured,
15100 scope,
15101 ..
15102 } = describe(&handler, &reader, OWNER, "s").await
15103 else {
15104 panic!("not a describe reply");
15105 };
15106 assert_eq!(status, ScopeStatus::NotLive);
15107 assert_eq!(daemon_incarnation, "incarnation-7");
15108 assert!(!owner_synced);
15109 assert!(owner_configured);
15110 assert!(scope.is_none());
15111
15112 let ModuleControlResponseToModule::ScopeDescribe {
15115 status,
15116 owner_configured,
15117 ..
15118 } = describe(&handler, &reader, "ghost", "s").await
15119 else {
15120 panic!("not a describe reply");
15121 };
15122 assert_eq!(status, ScopeStatus::NotLive);
15123 assert!(!owner_configured);
15124
15125 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
15127 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
15128 let ModuleControlResponseToModule::ScopeDescribe {
15129 status,
15130 scope_epoch,
15131 owner_synced,
15132 scope,
15133 ..
15134 } = describe(&handler, &reader, OWNER, "s").await
15135 else {
15136 panic!("not a describe reply");
15137 };
15138 assert_eq!(status, ScopeStatus::Live);
15139 assert_eq!(scope_epoch, Some(4));
15140 assert!(owner_synced);
15141 let stamp = scope.expect("a live scope carries its stamp");
15142 assert!(
15143 stamp.owner_authorized,
15144 "prefrontal-core is the default authority"
15145 );
15146 assert_eq!(stamp.kind, ScopeKind::Head);
15147 }
15148
15149 #[tokio::test]
15150 async fn scope_authority_owners_decides_owner_authorized() {
15151 let supervisor = SupervisorHandle::new();
15152 supervisor.set_spawn_nonce("broca", "b1".to_string());
15153 let handler = ControlHandler::new(Arc::new(Registry::default()))
15154 .with_supervisor(supervisor)
15155 .with_scope_authority_owners(vec!["broca".to_string()]);
15156 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
15157 let mut gated = head("s", 1);
15158 gated.attributes.agent_id = Some("agent".to_string());
15159 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
15160 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
15161 describe(&handler, &broca, "broca", "s").await
15162 else {
15163 panic!("not a describe reply");
15164 };
15165 assert!(scope.unwrap().owner_authorized);
15166 }
15167
15168 #[tokio::test]
15172 async fn scope_ops_and_the_scopes_capability_are_advertised() {
15173 let handler = ControlHandler::new(Arc::new(Registry::default()));
15174 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
15175 let ack = hello_via_sink(
15176 &handler,
15177 &ctx,
15178 &mut rx,
15179 hello_frame("m", PROTOCOL_VERSION, 1),
15180 )
15181 .await;
15182 let ack = parse_ack(&ack);
15183 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
15184 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
15185 }
15186 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
15187
15188 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
15189 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15190 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15191 let reply = handler
15192 .handle_control_frame(&client, frame)
15193 .await
15194 .unwrap()
15195 .pop()
15196 .unwrap();
15197 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15198 serde_json::from_slice(&reply.body).unwrap()
15199 else {
15200 panic!("not a server.describe reply");
15201 };
15202 assert!(
15203 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
15204 "{capabilities:?}"
15205 );
15206 }
15207
15208 const PLEXUS: &str = "plexus";
15211 const OTHER: &str = "other";
15212 const AFT: &str = "aft";
15213 const BROCA: &str = "broca";
15214 const MAGIC: &str = "magic-context";
15215
15216 fn nonce(module_id: &str) -> String {
15217 format!("nonce-{module_id}")
15218 }
15219
15220 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15221 let (tx, rx) = mpsc::channel(64);
15222 (
15223 RouteCtx {
15224 connection_id: ConnectionId::new(connection),
15225 egress: FrameSink::new(tx),
15226 },
15227 rx,
15228 )
15229 }
15230
15231 struct Rig {
15235 handler: ControlHandler,
15236 forwarding: Arc<ForwardingTable>,
15237 owner: RouteCtx,
15238 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15239 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
15240 generation: u64,
15241 next_connection: u64,
15242 _supervisor: Supervisor,
15243 }
15244
15245 async fn rig() -> Rig {
15246 rig_with_flow_support(true).await
15247 }
15248
15249 async fn rig_with_flow_support(flow_support: bool) -> Rig {
15250 rig_with_scope_capabilities(flow_support, false).await
15251 }
15252
15253 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
15254 let mut provides = Vec::new();
15255 if flow_support {
15256 provides.push("flow-scopes/v1");
15257 }
15258 if run_support {
15259 provides.push("agent-run-scopes/v1");
15260 }
15261 rig_with_target_capabilities(&provides).await
15262 }
15263
15264 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
15265 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
15266 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
15267 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
15270 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
15271 Frame::build(
15272 FrameType::Hello,
15273 control_flags(),
15274 0,
15275 0,
15276 connection,
15277 serde_json::to_vec(&body).unwrap(),
15278 )
15279 .unwrap()
15280 }
15281
15282 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
15283 let registry = Arc::new(Registry::default());
15284 let forwarding = Arc::new(ForwardingTable::default());
15285 let supervisor_handle = SupervisorHandle::new();
15286 let supervisor =
15287 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
15288 .with_handle(supervisor_handle.clone());
15289 supervisor
15290 .supervise_configured(
15291 ModuleSpec {
15292 module_id: OWNER.to_string(),
15293 program: PathBuf::from("/nonexistent/prefrontal-core"),
15294 args: Vec::new(),
15295 env: Vec::new(),
15296 reserved: false,
15297 reserved_prefixes: Vec::new(),
15298 protocol: ModuleProtocol::Subc,
15299 overlap: Default::default(),
15300 },
15301 false,
15302 )
15303 .unwrap();
15304 for module_id in [OWNER, AFT, BROCA, MAGIC] {
15305 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
15306 }
15307 let handler =
15308 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15309 .with_supervisor(supervisor_handle);
15310 let (owner, mut owner_rx) = wide_ctx(1);
15311 hello_via_sink(
15312 &handler,
15313 &owner,
15314 &mut owner_rx,
15315 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
15316 )
15317 .await;
15318 let mut modules = BTreeMap::new();
15319 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
15320 let (ctx, mut rx) = wide_ctx(connection);
15321 let hello = hello_with_capabilities(module_id, connection, provides);
15322 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
15323 modules.insert(module_id.to_string(), (ctx, rx));
15324 }
15325 Rig {
15326 handler,
15327 forwarding,
15328 owner,
15329 _owner_rx: owner_rx,
15330 modules,
15331 generation: 0,
15332 next_connection: 100,
15333 _supervisor: supervisor,
15334 }
15335 }
15336
15337 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
15338 ScopeCarrier::new(Principal::Reserved {
15339 module_id: module_id.to_string(),
15340 })
15341 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
15342 }
15343
15344 fn session(scope_epoch: u64) -> ScopeRecord {
15347 let mut record = head("s", scope_epoch);
15348 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
15349 record.attributes.agent_id = Some("agent-1".to_string());
15350 record.attributes.delegates = true;
15351 record
15352 }
15353
15354 impl Rig {
15355 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
15356 self.generation += 1;
15357 sync(&self.handler, &self.owner, self.generation, scopes)
15358 .await
15359 .expect("the owner's sync is accepted");
15360 }
15361
15362 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15363 ScopeSelector::new(
15364 Principal::Reserved {
15365 module_id: OWNER.to_string(),
15366 },
15367 scope_ref.to_string(),
15368 scope_epoch,
15369 )
15370 }
15371
15372 fn open_frame(
15373 &mut self,
15374 opener: Option<&str>,
15375 target: &str,
15376 scope: Option<ScopeSelector>,
15377 ) -> (
15378 RouteCtx,
15379 mpsc::Receiver<crate::router::OutboundFrame>,
15380 Frame,
15381 ) {
15382 self.next_connection += 1;
15383 let (ctx, rx) = wide_ctx(self.next_connection);
15384 let root = unique_project_root("scoped-open");
15385 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
15386 target: RouteTarget::ToolProvider {
15387 module_id: target.to_string(),
15388 },
15389 identity: BindIdentity::new(
15390 root.path().to_path_buf(),
15391 "unit".to_string(),
15392 "session".to_string(),
15393 ),
15394 consumer_identity: opener.map(|module_id| ConsumerIdentity {
15395 module_id: module_id.to_string(),
15396 launch_nonce: nonce(module_id),
15397 }),
15398 consumer_capabilities: None,
15399 role_versions: None,
15400 admission_facts: None,
15401 scope,
15402 })
15403 .unwrap();
15404 let frame = Frame::build(
15405 FrameType::Request,
15406 control_flags(),
15407 0,
15408 0,
15409 self.next_connection,
15410 body,
15411 )
15412 .unwrap();
15413 (ctx, rx, frame)
15414 }
15415
15416 async fn refused(
15418 &mut self,
15419 opener: Option<&str>,
15420 target: &str,
15421 scope: Option<ScopeSelector>,
15422 ) -> String {
15423 self.refusal_body(opener, target, scope).await["code"]
15424 .as_str()
15425 .unwrap()
15426 .to_string()
15427 }
15428
15429 async fn refusal_body(
15430 &mut self,
15431 opener: Option<&str>,
15432 target: &str,
15433 scope: Option<ScopeSelector>,
15434 ) -> Value {
15435 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
15436 let replies = tokio::time::timeout(
15437 Duration::from_secs(2),
15438 self.handler.handle_control_frame(&ctx, frame),
15439 )
15440 .await
15441 .expect("the open must be refused before waiting for a bind ack")
15442 .unwrap();
15443 assert_eq!(replies.len(), 1, "{replies:?}");
15444 assert_eq!(replies[0].header.ty, FrameType::Error);
15445 let (_, module_rx) = self.modules.get_mut(target).unwrap();
15446 assert!(
15447 module_rx.try_recv().is_err(),
15448 "a refused open relays nothing"
15449 );
15450 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
15451 parse_error(&replies[0])
15452 }
15453
15454 async fn relayed(
15456 &mut self,
15457 opener: Option<&str>,
15458 target: &str,
15459 scope: Option<ScopeSelector>,
15460 ) -> Relayed {
15461 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
15462 let handler = self.handler.clone();
15463 let task_ctx = ctx.clone();
15464 let task = tokio::spawn(async move {
15465 handler
15466 .handle_control_frame(&task_ctx, frame)
15467 .await
15468 .unwrap()
15469 });
15470 let (_, module_rx) = self.modules.get_mut(target).unwrap();
15471 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
15472 .await
15473 .expect("the target receives the relayed route.bind")
15474 .unwrap()
15475 .frame;
15476 Relayed {
15477 target: target.to_string(),
15478 client: ctx,
15479 client_rx: rx,
15480 task,
15481 bind,
15482 }
15483 }
15484
15485 async fn ack(&self, relayed: &Relayed) {
15486 let (module, _) = &self.modules[&relayed.target];
15487 self.handler
15488 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
15489 .await
15490 .unwrap();
15491 }
15492
15493 async fn bound(
15495 &mut self,
15496 opener: Option<&str>,
15497 target: &str,
15498 scope: Option<ScopeSelector>,
15499 ) -> Bound {
15500 let relayed = self.relayed(opener, target, scope).await;
15501 self.ack(&relayed).await;
15502 let Relayed {
15503 target,
15504 client,
15505 mut client_rx,
15506 task,
15507 bind,
15508 } = relayed;
15509 assert!(
15510 task.await.unwrap().is_empty(),
15511 "the open is answered by commit"
15512 );
15513 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
15514 Bound {
15515 target,
15516 client,
15517 client_rx,
15518 channel,
15519 epoch,
15520 bind,
15521 }
15522 }
15523
15524 fn live(&self, route: &Bound) -> bool {
15525 matches!(
15526 self.forwarding
15527 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
15528 .unwrap(),
15529 DataRoute::Client(DataRouteState::Bound(_))
15530 )
15531 }
15532 }
15533
15534 struct Relayed {
15535 target: String,
15536 client: RouteCtx,
15537 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15538 task: tokio::task::JoinHandle<Vec<Frame>>,
15539 bind: Frame,
15540 }
15541
15542 struct Bound {
15543 target: String,
15544 client: RouteCtx,
15545 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15546 channel: u16,
15547 epoch: u32,
15548 bind: Frame,
15549 }
15550
15551 impl Bound {
15552 fn closed_reason(&mut self) -> RouteCloseReason {
15555 let mut reason = None;
15556 let mut goodbye = false;
15557 while let Ok(outbound) = self.client_rx.try_recv() {
15558 let frame = outbound.frame;
15559 match frame.header.ty {
15560 FrameType::Goodbye => {
15561 assert_eq!(
15562 (frame.header.channel, frame.header.epoch),
15563 (self.channel, self.epoch)
15564 );
15565 goodbye = true;
15566 }
15567 FrameType::Push => {
15568 let ClientControlPush::RouteClosed {
15569 reason: r,
15570 module_id,
15571 ..
15572 } = serde_json::from_slice(&frame.body).unwrap()
15573 else {
15574 panic!("unexpected push");
15575 };
15576 assert_eq!(module_id, self.target);
15577 reason = Some(r);
15578 }
15579 other => panic!("unexpected frame {other:?}"),
15580 }
15581 }
15582 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
15583 reason.expect("the client is told why the route closed")
15584 }
15585
15586 fn untouched(&mut self) -> bool {
15587 self.client_rx.try_recv().is_err()
15588 }
15589
15590 fn stamp(&self) -> Option<ScopeStamp> {
15591 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
15592 ModuleControlRequest::RouteBind { scope, .. } => scope,
15593 other => panic!("expected a route.bind, got {other:?}"),
15594 }
15595 }
15596 }
15597
15598 #[tokio::test]
15599 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
15600 ) {
15601 let mut rig = rig().await;
15602 let mut record = session(1);
15603 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
15604 record.child_owners = vec![Principal::Reserved {
15605 module_id: MAGIC.to_string(),
15606 }];
15607 rig.sync(vec![record]).await;
15608 let scope = || Some(rig_selector("s", Some(1)));
15609
15610 rig.bound(Some(OWNER), PLEXUS, scope()).await;
15613 rig.bound(Some(AFT), OTHER, scope()).await;
15614 rig.bound(Some(BROCA), PLEXUS, scope()).await;
15615
15616 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
15620 assert_eq!(
15621 rig.refused(opener, target, scope()).await,
15622 error_codes::SCOPE_NOT_CARRIER,
15623 "{opener:?} -> {target}"
15624 );
15625 }
15626 }
15627
15628 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15629 ScopeSelector::new(
15630 Principal::Reserved {
15631 module_id: OWNER.to_string(),
15632 },
15633 scope_ref.to_string(),
15634 scope_epoch,
15635 )
15636 }
15637
15638 #[tokio::test]
15639 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
15640 let mut rig = rig().await;
15641 rig.sync(vec![session(1)]).await;
15642 for opener in [OWNER, AFT] {
15643 assert_eq!(
15644 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
15645 .await,
15646 error_codes::SCOPE_EPOCH_REQUIRED,
15647 "{opener}"
15648 );
15649 }
15650 }
15651
15652 #[tokio::test]
15653 async fn admission_separates_not_synced_not_live_and_ended() {
15654 let mut rig = rig().await;
15655 let code = rig
15657 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15658 .await;
15659 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
15660 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
15661
15662 let ghost = ScopeSelector::new(
15664 Principal::Reserved {
15665 module_id: "ghost".to_string(),
15666 },
15667 "s".to_string(),
15668 Some(1),
15669 );
15670 assert_eq!(
15671 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
15672 error_codes::SCOPE_NOT_LIVE
15673 );
15674
15675 rig.sync(vec![session(2)]).await;
15676 assert_eq!(
15677 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
15678 .await,
15679 error_codes::SCOPE_NOT_LIVE
15680 );
15681 for epoch in [1, 3] {
15682 assert_eq!(
15683 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
15684 .await,
15685 error_codes::SCOPE_ENDED,
15686 "epoch {epoch}"
15687 );
15688 }
15689 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
15691 .await;
15692 }
15693
15694 #[tokio::test]
15695 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
15696 let mut rig = rig().await;
15697 rig.sync(vec![session(1)]).await;
15698 let route = rig
15699 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15700 .await;
15701 let stamp = route.stamp().expect("a scoped bind carries the stamp");
15702 assert_eq!(stamp.scope_ref, "s");
15703 assert_eq!(stamp.scope_epoch, 1);
15704 assert_eq!(stamp.kind, ScopeKind::Head);
15705 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
15706 assert!(stamp.attributes.delegates);
15707 assert!(stamp.owner_authorized);
15708 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15709 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
15710
15711 let (broca, mut broca_rx) = wide_ctx(50);
15714 hello_via_sink(
15715 &rig.handler,
15716 &broca,
15717 &mut broca_rx,
15718 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
15719 )
15720 .await;
15721 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
15722 .await
15723 .unwrap();
15724 let own = ScopeSelector::new(
15725 Principal::Reserved {
15726 module_id: BROCA.to_string(),
15727 },
15728 "b".to_string(),
15729 Some(1),
15730 );
15731 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
15732 assert!(!route.stamp().unwrap().owner_authorized);
15733 }
15734
15735 #[tokio::test]
15736 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
15737 let mut rig = rig().await;
15738 let mut record = head("s", 1);
15739 record.carriers = vec![carrier(AFT, None)];
15740 let flow_id = "Flow:run-7/step_2!~";
15741 record.attributes.flow_id = Some(flow_id.to_string());
15742 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
15743 .await
15744 .unwrap();
15745 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
15746 panic!("not a sync reply");
15747 };
15748 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
15749 let route = rig
15750 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15751 .await;
15752 assert!(rig.live(&route), "the stamped bind committed");
15753 let stamp = route.stamp().expect("a flow scope carries a stamp");
15754 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
15755 assert_eq!(stamp.attributes.agent_id, None);
15756 assert!(!stamp.attributes.delegates);
15757 assert!(stamp.owner_authorized);
15758 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15759 assert_eq!(unscoped.stamp(), None);
15760 }
15761
15762 #[tokio::test]
15763 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
15764 let mut rig = rig_with_flow_support(false).await;
15765 let mut record = session(1);
15766 record.attributes.flow_id = Some("flow:7".to_string());
15767 rig.sync(vec![record]).await;
15768 for opener in [OWNER, AFT] {
15769 let body = rig
15770 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
15771 .await;
15772 assert_eq!(body["code"], "target_flow_unsupported");
15773 let message = body["message"].as_str().unwrap();
15774 for required in [PLEXUS, "flow-scopes/v1"] {
15775 assert!(message.contains(required), "{message}");
15776 }
15777 }
15778 }
15779
15780 #[tokio::test]
15781 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
15782 let mut rig = rig_with_flow_support(true).await;
15783 let mut record = session(1);
15784 record.attributes.flow_id = Some("flow:7".to_string());
15785 rig.sync(vec![record]).await;
15786 for opener in [OWNER, AFT] {
15787 let route = rig
15788 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
15789 .await;
15790 assert!(rig.live(&route));
15791 assert_eq!(
15792 route.stamp().unwrap().attributes.flow_id.as_deref(),
15793 Some("flow:7")
15794 );
15795 }
15796 }
15797
15798 #[tokio::test]
15799 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
15800 use std::future::Future;
15801
15802 let mut rig = rig_with_flow_support(true).await;
15803 let mut record = session(1);
15804 record.attributes.flow_id = Some("flow:7".to_string());
15805 rig.sync(vec![record]).await;
15806 let (client, mut client_rx, frame) =
15807 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
15808 for _ in 0..64 {
15811 client.egress.try_send(route_bind_ack(1)).unwrap();
15812 }
15813 let handler = rig.handler.clone();
15814 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15815 std::future::poll_fn(|cx| {
15816 assert!(open.as_mut().poll(cx).is_pending());
15817 std::task::Poll::Ready(())
15818 })
15819 .await;
15820 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15821
15822 let old_connection = rig.modules[PLEXUS].0.connection_id;
15823 rig.handler.cleanup_connection(old_connection).unwrap();
15824 let (replacement, mut replacement_rx) = wide_ctx(200);
15825 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
15826 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
15827 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
15828 let hello = Frame::build(
15829 FrameType::Hello,
15830 control_flags(),
15831 0,
15832 0,
15833 200,
15834 serde_json::to_vec(&body).unwrap(),
15835 )
15836 .unwrap();
15837 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
15838
15839 client_rx.try_recv().unwrap();
15840 let replies = tokio::time::timeout(Duration::from_secs(2), open)
15841 .await
15842 .expect("the replacement is refused without waiting for a bind ack")
15843 .unwrap();
15844 assert_eq!(replies.len(), 1);
15845 let body = parse_error(&replies[0]);
15846 assert_eq!(body["code"], "target_flow_unsupported");
15847 for required in [PLEXUS, "flow-scopes/v1"] {
15848 assert!(body["message"].as_str().unwrap().contains(required));
15849 }
15850 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
15851 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
15852 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15853 assert!(rig
15854 .handler
15855 .registry
15856 .get_module_by_connection(replacement.connection_id)
15857 .unwrap()
15858 .is_some());
15859 }
15860
15861 #[tokio::test]
15862 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
15863 use std::future::Future;
15864 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
15865 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15866 const NEXT_HASH: &str =
15867 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
15868 let mut rig = rig().await;
15869 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
15870 record.carriers.push(ScopeCarrier::new(Principal::Direct));
15871 rig.sync(vec![record.clone()]).await;
15872 let (client, mut rx, frame) = rig.open_frame(
15873 None,
15874 PLEXUS,
15875 Some(
15876 rig.selector("s", Some(1))
15877 .with_bind_proof(Some(PROOF.into())),
15878 ),
15879 );
15880 for _ in 0..64 {
15883 client.egress.try_send(route_bind_ack(1)).unwrap();
15884 }
15885 let handler = rig.handler.clone();
15886 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15887 std::future::poll_fn(|cx| {
15888 assert!(open.as_mut().poll(cx).is_pending());
15889 std::task::Poll::Ready(())
15890 })
15891 .await;
15892 record.bind_proof_sha256 = Some(NEXT_HASH.into());
15893 rig.sync(vec![record]).await;
15894 rx.try_recv().unwrap();
15895 let replies = tokio::time::timeout(Duration::from_secs(2), open)
15896 .await
15897 .unwrap()
15898 .unwrap();
15899 assert_eq!(
15900 parse_error(&replies[0])["code"],
15901 error_codes::SCOPE_BIND_PROOF_MISMATCH
15902 );
15903 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
15904 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
15905 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15906 }
15907
15908 #[tokio::test]
15909 async fn scope_bind_proof_reserved_openers_ignore_proof() {
15910 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15911 let mut rig = rig().await;
15912 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15913 .await;
15914 for opener in [OWNER, AFT] {
15915 for proof in [None, Some("malformed+proof=")] {
15916 let route = rig
15917 .bound(
15918 Some(opener),
15919 PLEXUS,
15920 Some(
15921 rig.selector("s", Some(1))
15922 .with_bind_proof(proof.map(str::to_string)),
15923 ),
15924 )
15925 .await;
15926 assert!(rig.live(&route));
15927 }
15928 }
15929 }
15930
15931 #[tokio::test]
15932 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
15933 use std::future::Future;
15934 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15935 const NEXT_HASH: &str =
15936 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
15937 let mut rig = rig().await;
15938 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15939 .await;
15940 let (client, mut rx, frame) =
15941 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
15942 for _ in 0..64 {
15943 client.egress.try_send(route_bind_ack(1)).unwrap();
15944 }
15945 let handler = rig.handler.clone();
15946 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15947 std::future::poll_fn(|cx| {
15948 assert!(open.as_mut().poll(cx).is_pending());
15949 std::task::Poll::Ready(())
15950 })
15951 .await;
15952 rig.sync(vec![
15953 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
15954 ])
15955 .await;
15956 rx.try_recv().unwrap();
15957 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15958 let bind = std::future::poll_fn(|cx| {
15959 assert!(open.as_mut().poll(cx).is_pending());
15960 module_rx.poll_recv(cx)
15961 })
15962 .await
15963 .unwrap()
15964 .frame;
15965 handler
15966 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
15967 .await
15968 .unwrap();
15969 let replies = tokio::time::timeout(Duration::from_secs(2), open)
15970 .await
15971 .unwrap()
15972 .unwrap();
15973 assert_eq!(replies.len(), 1);
15974 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
15977 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15978 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
15979 let route = rig
15980 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
15981 .await;
15982 assert!(rig.live(&route));
15983 }
15984
15985 #[tokio::test]
15986 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
15987 {
15988 let mut rig = rig_with_flow_support(false).await;
15989 rig.sync(vec![session(1)]).await;
15990 let route = rig
15991 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15992 .await;
15993 assert!(rig.live(&route));
15994 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
15995 let mut record = session(1);
15996 record.attributes.flow_id = Some("flow:7".to_string());
15997 rig.sync(vec![record]).await;
15998 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
15999 assert!(rig.live(&unscoped));
16000 assert_eq!(unscoped.stamp(), None);
16001 }
16002
16003 #[tokio::test]
16004 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
16005 let mut rig = rig().await;
16006 let mut record = session(1);
16007 record.attributes.flow_id = Some("flow:7".to_string());
16008 rig.sync(vec![record.clone()]).await;
16009 let before = rig.forwarding.published_scope_tag(OWNER, "s");
16010 let mut owner_route = rig
16011 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16012 .await;
16013 let mut carrier_route = rig
16014 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
16015 .await;
16016 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16017 rig.sync(vec![record.clone()]).await;
16018 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
16019 assert!(rig.live(&owner_route) && owner_route.untouched());
16020 assert!(rig.live(&carrier_route) && carrier_route.untouched());
16021
16022 record.attributes.flow_id = Some("flow:8".to_string());
16023 rig.sync(vec![record]).await;
16024 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
16025 let before = before.unwrap();
16026 assert_eq!(after.scope_epoch, before.scope_epoch);
16027 assert!(after.version > before.version);
16028 for route in [&mut owner_route, &mut carrier_route] {
16029 assert!(!rig.live(route));
16030 assert_eq!(
16031 route.closed_reason(),
16032 RouteCloseReason::ScopeDelegationChanged
16033 );
16034 }
16035 assert!(rig.live(&unscoped) && unscoped.untouched());
16036 for target in [PLEXUS, OTHER] {
16039 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
16040 let goodbye = module_rx
16041 .try_recv()
16042 .expect("the provider sees the drain")
16043 .frame;
16044 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16045 assert!(module_rx.try_recv().is_err());
16046 }
16047 let rebound = rig
16048 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16049 .await;
16050 assert_eq!(
16051 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
16052 Some("flow:8")
16053 );
16054 }
16055
16056 #[tokio::test]
16061 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
16062 let mut rig = rig().await;
16063 rig.sync(vec![session(1)]).await;
16064 let mut cotenant = rig
16065 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16066 .await;
16067
16068 let mut changed = session(1);
16069 changed.child_owners.push(Principal::Reserved {
16070 module_id: MAGIC.to_string(),
16071 });
16072 let mut ended = None;
16073 for (code, next) in [
16074 (error_codes::SCOPE_CHANGED, vec![changed]),
16075 (error_codes::SCOPE_ENDED, Vec::new()),
16076 ] {
16077 let relayed = rig
16078 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16079 .await;
16080 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
16081 ended = Some(next.is_empty());
16082 rig.sync(next).await;
16083 rig.ack(&relayed).await;
16084 let replies = relayed.task.await.unwrap();
16085 assert_eq!(replies.len(), 1, "{replies:?}");
16086 assert_eq!(parse_error(&replies[0])["code"], code);
16087 assert_eq!(
16088 subc_protocol::error_codes::is_retryable_route_open(code),
16089 code == error_codes::SCOPE_CHANGED
16090 );
16091 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16092 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16094 let mut goodbyes = Vec::new();
16097 while let Ok(outbound) = plexus_rx.try_recv() {
16098 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
16099 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
16100 }
16101 assert!(
16102 goodbyes.contains(&(bind_channel, bind_epoch)),
16103 "{goodbyes:?}"
16104 );
16105 assert!(rig
16106 .handler
16107 .registry
16108 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
16109 .unwrap()
16110 .is_some());
16111 }
16112 assert_eq!(ended, Some(true));
16113 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
16116 }
16117
16118 #[tokio::test]
16121 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
16122 struct Case {
16123 name: &'static str,
16124 change: fn(&mut ScopeRecord),
16125 closed: [Option<RouteCloseReason>; 4],
16128 }
16129 use RouteCloseReason::*;
16130 let cases = [
16131 Case {
16132 name: "a carrier entry removed",
16133 change: |r| {
16134 r.carriers.retain(|c| {
16135 c.principal
16136 != Principal::Reserved {
16137 module_id: AFT.to_string(),
16138 }
16139 })
16140 },
16141 closed: [None, Some(ScopeCarrierRemoved), None, None],
16142 },
16143 Case {
16144 name: "a target removed from a carrier",
16145 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
16146 closed: [None, None, None, Some(ScopeCarrierRemoved)],
16147 },
16148 Case {
16149 name: "a bare carrier narrowed to targets",
16150 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
16151 closed: [None, Some(ScopeCarrierRemoved), None, None],
16152 },
16153 Case {
16154 name: "delegates turned off",
16155 change: |r| r.attributes.delegates = false,
16156 closed: [Some(ScopeDelegationChanged); 4],
16157 },
16158 Case {
16159 name: "agent_id changed",
16160 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
16161 closed: [Some(ScopeDelegationChanged); 4],
16162 },
16163 Case {
16164 name: "a carrier added, child owners changed, the record re-sent",
16165 change: |r| {
16166 r.carriers.push(carrier(MAGIC, None));
16167 r.child_owners.push(Principal::Reserved {
16168 module_id: MAGIC.to_string(),
16169 });
16170 },
16171 closed: [None; 4],
16172 },
16173 Case {
16174 name: "a target added",
16175 change: |r| {
16176 r.carriers[1]
16177 .targets
16178 .as_mut()
16179 .unwrap()
16180 .push("third".to_string())
16181 },
16182 closed: [None; 4],
16183 },
16184 Case {
16185 name: "delegates turned on",
16186 change: |r| r.attributes.delegates = true,
16187 closed: [None; 4],
16188 },
16189 ];
16190 for case in cases {
16191 let mut rig = rig().await;
16192 rig.sync(vec![session(1)]).await;
16193 let scope = || Some(rig_selector("s", Some(1)));
16194 let mut routes = [
16195 rig.bound(Some(OWNER), PLEXUS, scope()).await,
16196 rig.bound(Some(AFT), PLEXUS, scope()).await,
16197 rig.bound(Some(BROCA), PLEXUS, scope()).await,
16198 rig.bound(Some(BROCA), OTHER, scope()).await,
16199 ];
16200 let mut record = session(1);
16201 (case.change)(&mut record);
16202 rig.sync(vec![record]).await;
16203 for (index, expected) in case.closed.iter().enumerate() {
16204 let route = &mut routes[index];
16205 match expected {
16206 Some(reason) => {
16207 assert!(!rig.live(route), "{}: route {index} still live", case.name);
16208 assert_eq!(
16209 route.closed_reason(),
16210 *reason,
16211 "{}: route {index}",
16212 case.name
16213 );
16214 }
16215 None => {
16216 assert!(rig.live(route), "{}: route {index} closed", case.name);
16217 assert!(
16218 route.untouched(),
16219 "{}: route {index} was told something",
16220 case.name
16221 );
16222 }
16223 }
16224 }
16225 }
16226 }
16227
16228 #[tokio::test]
16229 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
16230 for next in [Vec::new(), vec![session(2)]] {
16232 let mut rig = rig().await;
16233 rig.sync(vec![session(1)]).await;
16234 let mut route = rig
16235 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16236 .await;
16237 rig.sync(next).await;
16238 assert!(!rig.live(&route));
16239 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
16240 }
16241
16242 let mut rig = rig().await;
16245 let mut child = session(1);
16246 child.scope_ref = "child".to_string();
16247 child.kind = ScopeKind::Worker;
16248 child.parent = Some(ScopeParent::new(
16249 Principal::Reserved {
16250 module_id: OWNER.to_string(),
16251 },
16252 "s".to_string(),
16253 1,
16254 ));
16255 rig.sync(vec![session(1), child.clone()]).await;
16256 let mut child_route = rig
16257 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
16258 .await;
16259 assert_eq!(
16260 child_route.stamp().unwrap().parent_state,
16261 Some(ParentState::Linked)
16262 );
16263 rig.sync(vec![child]).await;
16264 assert!(!rig.live(&child_route));
16265 assert_eq!(
16266 child_route.closed_reason(),
16267 RouteCloseReason::ScopeParentEnded
16268 );
16269 }
16270
16271 #[tokio::test]
16272 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
16273 ) {
16274 let mut rig = rig().await;
16275 rig.sync(vec![session(1)]).await;
16276 let mut route = rig
16277 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16278 .await;
16279 let before = rig.forwarding.published_scope_tag(OWNER, "s");
16280 rig.sync(vec![session(1)]).await;
16281 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
16282 assert!(rig.live(&route) && route.untouched());
16283
16284 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
16288 .forwarding
16289 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16290 .unwrap()
16291 else {
16292 panic!("the route is bound");
16293 };
16294 binding.flow.acquire_tagged(9, false).await.unwrap();
16295 let mut widened = session(1);
16296 widened.carriers.push(carrier(MAGIC, None));
16297 rig.sync(vec![widened]).await;
16298 assert!(rig.live(&route) && route.untouched());
16299 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16300 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
16301 assert_eq!(binding.flow.in_flight(), 1);
16304 binding
16305 .flow
16306 .acquire_tagged(10, false)
16307 .await
16308 .expect("the flow is still open");
16309 }
16310
16311 #[tokio::test]
16314 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
16315 let mut rig = rig().await;
16316 rig.sync(vec![session(1)]).await;
16317 let mut on_incumbent = rig
16318 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16319 .await;
16320
16321 let (candidate, _candidate_rx) = wide_ctx(9);
16324 let registration = rig
16325 .handler
16326 .registry
16327 .register_candidate_with_control_ops(
16328 manifest(PLEXUS, PROTOCOL_VERSION),
16329 PROTOCOL_VERSION,
16330 candidate.connection_id,
16331 module_baseline_control_ops(),
16332 )
16333 .unwrap();
16334 rig.forwarding
16335 .register_candidate_module_connection(
16336 candidate.connection_id,
16337 PLEXUS.to_string(),
16338 PROTOCOL_VERSION,
16339 manifest_concurrency(®istration.manifest),
16340 candidate.egress.clone(),
16341 )
16342 .unwrap();
16343 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
16344 rig.handler
16345 .registry
16346 .promote_candidate(PLEXUS)
16347 .unwrap()
16348 .unwrap();
16349 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
16350
16351 rig.sync(Vec::new()).await;
16352 assert!(!rig.live(&on_incumbent));
16353 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
16354 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16355 let goodbye = incumbent_rx
16356 .try_recv()
16357 .expect("the superseded endpoint is told")
16358 .frame;
16359 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16360 }
16361 }
16362}
16363
16364#[cfg(test)]
16365mod concurrency_default_exposure_tests {
16366 use super::*;
16367
16368 fn hello_body(role_json: &str) -> Vec<u8> {
16369 format!(
16370 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
16371 )
16372 .into_bytes()
16373 }
16374
16375 fn manifest_from(body: &[u8]) -> ModuleManifest {
16376 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
16377 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
16378 .expect("manifest parses")
16379 }
16380
16381 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
16382
16383 #[test]
16384 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
16385 let body = hello_body(&format!(
16386 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
16387 ));
16388 let manifest = manifest_from(&body);
16389 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16392 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
16393 }
16394
16395 #[test]
16396 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
16397 let body = hello_body(&format!(
16398 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
16399 ));
16400 let manifest = manifest_from(&body);
16401 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16402 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16403 }
16404
16405 #[test]
16406 fn non_management_roles_are_never_reported() {
16407 let body = hello_body(
16408 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
16409 );
16410 let manifest = manifest_from(&body);
16411 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16412 }
16413}