1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_event_declarations,
25 validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
26 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
27 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
28 },
29 scope::{
30 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
31 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
32 SCOPE_SYNC_OP,
33 },
34 session::{
35 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
36 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
37 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
38 ROLE_VERSIONS_FIELD,
39 },
40 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
41 Priority, RouteTarget, PROTOCOL_VERSION,
42};
43use tokio::time::{timeout_at, Instant};
44use tracing::{debug, info, warn};
45
46use crate::{
47 caller_identity::CallerIdentity,
48 capability_requirements::{
49 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
50 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
51 RuntimeModule,
52 },
53 child_roster::DaemonShutdownFlag,
54 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
55 forwarding::{
56 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
57 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
58 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
59 },
60 observability::{
61 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
62 },
63 privileged_prompt::{prompt_text, PrivilegedOperation},
64 provenance::{
65 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
66 },
67 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
68 router::{RouteCtx, RouterError},
69 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
70 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
71 stderr_tail::{CaptureState, TailEntry},
72 supervise::{
73 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
74 SupervisorHandle, SwapHelloAdmission,
75 },
76 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
77};
78
79pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
85
86const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
87const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
88const CAP_PING_PONG: &str = "ping_pong_v1";
89const CAP_SESSION_ATTACH: &str = "session_attach_v1";
90const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
91
92const SUBC_CONTROL_OPS: &[&str] = &[
93 ops::ACTIVITY_RECORD,
94 ops::ACTIVITY_QUERY,
95 ops::ACTIVITY_SUBSCRIBE,
96 ops::SERVER_DESCRIBE,
97 ops::CATALOG_LIST,
98 ops::ROUTE_OPEN,
99 ops::ROUTE_POLL,
100 ops::ROUTE_CLOSING,
101 ops::ROUTE_CLOSED,
102 ops::SUPERVISOR_LIST,
103 ops::SUPERVISOR_RESTART,
104 ops::SUPERVISOR_SWAP,
105 ops::SUPERVISOR_RELOAD,
106 ops::SUPERVISOR_RESCAN,
107 ops::SUPERVISOR_RELEASE_RESERVED,
108 ops::SUPERVISOR_SET_ENABLED,
109 ops::SUPERVISOR_HEALTH_PROBE,
110 ops::SUPERVISOR_HEALTH,
111 ops::SUPERVISOR_STDERR_TAIL,
112 ops::SUPERVISOR_TERMINALS,
113 ops::SUPERVISOR_ROUTES,
114 ops::SUPERVISOR_PROVENANCE,
115 ops::SUPERVISOR_SPAWN_SNAPSHOT,
116 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
117];
118
119const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
126
127mod rescan_checked;
128use rescan_checked::{RescanMode, RescanPlan};
129
130const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
131 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
132 "supervisor.live_roots",
133 SCOPE_SYNC_OP,
134 SCOPE_APPLY_OP,
135 SCOPE_DESCRIBE_OP,
136 "operator.confirm",
137];
138
139const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
143
144const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
145
146pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
154
155pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
170
171pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
189
190const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
191const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
192
193fn reload_verdict(
194 configured: &Path,
195 spawned_from: Option<&Path>,
196 image: subc_control::RunningImageAgreement,
197) -> PendingReloadVerdict {
198 let path = match spawned_from {
199 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
200 ReloadPathAgreement::Match
201 }
202 Some(spawned_from) => ReloadPathAgreement::Mismatch {
203 configured: configured.to_path_buf(),
204 spawned_from: spawned_from.to_path_buf(),
205 },
206 None => ReloadPathAgreement::Unavailable {
207 reason: if matches!(
208 image,
209 subc_control::RunningImageAgreement::Unavailable {
210 reason: subc_control::RunningImageUnavailableReason::NotRunning
211 }
212 ) {
213 ReloadPathUnavailableReason::NotRunning
214 } else {
215 ReloadPathUnavailableReason::SpawnedPathUnavailable
216 },
217 },
218 };
219 PendingReloadVerdict {
220 path,
221 image,
222 spec: None,
223 }
224}
225
226fn launch_spec_agreement(
227 configured: &crate::supervise::ModuleSpec,
228 running: bool,
229 spawned: Option<&crate::supervise::LaunchSpecDigest>,
230) -> subc_control::LaunchSpecAgreement {
231 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
232 if !running {
233 return Agreement::Unavailable {
234 reason: Reason::NotRunning,
235 };
236 }
237 match spawned {
238 None => Agreement::Unavailable {
239 reason: Reason::SpawnedSpecUnavailable,
240 },
241 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
242 Agreement::Match
243 }
244 Some(_) => Agreement::Mismatch,
245 }
246}
247
248fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
249 if configured == spawned_from {
250 return true;
251 }
252 #[cfg(windows)]
253 {
254 match (
260 std::fs::canonicalize(configured),
261 std::fs::canonicalize(spawned_from),
262 ) {
263 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
264 _ => false,
265 }
266 }
267 #[cfg(not(windows))]
268 {
269 false
270 }
271}
272
273#[derive(Clone)]
274struct DaemonProvenanceFacts {
275 build: DaemonBuildProvenance,
276 pid: Option<u32>,
277 started_at_ms: Option<u64>,
278 start_clock: Option<crate::clock::StartClock>,
279 executable_path: Option<PathBuf>,
280 executable_identity: Option<SpawnedFileIdentity>,
281 process_start_time: Option<u64>,
282 probe: ExecutableIdentityProbe,
283}
284
285impl Default for DaemonProvenanceFacts {
286 fn default() -> Self {
287 Self {
288 build: DaemonBuildProvenance {
289 build_git_sha: None,
290 build_lock_digest: None,
291 },
292 pid: None,
293 started_at_ms: None,
294 start_clock: None,
295 executable_path: None,
296 executable_identity: None,
297 process_start_time: None,
298 probe: ExecutableIdentityProbe::default(),
299 }
300 }
301}
302
303#[derive(Debug, Clone)]
304struct SupervisorRescanContext {
305 supervisor: Supervisor,
306 config_path: PathBuf,
307 configured_port: Option<u16>,
308 storage_config: Option<crate::daemon_config::StorageConfig>,
309 admission_facts_carrier_module_id: Option<String>,
310 admission_facts_targets: Option<Vec<String>>,
311 scope_authority_owners: Vec<String>,
312 privileged_operator_prompt: PrivilegedOperatorPrompt,
313 generation: Arc<std::sync::atomic::AtomicU64>,
319}
320
321const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
342 "reloading",
343 "supervisor_not_live",
344 "registration_not_active",
345 "no_forwarding_connection",
346 "relay_send_failed",
347];
348
349#[derive(Clone)]
351pub struct ControlHandler {
352 registry: Arc<Registry>,
353 activity: Option<crate::activity::ActivityLedger>,
354 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
355 forwarding: Arc<ForwardingTable>,
356 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
357 supervisor: SupervisorHandle,
358 subc_capabilities: Arc<[String]>,
359 route_bind_relay_timeout: Duration,
363 route_bind_relay_timeouts: BTreeMap<String, Duration>,
367 route_bind_breakers: RouteBindBreakers,
370 route_bind_concurrency: RouteBindConcurrency,
373 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
378 route_bind_breaker_threshold: u32,
380 route_bind_breaker_cooldown: Duration,
382 health_probe_timeout: Duration,
383 storage_config: Option<crate::daemon_config::StorageConfig>,
386 machine_id: Option<crate::machine_id::MachineId>,
390 admission_facts_carrier_module_id: Option<String>,
391 admission_facts_targets: Option<Vec<String>>,
392 scopes: Arc<RwLock<ScopeTable>>,
396 scope_deadline_added: Arc<tokio::sync::Notify>,
400 scope_authority_owners: Vec<String>,
403 privileged_operator_prompt: PrivilegedOperatorPrompt,
405 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
408 #[cfg(test)]
409 caller_lookup: Option<TestCallerLookup>,
410 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
413 rescan: Option<SupervisorRescanContext>,
414 connected_clients: ConnectedClients,
415 counters: DaemonCounters,
416 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
417 daemon_shutdown: DaemonShutdownFlag,
422 daemon_provenance: DaemonProvenanceFacts,
423 #[cfg(test)]
424 control_dispatch_delay: Option<Duration>,
425 #[cfg(test)]
426 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
427 #[cfg(test)]
431 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
432 #[cfg(test)]
438 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
439}
440
441impl fmt::Debug for ControlHandler {
442 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
443 f.debug_struct("ControlHandler")
444 .field("registry", &self.registry)
445 .field("forwarding", &self.forwarding)
446 .field("process_liveness", &self.process_liveness.is_some())
447 .field("supervisor", &self.supervisor)
448 .field("subc_capabilities", &self.subc_capabilities)
449 .finish()
450 }
451}
452
453struct RouteOpenRequest {
454 target: RouteTarget,
455 identity: BindIdentity,
456 consumer_identity: Option<ConsumerIdentity>,
457 consumer_capabilities: Option<Vec<String>>,
458 role_versions: Option<BTreeMap<String, String>>,
459 admission_facts: Option<serde_json::Value>,
460 scope: Option<ScopeSelector>,
461}
462
463#[derive(Clone, Copy)]
464enum RouteOpenPhase {
465 Admission,
466 ProjectRoot,
467 EgressReservation,
468 BindSend,
469 BindAckWait,
470}
471
472struct RouteOpenTimings {
476 module_id: String,
477 principal: String,
478 admission: Duration,
479 project_root: Duration,
480 egress_reservation: Duration,
481 bind_send: Duration,
482 bind_ack_wait: Duration,
483 phase: Option<RouteOpenPhase>,
484 phase_started_at: Instant,
485}
486
487impl RouteOpenTimings {
488 fn new() -> Self {
489 Self {
490 module_id: String::new(),
491 principal: "unresolved".to_string(),
493 admission: Duration::ZERO,
494 project_root: Duration::ZERO,
495 egress_reservation: Duration::ZERO,
496 bind_send: Duration::ZERO,
497 bind_ack_wait: Duration::ZERO,
498 phase: Some(RouteOpenPhase::Admission),
499 phase_started_at: Instant::now(),
500 }
501 }
502
503 fn finish(&mut self) {
504 let elapsed = self.phase_started_at.elapsed();
505 match self.phase.take() {
506 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
507 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
508 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
509 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
510 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
511 None => {}
512 }
513 }
514
515 fn enter(&mut self, phase: RouteOpenPhase) {
516 self.finish();
517 self.phase = Some(phase);
518 self.phase_started_at = Instant::now();
519 }
520}
521
522struct RouteBindReservationGuard {
523 forwarding: Arc<ForwardingTable>,
524 endpoint: ModuleEndpointId,
525 relay_corr: u64,
526 armed: bool,
527}
528
529struct ModuleControlRpcGuard {
530 forwarding: Arc<ForwardingTable>,
531 endpoint: ModuleEndpointId,
532 corr: u64,
533 armed: bool,
534}
535
536impl ModuleControlRpcGuard {
537 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
538 Self {
539 forwarding,
540 endpoint,
541 corr,
542 armed: true,
543 }
544 }
545
546 fn disarm(&mut self) {
547 self.armed = false;
548 }
549}
550
551impl Drop for ModuleControlRpcGuard {
552 fn drop(&mut self) {
553 if self.armed {
554 let _ = self
555 .forwarding
556 .cancel_module_control_rpc(self.endpoint, self.corr);
557 }
558 }
559}
560
561impl RouteBindReservationGuard {
562 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
563 Self {
564 forwarding,
565 endpoint,
566 relay_corr,
567 armed: true,
568 }
569 }
570
571 fn release_and_disarm(&mut self) {
572 if !self.armed {
573 return;
574 }
575 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
576 self.endpoint,
577 self.relay_corr,
578 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
579 ) {
580 send_goodbye_target_best_effort(
581 &self.forwarding.counters(),
582 &target,
583 "canceled route.bind",
584 );
585 }
586 self.armed = false;
587 }
588
589 fn disarm(&mut self) {
590 self.armed = false;
591 }
592}
593
594impl Drop for RouteBindReservationGuard {
595 fn drop(&mut self) {
596 self.release_and_disarm();
597 }
598}
599
600#[derive(Debug, Clone, Default)]
630pub(crate) struct RouteBindBreakers {
631 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
632}
633
634#[derive(Debug, Clone, Default)]
635pub(crate) struct RouteBindConcurrency {
636 modules: Arc<Mutex<HashMap<String, usize>>>,
637}
638
639struct RouteBindConcurrencyGuard {
640 concurrency: RouteBindConcurrency,
641 module_id: String,
642}
643
644impl RouteBindConcurrency {
645 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
648 let mut modules = self
649 .modules
650 .lock()
651 .expect("route.bind concurrency mutex poisoned");
652 let in_flight = modules.entry(module_id.to_string()).or_default();
653 if *in_flight >= limit {
654 return Err(*in_flight);
655 }
656 *in_flight += 1;
657 Ok(RouteBindConcurrencyGuard {
658 concurrency: self.clone(),
659 module_id: module_id.to_string(),
660 })
661 }
662}
663
664impl Drop for RouteBindConcurrencyGuard {
665 fn drop(&mut self) {
666 let mut modules = self
667 .concurrency
668 .modules
669 .lock()
670 .expect("route.bind concurrency mutex poisoned");
671 let remove = {
672 let in_flight = modules
673 .get_mut(&self.module_id)
674 .expect("admitted route.bind has a concurrency entry");
675 *in_flight -= 1;
676 *in_flight == 0
677 };
678 if remove {
679 modules.remove(&self.module_id);
680 }
681 }
682}
683
684#[derive(Debug, Default)]
685struct ModuleBreakerState {
686 consecutive_timeouts: u32,
688 cooldown_until: Option<Instant>,
691 probe_in_flight: Option<Arc<()>>,
696}
697
698enum RouteBindAdmission<'a> {
700 Admitted {
701 guard: RouteBindBreakerGuard<'a>,
702 probe: bool,
705 },
706 Refused {
707 consecutive_timeouts: u32,
708 retry_in: Duration,
712 probe_in_flight: bool,
713 },
714}
715
716struct RouteBindBreakerGuard<'a> {
723 breakers: RouteBindBreakers,
724 module_id: &'a str,
725 probe_token: Option<Arc<()>>,
726 settled: bool,
727}
728
729impl RouteBindBreakerGuard<'_> {
730 fn record_accepted(&mut self) -> bool {
734 self.settled = true;
735 self.breakers.record_accepted(self.module_id)
736 }
737
738 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
741 self.settled = true;
742 self.breakers.record_timeout(
743 self.module_id,
744 self.probe_token.as_ref(),
745 threshold,
746 cooldown,
747 )
748 }
749
750 fn record_inconclusive(&mut self) {
759 self.settled = true;
760 self.breakers
761 .record_inconclusive(self.module_id, self.probe_token.as_ref());
762 }
763}
764
765impl Drop for RouteBindBreakerGuard<'_> {
766 fn drop(&mut self) {
767 if !self.settled {
768 self.breakers
769 .record_inconclusive(self.module_id, self.probe_token.as_ref());
770 }
771 }
772}
773
774struct BreakerOpened {
778 consecutive_timeouts: u32,
779 reopened_after_probe: bool,
782}
783
784impl RouteBindBreakers {
785 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
786 self.modules
787 .lock()
788 .expect("route.bind breaker mutex poisoned")
789 }
790
791 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
794 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
795 probe: probe_token.is_some(),
796 guard: RouteBindBreakerGuard {
797 breakers: self.clone(),
798 module_id,
799 probe_token,
800 settled: false,
801 },
802 };
803
804 let mut modules = self.lock();
805 let Some(state) = modules.get_mut(module_id) else {
806 return admitted(None);
807 };
808 let Some(cooldown_until) = state.cooldown_until else {
809 return admitted(None);
810 };
811 if state.probe_in_flight.is_some() {
812 return RouteBindAdmission::Refused {
813 consecutive_timeouts: state.consecutive_timeouts,
814 retry_in: Duration::ZERO,
815 probe_in_flight: true,
816 };
817 }
818 let now = Instant::now();
819 if now < cooldown_until {
820 return RouteBindAdmission::Refused {
821 consecutive_timeouts: state.consecutive_timeouts,
822 retry_in: cooldown_until - now,
823 probe_in_flight: false,
824 };
825 }
826 let token = Arc::new(());
827 state.probe_in_flight = Some(Arc::clone(&token));
828 admitted(Some(token))
829 }
830
831 fn record_accepted(&self, module_id: &str) -> bool {
832 self.lock()
833 .remove(module_id)
834 .is_some_and(|state| state.cooldown_until.is_some())
835 }
836
837 fn record_timeout(
838 &self,
839 module_id: &str,
840 probe_token: Option<&Arc<()>>,
841 threshold: u32,
842 cooldown: Duration,
843 ) -> Option<BreakerOpened> {
844 let mut modules = self.lock();
845 let state = modules.entry(module_id.to_string()).or_default();
846 let was_open = state.cooldown_until.is_some();
847 let was_probe = Self::owns_probe(state, probe_token);
848 if was_probe {
849 state.probe_in_flight = None;
850 }
851 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
852 if state.consecutive_timeouts < threshold {
853 return None;
854 }
855 state.cooldown_until = Some(Instant::now() + cooldown);
856 Some(BreakerOpened {
857 consecutive_timeouts: state.consecutive_timeouts,
858 reopened_after_probe: was_open && was_probe,
859 })
860 }
861
862 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
863 state
867 .probe_in_flight
868 .as_ref()
869 .zip(token)
870 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
871 }
872
873 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
874 if let Some(state) = self.lock().get_mut(module_id) {
875 if Self::owns_probe(state, probe_token) {
876 state.probe_in_flight = None;
877 }
878 }
879 }
880
881 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
896 self.lock()
897 .remove(module_id)
898 .map(|state| state.consecutive_timeouts)
899 .filter(|discarded| *discarded > 0)
900 }
901
902 fn open_snapshot(&self) -> Option<serde_json::Value> {
909 let now = Instant::now();
910 let modules = self.lock();
911 let open = modules
912 .iter()
913 .filter_map(|(module_id, state)| {
914 let cooldown_until = state.cooldown_until?;
915 Some((
916 module_id.clone(),
917 serde_json::json!({
918 "consecutive_timeouts": state.consecutive_timeouts,
919 "cooldown_remaining_ms":
920 cooldown_until.saturating_duration_since(now).as_millis() as u64,
921 "probe_in_flight": state.probe_in_flight.is_some(),
922 }),
923 ))
924 })
925 .collect::<serde_json::Map<String, serde_json::Value>>();
926 (!open.is_empty()).then_some(serde_json::Value::Object(open))
927 }
928}
929
930impl ControlHandler {
931 pub fn new(registry: Arc<Registry>) -> Self {
932 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
933 }
934
935 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
936 let counters = forwarding.counters();
937 let route_bind_breakers = forwarding.route_bind_breakers();
941 let route_bind_concurrency = forwarding.route_bind_concurrency();
942 let route_outages = forwarding.route_outages();
943 Self {
944 registry,
945 activity: None,
946 activity_streams: Arc::new(Mutex::new(HashMap::new())),
947 forwarding,
948 process_liveness: None,
949 supervisor: SupervisorHandle::new(),
950 subc_capabilities: Arc::from([
951 CAP_MANIFEST_REGISTRATION.to_string(),
952 CAP_CHANNEL_LIFECYCLE.to_string(),
953 CAP_PING_PONG.to_string(),
954 CAP_SESSION_ATTACH.to_string(),
955 CAP_ADMISSION_FACTS_RELAY.to_string(),
956 CAP_SCOPES_V1.to_string(),
957 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
958 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
959 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
960 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
961 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
962 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
963 subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
964 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
965 subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
966 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
967 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
968 ]),
969 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
970 route_bind_relay_timeouts: BTreeMap::new(),
971 route_bind_breakers,
972 route_bind_concurrency,
973 route_outages,
974 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
975 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
976 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
977 storage_config: None,
978 machine_id: None,
979 admission_facts_carrier_module_id: None,
980 admission_facts_targets: None,
981 scopes: Arc::new(RwLock::new(ScopeTable::new(
982 crate::daemon_config::default_scope_authority_owners(),
983 ))),
984 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
985 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
986 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
987 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
988 #[cfg(test)]
989 caller_lookup: None,
990 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
991 rescan: None,
992 connected_clients: ConnectedClients::new(),
993 counters,
994 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
995 daemon_shutdown: DaemonShutdownFlag::default(),
996 daemon_provenance: DaemonProvenanceFacts::default(),
997 #[cfg(test)]
998 control_dispatch_delay: None,
999 #[cfg(test)]
1000 provenance_probe_override: None,
1001 #[cfg(test)]
1002 route_bind_relay_sent_hold: None,
1003 #[cfg(test)]
1004 hello_registry_to_forwarding_hold: None,
1005 }
1006 }
1007
1008 pub fn with_storage_config(
1011 mut self,
1012 storage_config: Option<crate::daemon_config::StorageConfig>,
1013 ) -> Self {
1014 self.storage_config = storage_config;
1015 self
1016 }
1017
1018 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1021 self.activity = Some(crate::activity::ActivityLedger::open(
1022 run_dir.join("activity.jsonl"),
1023 incarnation,
1024 ));
1025 self.supervisor.set_activity(self.activity.clone());
1026 self
1027 }
1028
1029 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1032 self.machine_id = machine_id;
1033 self
1034 }
1035
1036 pub fn with_admission_facts_config(
1040 mut self,
1041 carrier_module_id: Option<String>,
1042 targets: Option<Vec<String>>,
1043 ) -> Self {
1044 self.admission_facts_carrier_module_id = carrier_module_id;
1045 self.admission_facts_targets = targets;
1046 self
1047 }
1048
1049 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1053 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1054 self.scope_authority_owners = owners;
1055 self
1056 }
1057
1058 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1060 self.privileged_operator_prompt = mode;
1061 self
1062 }
1063
1064 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1067 self.route_bind_relay_timeout = timeout;
1068 self
1069 }
1070
1071 pub fn with_route_bind_relay_timeouts(
1077 mut self,
1078 timeouts: impl IntoIterator<Item = (String, Duration)>,
1079 ) -> Self {
1080 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1081 self
1082 }
1083
1084 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1090 self.route_bind_relay_timeouts
1091 .get(module_id)
1092 .copied()
1093 .unwrap_or(self.route_bind_relay_timeout)
1094 }
1095
1096 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1104 self.route_bind_breaker_threshold = threshold.max(1);
1105 self.route_bind_breaker_cooldown = cooldown;
1106 self
1107 }
1108
1109 #[cfg(test)]
1110 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1111 self.health_probe_timeout = timeout;
1112 self
1113 }
1114
1115 #[cfg(test)]
1116 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1117 self.control_dispatch_delay = Some(delay);
1118 self
1119 }
1120
1121 #[cfg(test)]
1125 pub(crate) fn with_route_bind_relay_sent_hold(
1126 mut self,
1127 hold: Arc<tokio::sync::Barrier>,
1128 ) -> Self {
1129 self.route_bind_relay_sent_hold = Some(hold);
1130 self
1131 }
1132
1133 #[cfg(test)]
1138 pub(crate) fn with_hello_registry_to_forwarding_hold(
1139 mut self,
1140 hold: Arc<std::sync::Barrier>,
1141 ) -> Self {
1142 self.hello_registry_to_forwarding_hold = Some(hold);
1143 self
1144 }
1145
1146 #[cfg(test)]
1147 fn hold_between_registry_and_forwarding(&self) {
1148 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1149 hold.wait();
1150 hold.wait();
1151 }
1152 }
1153
1154 pub fn with_process_liveness(
1155 mut self,
1156 process_liveness: Arc<dyn ModuleProcessLiveness>,
1157 ) -> Self {
1158 self.process_liveness = Some(process_liveness);
1159 self
1160 }
1161
1162 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1163 self.supervisor = supervisor;
1164 self.supervisor.set_activity(self.activity.clone());
1165 self
1166 }
1167
1168 pub fn with_daemon_provenance(
1169 mut self,
1170 pid: u32,
1171 started_at_ms: u64,
1172 executable_path: Option<PathBuf>,
1173 build_git_sha: Option<String>,
1174 build_lock_digest: Option<String>,
1175 ) -> Self {
1176 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1177 let process_start_time = process_start_time(pid);
1178 self.daemon_provenance = DaemonProvenanceFacts {
1179 build: DaemonBuildProvenance {
1180 build_git_sha,
1181 build_lock_digest,
1182 },
1183 pid: Some(pid),
1184 started_at_ms: Some(started_at_ms),
1185 start_clock: None,
1186 executable_path,
1187 executable_identity,
1188 process_start_time,
1189 probe: ExecutableIdentityProbe::default(),
1190 };
1191 self
1192 }
1193
1194 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1195 self.daemon_provenance.start_clock = Some(clock);
1196 self
1197 }
1198
1199 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1201 self.scopes
1202 .write()
1203 .unwrap_or_else(|poisoned| poisoned.into_inner())
1204 .set_wall_clock(Arc::new(clock));
1205 self
1206 }
1207
1208 #[cfg(test)]
1209 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1210 self.provenance_probe_override = Some(result);
1211 self
1212 }
1213
1214 pub fn with_capability_config(
1218 self,
1219 modules: impl IntoIterator<Item = (String, bool)>,
1220 reserved_capabilities: BTreeMap<String, String>,
1221 ) -> Self {
1222 self.capability_evaluator
1223 .configure(modules, reserved_capabilities);
1224 self
1225 }
1226
1227 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1231 self.daemon_shutdown = flag;
1232 self
1233 }
1234
1235 pub fn with_supervisor_rescan(
1236 mut self,
1237 supervisor: Supervisor,
1238 config_path: impl Into<PathBuf>,
1239 configured_port: Option<u16>,
1240 ) -> Self {
1241 self.rescan = Some(SupervisorRescanContext {
1242 supervisor,
1243 config_path: config_path.into(),
1244 configured_port,
1245 storage_config: self.storage_config.clone(),
1246 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1247 admission_facts_targets: self.admission_facts_targets.clone(),
1248 scope_authority_owners: self.scope_authority_owners.clone(),
1249 privileged_operator_prompt: self.privileged_operator_prompt,
1250 generation: Arc::default(),
1251 });
1252 self
1253 }
1254
1255 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1256 self.connected_clients = connected_clients;
1257 self
1258 }
1259
1260 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1261 Arc::clone(&self.forwarding)
1262 }
1263
1264 pub(crate) fn counters(&self) -> DaemonCounters {
1265 self.counters.clone()
1266 }
1267
1268 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1271 tokio::spawn(async move {
1272 loop {
1273 self.capability_evaluator
1274 .wait_for_change_or_deadline()
1275 .await;
1276 self.refresh_capability_requirements();
1277 }
1278 });
1279 }
1280
1281 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1287 tokio::spawn(async move {
1288 loop {
1289 let has_deadlines = self
1290 .scopes
1291 .read()
1292 .unwrap_or_else(|poisoned| poisoned.into_inner())
1293 .has_deadlines();
1294 if !has_deadlines {
1295 self.scope_deadline_added.notified().await;
1296 continue;
1297 }
1298 tokio::time::sleep(Duration::from_secs(1)).await;
1299 if let Err(error) = self.sweep_expired_scopes() {
1300 warn!(%error, "scope expiry sweep failed");
1301 }
1302 }
1303 });
1304 }
1305
1306 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1307 let mut table = self
1308 .scopes
1309 .write()
1310 .unwrap_or_else(|poisoned| poisoned.into_inner());
1311 let swept = table.sweep_expired();
1312 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1313 drop(table);
1314 self.close_scope_drained_routes(drained);
1315 Ok(())
1316 }
1317
1318 fn publish_scope_changes(
1319 &self,
1320 changes: &[crate::scopes::ScopeTagChange],
1321 expired: &[crate::scopes::ScopeExpired],
1322 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1323 if changes.is_empty() {
1324 return Ok(Vec::new());
1325 }
1326 let drained = self
1327 .forwarding
1328 .publish_scope_changes(changes)
1329 .map_err(RouterError::Forwarding)?;
1330 let mut counts = HashMap::new();
1331 for route in &drained {
1332 *counts
1333 .entry((
1334 route.scope.owner.as_str(),
1335 route.scope.scope_ref.as_str(),
1336 route.scope.tag.scope_epoch,
1337 ))
1338 .or_insert(0usize) += 1;
1339 }
1340 for scope in expired {
1341 let routes_closed = counts
1342 .get(&(
1343 scope.owner.as_str(),
1344 scope.scope_ref.as_str(),
1345 scope.scope_epoch,
1346 ))
1347 .copied()
1348 .unwrap_or(0);
1349 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1350 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1351 routes_closed, "scope expired");
1352 }
1353 Ok(drained)
1354 }
1355
1356 fn runtime_capability_snapshot(
1357 &self,
1358 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1359 let runtime = self
1360 .supervisor
1361 .list()
1362 .into_iter()
1363 .map(|module| {
1364 let status = module.status().map_err(|err| {
1365 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1366 })?;
1367 Ok(RuntimeModule {
1368 module_id: status.module_id,
1369 state: status.state,
1370 enabled: status.enabled,
1371 })
1372 })
1373 .collect::<Result<Vec<_>, RouterError>>()?;
1374 let registrations = self.registry.list_registered_modules().map_err(|err| {
1380 RouterError::backend(
1381 0,
1382 0,
1383 format!("failed to list capability registrations: {err}"),
1384 )
1385 })?;
1386 let registrations = registrations
1387 .into_iter()
1388 .map(|registration| RegisteredModule {
1389 module_id: registration.manifest.module_id,
1390 module_version: registration.manifest.module_version,
1391 capabilities: registration.manifest.capabilities,
1392 })
1393 .collect();
1394 Ok((runtime, registrations))
1395 }
1396
1397 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1404 let cached_registration = RegisteredModule {
1405 module_id: registration.manifest.module_id.clone(),
1406 module_version: registration.manifest.module_version.clone(),
1407 capabilities: registration.manifest.capabilities.clone(),
1408 };
1409 if self.capability_evaluator.record_hello(&cached_registration) {
1410 warn!(
1411 module_id = %cached_registration.module_id,
1412 "capability claims drifted from the cached manifest"
1413 );
1414 }
1415 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1416 self.enforce_capability_denies();
1417 }
1418 self.refresh_capability_requirements();
1419 }
1420
1421 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1425 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1426 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1427 self.supervisor.set_swap_promotion_observer(observer);
1428 }
1429
1430 pub fn refresh_capability_requirements(&self) {
1431 match self.runtime_capability_snapshot() {
1432 Ok((runtime, registrations)) => {
1433 log_requirement_events(
1434 self.capability_evaluator
1435 .evaluate_now(&runtime, ®istrations),
1436 &self.daemon_shutdown,
1437 );
1438 }
1439 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1440 }
1441 }
1442
1443 fn enforce_capability_denies(&self) {
1447 let registrations = match self.registry.list_registered_modules() {
1448 Ok(snapshot) => snapshot,
1449 Err(err) => {
1450 warn!(error = %err, "failed to read registrations for capability deny census");
1451 return;
1452 }
1453 };
1454 let manifests = registrations
1455 .into_iter()
1456 .map(|registration| {
1457 (
1458 registration.manifest.module_id.clone(),
1459 registration.manifest,
1460 )
1461 })
1462 .collect::<BTreeMap<_, _>>();
1463 let census = match self.forwarding.route_census(None) {
1464 Ok(census) => census,
1465 Err(err) => {
1466 warn!(error = %err, "failed to read route census for capability deny enforcement");
1467 return;
1468 }
1469 };
1470
1471 for (target_module_id, routes) in census {
1472 let Some(target_manifest) = manifests.get(&target_module_id) else {
1473 continue;
1474 };
1475 let mut closed_routes = Vec::new();
1476 let mut goodbyes = Vec::new();
1477 for route in routes {
1478 let Principal::Reserved {
1479 module_id: opening_module_id,
1480 } = &route.principal
1481 else {
1482 continue;
1483 };
1484 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1485 continue;
1486 };
1487 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1488 continue;
1489 };
1490
1491 match self.forwarding.release_client_route(
1492 route.goodbye_target.connection_id,
1493 route.goodbye_target.channel,
1494 route.goodbye_target.epoch,
1495 ) {
1496 Ok(RouteRelease::Removed(module_goodbye)) => {
1497 warn!(
1498 opening_module_id,
1499 target_module_id,
1500 capability,
1501 "force-closing route because an attested capability deny edge now matches"
1502 );
1503 goodbyes.push(module_goodbye);
1504 goodbyes.push(route.goodbye_target.clone());
1505 closed_routes.push(route);
1506 }
1507 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1508 Err(err) => warn!(
1509 opening_module_id,
1510 target_module_id,
1511 capability,
1512 error = %err,
1513 "failed to force-close capability-denied route"
1514 ),
1515 }
1516 }
1517
1518 if closed_routes.is_empty() {
1519 continue;
1520 }
1521 send_route_control_pushes(
1522 &self.forwarding,
1523 closed_routes,
1524 ClientControlPush::RouteClosed {
1525 module_id: target_module_id,
1526 channels: Vec::new(),
1527 reason: RouteCloseReason::CapabilityDenied,
1528 drained: false,
1529 abandoned: 0,
1530 excluded_subscriptions: 0,
1531 terminal: Some(false),
1532 },
1533 );
1534 self.emit_route_goodbyes(goodbyes);
1537 }
1538 }
1539
1540 fn not_ready_reason(
1546 &self,
1547 registration: &crate::registry::ModuleRegistration,
1548 ) -> Option<NotReadyReason> {
1549 if !registration.ready {
1550 return Some(NotReadyReason {
1551 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1552 capability: None,
1553 });
1554 }
1555 self.first_unprovided_required_capability(registration)
1556 .map(|capability| NotReadyReason {
1557 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1558 capability: Some(capability),
1559 })
1560 }
1561
1562 fn first_unprovided_required_capability(
1580 &self,
1581 registration: &crate::registry::ModuleRegistration,
1582 ) -> Option<String> {
1583 let required = registration
1584 .manifest
1585 .capabilities
1586 .iter()
1587 .flat_map(|declarations| declarations.requires.iter())
1588 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1589 .map(|requirement| requirement.capability.as_str())
1590 .collect::<BTreeSet<_>>();
1591 if required.is_empty() {
1592 return None;
1593 }
1594 let module_id = registration.manifest.module_id.as_str();
1595 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1596 if required
1597 .iter()
1598 .any(|capability| verdict(capability).is_none())
1599 {
1600 self.refresh_capability_requirements();
1601 }
1602 required
1603 .into_iter()
1604 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1605 .map(str::to_string)
1606 }
1607
1608 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1609 self.capability_evaluator
1610 .statuses()
1611 .into_iter()
1612 .map(capability_requirement_status)
1613 .collect()
1614 }
1615
1616 fn deregister_connection(
1623 &self,
1624 connection_id: ConnectionId,
1625 reason: RegistrationEndReason,
1626 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1627 self.registry
1628 .deregister_connection_with_reason(connection_id, reason)
1629 }
1630
1631 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1632 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1633 return None;
1634 }
1635 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1636 parse_client_control_request(&frame.body)
1637 else {
1638 return None;
1639 };
1640 Some(target_module_id(&target).to_string())
1641 }
1642
1643 pub(crate) fn route_open_capacity_refusal(
1644 &self,
1645 ctx: &RouteCtx,
1646 frame: &Frame,
1647 target_module_id: &str,
1648 in_flight: usize,
1649 limit: usize,
1650 ) -> Result<Frame, RouterError> {
1651 let caller = self.route_open_caller_from_frame(frame);
1654 self.route_open_admission_refusal_frame(
1655 ctx,
1656 &caller,
1657 frame,
1658 target_module_id,
1659 "open_admission_full",
1660 (in_flight, limit),
1661 format!(
1662 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1663 ),
1664 )
1665 }
1666
1667 fn route_open_target_capacity_refusal(
1668 &self,
1669 ctx: &RouteCtx,
1670 caller: &RouteOpenCaller,
1671 frame: &Frame,
1672 target_module_id: &str,
1673 in_flight: usize,
1674 ) -> Result<Frame, RouterError> {
1675 self.route_open_admission_refusal_frame(
1676 ctx,
1677 caller,
1678 frame,
1679 target_module_id,
1680 "target_binds_full",
1681 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1682 format!(
1683 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1684 ),
1685 )
1686 }
1687
1688 #[allow(clippy::too_many_arguments)]
1696 fn route_open_admission_refusal_frame(
1697 &self,
1698 ctx: &RouteCtx,
1699 caller: &RouteOpenCaller,
1700 frame: &Frame,
1701 target_module_id: &str,
1702 reason: &'static str,
1703 (in_flight, limit): (usize, usize),
1704 message: impl Into<String>,
1705 ) -> Result<Frame, RouterError> {
1706 let code = error_codes::TARGET_UNAVAILABLE;
1707 self.counters.increment_route_open_refused(code);
1708 info!(
1709 target: "control",
1710 code,
1711 reason,
1712 module_id = ?target_module_id,
1713 connection_id = ctx.connection_id.get(),
1714 principal = %caller.principal,
1715 scope = %caller.scope,
1716 harness = %caller.harness,
1717 in_flight,
1718 limit,
1719 "route.open refused"
1720 );
1721 control_error_frame(frame, code, message.into())
1722 }
1723
1724 #[cfg(test)]
1730 pub fn handle_control(
1731 &self,
1732 connection_id: ConnectionId,
1733 frame: Frame,
1734 ) -> Result<Vec<Frame>, RouterError> {
1735 match frame.header.ty {
1736 FrameType::Ping => Ok(vec![pong(&frame)?]),
1737 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1738 FrameType::Goodbye => self.handle_goodbye(connection_id),
1739 ty => Ok(vec![control_error_frame(
1740 &frame,
1741 "unsupported_control_frame",
1742 format!("unsupported channel-0 frame {ty:?}"),
1743 )?]),
1744 }
1745 }
1746
1747 pub async fn handle_control_frame(
1748 &self,
1749 ctx: &RouteCtx,
1750 frame: Frame,
1751 ) -> Result<Vec<Frame>, RouterError> {
1752 self.handle_control_frame_timed(ctx, frame, None).await
1753 }
1754
1755 pub(crate) fn record_caller_endpoints(
1756 &self,
1757 connection: ConnectionId,
1758 endpoints: Option<(SocketAddr, SocketAddr)>,
1759 ) {
1760 let mut callers = self
1761 .caller_endpoints
1762 .lock()
1763 .unwrap_or_else(|p| p.into_inner());
1764 callers.remove(&connection);
1765 if let Some(endpoints) = endpoints {
1766 callers.insert(connection, endpoints);
1767 }
1768 }
1769
1770 #[cfg(test)]
1771 pub(crate) fn with_caller_lookup_for_test(
1772 mut self,
1773 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1774 ) -> Self {
1775 self.caller_lookup = Some(Arc::new(lookup));
1776 self
1777 }
1778
1779 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1780 let endpoints = self
1781 .caller_endpoints
1782 .lock()
1783 .unwrap_or_else(|p| p.into_inner())
1784 .get(&connection)
1785 .copied();
1786 #[cfg(test)]
1787 let lookup = self.caller_lookup.clone();
1788 tokio::task::spawn_blocking(move || {
1789 #[cfg(test)]
1790 if let Some(lookup) = lookup {
1791 return lookup(endpoints);
1792 }
1793 endpoints
1794 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1795 .unwrap_or_default()
1796 })
1797 .await
1798 .unwrap_or_default()
1799 }
1800
1801 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1802 frame.header.channel == 0
1803 && frame.header.ty == FrameType::Request
1804 && self
1805 .forwarding
1806 .module_endpoint_for_connection(ctx.connection_id)
1807 .is_ok_and(|endpoint| endpoint.is_none())
1808 && parse_client_control_request(&frame.body)
1809 .ok()
1810 .is_some_and(|request| privileged_operation(&request).is_some())
1811 }
1812
1813 pub(crate) async fn handle_control_frame_timed(
1814 &self,
1815 ctx: &RouteCtx,
1816 frame: Frame,
1817 dispatch_started_at: Option<StdInstant>,
1818 ) -> Result<Vec<Frame>, RouterError> {
1819 match frame.header.ty {
1820 FrameType::Ping => Ok(vec![pong(&frame)?]),
1821 FrameType::Hello => {
1822 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1823 }
1824 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1825 FrameType::Cancel => {
1826 if self
1829 .forwarding
1830 .operator_confirms()
1831 .cancel(ctx.connection_id, frame.header.corr)
1832 || self
1833 .supervisor
1834 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1835 || self.cancel_activity_subscription(ctx, &frame)?
1836 {
1837 Ok(Vec::new())
1838 } else {
1839 Ok(vec![control_error_frame(
1840 &frame,
1841 "unknown_subscription",
1842 "no subscription has this correlation id",
1843 )?])
1844 }
1845 }
1846 FrameType::Request => {
1847 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1850 if op
1851 .as_ref()
1852 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1853 {
1854 return self.handle_activity_record(ctx, frame).await;
1855 }
1856 if op.as_ref().is_some_and(|probe| {
1857 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1858 }) {
1859 return self.handle_activity_read(ctx, frame).await;
1860 }
1861 if op
1862 .as_ref()
1863 .is_some_and(|probe| probe.op == "operator.confirm")
1864 {
1865 return self.handle_operator_confirm(ctx, frame);
1866 }
1867 if op
1868 .as_ref()
1869 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1870 {
1871 return self.handle_scope_served_by(ctx.connection_id, frame);
1872 }
1873 if self
1874 .forwarding
1875 .module_endpoint_for_connection(ctx.connection_id)
1876 .map_err(RouterError::Forwarding)?
1877 .is_some()
1878 {
1879 if !is_known_module_request_op(&frame.body) {
1880 return Ok(vec![control_error_frame(
1881 &frame,
1882 "unsupported_control_frame",
1883 "module-originated channel-0 REQUEST is not supported",
1884 )?]);
1885 }
1886 let request = match parse_module_control_request_from_module(&frame.body) {
1887 Ok(request) => request,
1888 Err((err, ControlRequestBodyError::UnknownOp)) => {
1889 return Ok(vec![control_error_frame(
1890 &frame,
1891 "unsupported_control_frame",
1892 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1893 )?])
1894 }
1895 Err((err, ControlRequestBodyError::InvalidBody)) => {
1896 return Ok(vec![control_error_frame(
1897 &frame,
1898 "invalid_control_body",
1899 format!("malformed module control body: {err}"),
1900 )?])
1901 }
1902 };
1903 let op = module_control_request_op(&request);
1904 let corr = frame.header.corr;
1905 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1906 let result =
1907 self.handle_module_control_request(ctx.connection_id, frame, request);
1908 log_slow_control_dispatch(
1909 dispatch_started_at,
1910 op,
1911 ctx.connection_id,
1912 corr,
1913 None,
1914 );
1915 return result;
1916 }
1917
1918 if is_known_module_request_op(&frame.body) {
1919 return Ok(vec![control_error_frame(
1920 &frame,
1921 "not_registered",
1922 "catalog.update requires an active module registration owned by this connection",
1923 )?]);
1924 }
1925
1926 let request = match parse_client_control_request(&frame.body) {
1927 Ok(request) => request,
1928 Err((err, ControlRequestBodyError::UnknownOp)) => {
1929 return Ok(vec![control_error_frame(
1930 &frame,
1931 "unknown_control_op",
1932 format!("unknown client control op: {err}"),
1933 )?])
1934 }
1935 Err((err, ControlRequestBodyError::InvalidBody)) => {
1936 return Ok(vec![control_error_frame(
1937 &frame,
1938 "invalid_control_body",
1939 format!("malformed client control body: {err}"),
1940 )?])
1941 }
1942 };
1943 let op = client_control_request_op(&request);
1944 let corr = frame.header.corr;
1945 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1946 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1947 #[cfg(test)]
1948 if let Some(delay) = self.control_dispatch_delay {
1949 tokio::time::sleep(delay).await;
1950 }
1951 let result = self
1952 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1953 .await;
1954 if let Some(timings) = route_open_timings.as_mut() {
1955 timings.finish();
1956 }
1957 log_slow_control_dispatch(
1958 dispatch_started_at,
1959 op,
1960 ctx.connection_id,
1961 corr,
1962 route_open_timings.as_ref(),
1963 );
1964 result
1965 }
1966 FrameType::Push => {
1967 let Some(endpoint) = self
1968 .forwarding
1969 .module_endpoint_for_connection(ctx.connection_id)
1970 .map_err(RouterError::Forwarding)?
1971 else {
1972 return Ok(vec![control_error_frame(
1973 &frame,
1974 "unsupported_control_frame",
1975 "client-originated channel-0 PUSH is not supported",
1976 )?]);
1977 };
1978 self.handle_status_update(endpoint, frame)
1979 }
1980 FrameType::Response | FrameType::Error
1981 if self
1982 .forwarding
1983 .module_endpoint_for_connection(ctx.connection_id)
1984 .map_err(RouterError::Forwarding)?
1985 .is_some() =>
1986 {
1987 self.handle_module_relay_response(ctx.connection_id, frame)
1988 }
1989 ty => Ok(vec![control_error_frame(
1990 &frame,
1991 "unsupported_control_frame",
1992 format!("unsupported channel-0 frame {ty:?}"),
1993 )?]),
1994 }
1995 }
1996
1997 pub fn cleanup_connection(
1998 &self,
1999 connection_id: ConnectionId,
2000 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2001 self.caller_endpoints
2002 .lock()
2003 .unwrap_or_else(|p| p.into_inner())
2004 .remove(&connection_id);
2005 self.forwarding
2006 .operator_confirms()
2007 .connection_closed(connection_id);
2008 self.cleanup_connection_with_end_reason(
2009 connection_id,
2010 RegistrationEndReason::ConnectionClosed,
2011 )
2012 }
2013
2014 fn cleanup_connection_with_end_reason(
2015 &self,
2016 connection_id: ConnectionId,
2017 requested_reason: RegistrationEndReason,
2018 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2019 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2020 self.registry
2021 .get_module_by_connection(connection_id)?
2022 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2023 .and_then(|module| module.registration_end_reason().ok().flatten())
2024 .unwrap_or(requested_reason)
2025 } else {
2026 requested_reason
2027 };
2028 let crash_closed = self
2029 .registry
2030 .get_module_by_connection(connection_id)?
2031 .and_then(|registration| {
2032 self.forwarding
2033 .module_endpoint_for_connection(connection_id)
2034 .ok()
2035 .flatten()
2036 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2037 .map(|routes| (registration.manifest.module_id, routes))
2038 });
2039 let crash_closed = crash_closed.map(|(module_id, routes)| {
2040 let terminal = match self.supervisor.get(&module_id) {
2041 None => false,
2042 Some(module) => match module.will_recover_after_connection_loss() {
2043 Ok(will_recover) => !will_recover,
2044 Err(err) => {
2045 warn!(
2046 %module_id,
2047 error = %err,
2048 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2049 );
2050 false
2051 }
2052 },
2053 };
2054 let (reason, drained) = match self
2058 .forwarding
2059 .daemon_drain_result_for_connection(connection_id)
2060 {
2061 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2062 Ok(None) => (RouteCloseReason::Crash, false),
2063 Err(err) => {
2064 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2065 (RouteCloseReason::Crash, false)
2066 }
2067 };
2068 (module_id, routes, reason, drained, terminal)
2069 });
2070 let registrations = self.deregister_connection(connection_id, end_reason);
2071 let cleanup = if crash_closed.is_some() {
2072 self.forwarding.cleanup_connection_counted(connection_id)
2073 } else {
2074 self.forwarding
2080 .cleanup_connection(connection_id)
2081 .map(|released| crate::forwarding::ConnectionCleanup {
2082 released,
2083 abandoned_relays: 0,
2084 })
2085 };
2086 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2091 let abandoned = cleanup
2092 .as_ref()
2093 .map_or(0, |cleanup| cleanup.abandoned_relays);
2094 send_route_control_pushes(
2095 &self.forwarding,
2096 routes,
2097 ClientControlPush::RouteClosed {
2098 module_id,
2099 channels: Vec::new(),
2100 reason,
2101 drained,
2102 abandoned,
2103 excluded_subscriptions: 0,
2104 terminal: Some(terminal),
2105 },
2106 );
2107 }
2108 if let Ok(cleanup) = cleanup {
2109 self.emit_route_goodbyes(cleanup.released);
2110 }
2111 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2115 crate::supervise::notify_registration_release();
2116 self.capability_evaluator.wake_deadline_loop();
2117 self.refresh_capability_requirements();
2118 }
2119 self.supervisor.remove_spawn_subscribers(connection_id);
2120 self.activity_streams
2121 .lock()
2122 .unwrap_or_else(|p| p.into_inner())
2123 .retain(|(id, _), task| {
2124 if *id == connection_id {
2125 task.abort();
2126 false
2127 } else {
2128 true
2129 }
2130 });
2131 self.hello_launch_nonces
2134 .lock()
2135 .unwrap_or_else(|poisoned| poisoned.into_inner())
2136 .forget(connection_id);
2137 self.scopes
2138 .write()
2139 .unwrap_or_else(|poisoned| poisoned.into_inner())
2140 .release_connection(connection_id);
2141 registrations
2142 }
2143
2144 pub(crate) fn handle_route_goodbye(
2145 &self,
2146 connection_id: ConnectionId,
2147 route_channel: u16,
2148 route_epoch: u32,
2149 ) -> Result<bool, RouterError> {
2150 debug!(
2151 connection_id = connection_id.get(),
2152 route_channel, route_epoch, "handling route GOODBYE"
2153 );
2154 let RouteRelease::Removed(released_route) = self
2155 .forwarding
2156 .release_client_route(connection_id, route_channel, route_epoch)
2157 .map_err(RouterError::Forwarding)?
2158 else {
2159 return Ok(false);
2160 };
2161 self.emit_route_goodbyes(vec![released_route]);
2162 Ok(true)
2163 }
2164
2165 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2166 for released in released_routes {
2167 let frame = match Frame::build_with_version(
2168 released.negotiated_ver,
2169 FrameType::Goodbye,
2170 control_flags(),
2171 released.channel,
2172 released.epoch,
2173 0,
2174 Vec::new(),
2175 ) {
2176 Ok(frame) => frame,
2177 Err(err) => {
2178 warn!(
2179 route_channel = released.channel,
2180 error = %err,
2181 "failed to build route GOODBYE frame"
2182 );
2183 continue;
2184 }
2185 };
2186 if !released.close_on_delivery_failure() {
2187 crate::forwarding::send_module_route_goodbye(
2188 &self.counters,
2189 &released.sink,
2190 frame,
2191 released.module_id.as_deref(),
2192 "client route released",
2193 );
2194 continue;
2195 }
2196 if let Err(err) = released.sink.try_send(frame) {
2197 warn!(
2198 target_connection_id = released.connection_id.get(),
2199 route_channel = released.channel,
2200 error = %err,
2201 "route GOODBYE was not delivered to client; closing target connection"
2202 );
2203 if self
2204 .forwarding
2205 .escalate_client_delivery_failure(
2206 released.connection_id,
2207 released.channel,
2208 released.epoch,
2209 CloseReason::new(
2210 "route_goodbye_delivery_failed",
2211 format!(
2212 "failed to enqueue route GOODBYE for channel {}: {err}",
2213 released.channel
2214 ),
2215 ),
2216 crate::forwarding::UndeliveredFrame {
2217 module_id: released.module_id.as_deref(),
2218 sink: &released.sink,
2219 },
2220 )
2221 .unwrap_or(false)
2222 {
2223 self.counters.increment_goodbye_relay_client_failed();
2224 }
2225 }
2226 }
2227 }
2228
2229 fn send_abandoned_route_bind_goodbye(
2241 &self,
2242 module_sink: &crate::FrameSink,
2243 negotiated_ver: u8,
2244 module_channel: u16,
2245 module_epoch: u32,
2246 ) {
2247 let frame = match Frame::build_with_version(
2248 negotiated_ver,
2249 FrameType::Goodbye,
2250 control_flags(),
2251 module_channel,
2252 module_epoch,
2253 0,
2254 Vec::new(),
2255 ) {
2256 Ok(frame) => frame,
2257 Err(err) => {
2258 warn!(
2259 route_channel = module_channel,
2260 error = %err,
2261 "failed to build GOODBYE for abandoned route.bind"
2262 );
2263 return;
2264 }
2265 };
2266 crate::forwarding::send_module_route_goodbye(
2267 &self.counters,
2268 module_sink,
2269 frame,
2270 None,
2271 "abandoned route.bind",
2272 );
2273 }
2274
2275 fn handle_hello(
2276 &self,
2277 connection_id: ConnectionId,
2278 sink: Option<crate::FrameSink>,
2279 frame: Frame,
2280 ) -> Result<Vec<Frame>, RouterError> {
2281 debug!(
2282 connection_id = connection_id.get(),
2283 corr = frame.header.corr,
2284 "handling HELLO"
2285 );
2286 if self
2290 .registry
2291 .get_module_by_connection(connection_id)
2292 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2293 .is_some()
2294 {
2295 return Ok(vec![control_error_frame(
2296 &frame,
2297 "invalid_hello",
2298 "connection is already registered as a module",
2299 )?]);
2300 }
2301 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2302 Ok(value) => value,
2303 Err(err) => {
2304 return Ok(vec![control_error_frame(
2305 &frame,
2306 "invalid_hello",
2307 format!("malformed HELLO body: {err}"),
2308 )?])
2309 }
2310 };
2311 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2312 return Ok(vec![control_error_frame(
2313 &frame,
2314 "invalid_capability_grammar",
2315 err.to_string(),
2316 )?]);
2317 }
2318 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2319 return Ok(vec![control_error_frame(
2320 &frame,
2321 "invalid_manifest",
2322 err.to_string(),
2323 )?]);
2324 }
2325 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2326 return Ok(vec![control_error_frame(
2327 &frame,
2328 "invalid_event_declaration",
2329 err.to_string(),
2330 )?]);
2331 }
2332 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2333 return Ok(vec![control_error_frame(
2334 &frame,
2335 "invalid_manifest",
2336 err.to_string(),
2337 )?]);
2338 }
2339 if let Some(provenance) = hello_value
2340 .get("manifest")
2341 .and_then(|manifest| manifest.get("provenance"))
2342 {
2343 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2344 return Ok(vec![control_error_frame(
2345 &frame,
2346 "invalid_manifest",
2347 format!("malformed manifest provenance: {err}"),
2348 )?]);
2349 }
2350 }
2351 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2352 let module_id = hello_value["manifest"]["module_id"]
2353 .as_str()
2354 .unwrap_or("<unknown>");
2355 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2356 }
2357 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2358 Ok(hello) => hello,
2359 Err(err) => {
2360 return Ok(vec![control_error_frame(
2361 &frame,
2362 "invalid_hello",
2363 format!("malformed HELLO body: {err}"),
2364 )?])
2365 }
2366 };
2367
2368 if hello.protocol_ver != hello.manifest.protocol_ver {
2369 return Ok(vec![control_error_frame(
2370 &frame,
2371 "invalid_manifest",
2372 format!(
2373 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2374 hello.protocol_ver, hello.manifest.protocol_ver
2375 ),
2376 )?]);
2377 }
2378
2379 if hello.manifest.module_id.trim().is_empty() {
2380 return Ok(vec![control_error_frame(
2381 &frame,
2382 "invalid_manifest",
2383 "manifest module_id must not be empty",
2384 )?]);
2385 }
2386
2387 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2388 Ok(negotiated_ver) => negotiated_ver,
2389 Err(message) => {
2390 return Ok(vec![control_error_frame(
2391 &frame,
2392 "version_unsupported",
2393 message,
2394 )?])
2395 }
2396 };
2397
2398 let swap_admission = self
2406 .supervisor
2407 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2408 if swap_admission == SwapHelloAdmission::Refused {
2409 warn!(
2410 module_id = %hello.manifest.module_id,
2411 connection_id = connection_id.get(),
2412 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2413 );
2414 return Ok(vec![control_error_frame(
2415 &frame,
2416 "swap_token_invalid",
2417 format!(
2418 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2419 hello.manifest.module_id
2420 ),
2421 )?]);
2422 }
2423 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2424
2425 let nonce_verified = if swap_candidate {
2430 true
2431 } else {
2432 match self.supervisor.reserved_hello_verification(
2433 &hello.manifest.module_id,
2434 hello.launch_nonce.as_deref(),
2435 ) {
2436 Ok(verified) => verified,
2437 Err(rejection) => {
2438 let message = match rejection {
2439 ReservedHelloRejection::Exact { module_id } => format!(
2440 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2441 ),
2442 ReservedHelloRejection::Prefix {
2443 prefix,
2444 owner_module_id,
2445 } => format!(
2446 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2447 hello.manifest.module_id
2448 ),
2449 };
2450 return Ok(vec![control_error_frame(
2451 &frame,
2452 "reserved_module",
2453 message,
2454 )?]);
2455 }
2456 }
2457 };
2458
2459 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2460 &hello.manifest.module_id,
2461 hello.manifest.capabilities.as_ref(),
2462 );
2463 if let Some(refusal) = reserved_capability_refusals.first() {
2464 let capability = refusal.capability.clone();
2465 let bound_module = refusal.claimants[0].clone();
2466 log_duplicate_claim_events(reserved_capability_refusals);
2467 return Ok(vec![control_error_frame(
2468 &frame,
2469 "reserved_capability",
2470 format!(
2471 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2472 capability, bound_module, hello.manifest.module_id
2473 ),
2474 )?]);
2475 }
2476
2477 if self
2480 .forwarding
2481 .connection_has_client_routes(connection_id)
2482 .map_err(RouterError::Forwarding)?
2483 {
2484 return Ok(vec![control_error_frame(
2485 &frame,
2486 "invalid_hello",
2487 "connection has open client routes and cannot also register as a module",
2488 )?]);
2489 }
2490
2491 self.hello_launch_nonces
2496 .lock()
2497 .unwrap_or_else(|poisoned| poisoned.into_inner())
2498 .record(connection_id, hello.launch_nonce.as_deref());
2499 let control_ops = effective_module_control_ops(hello.control_ops);
2500 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2503 if swap_candidate {
2504 return self.register_swap_candidate(
2505 connection_id,
2506 sink,
2507 &frame,
2508 hello.manifest,
2509 negotiated_ver,
2510 control_ops,
2511 hello_ack,
2512 );
2513 }
2514 let registration = match self
2521 .registry
2522 .register_unpublished_verified_with_control_ops(
2523 hello.manifest,
2524 negotiated_ver,
2525 connection_id,
2526 control_ops,
2527 nonce_verified,
2528 ) {
2529 Ok(registration) => registration,
2530 Err(RegistryError::DuplicateModuleId { module_id }) => {
2531 return Ok(vec![control_error_frame(
2532 &frame,
2533 "duplicate_module_id",
2534 format!(
2535 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2536 ),
2537 )?])
2538 }
2539 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2540 return Ok(vec![control_error_frame(
2541 &frame,
2542 "invalid_module_id",
2543 err.to_string(),
2544 )?])
2545 }
2546 Err(err) => {
2547 return Ok(vec![control_error_frame(
2548 &frame,
2549 "registry_error",
2550 err.to_string(),
2551 )?])
2552 }
2553 };
2554 #[cfg(test)]
2555 self.hold_between_registry_and_forwarding();
2556
2557 let reply = if let Some(sink) = sink {
2558 let concurrency = manifest_concurrency(®istration.manifest);
2574 if let Err(err) = self.forwarding.register_module_connection_acked(
2575 connection_id,
2576 registration.manifest.module_id.clone(),
2577 negotiated_ver,
2578 concurrency,
2579 sink,
2580 hello_ack,
2581 ) {
2582 if matches!(
2587 self.deregister_connection(
2588 connection_id,
2589 RegistrationEndReason::RegistrationFailed,
2590 ),
2591 Ok(r) if !r.is_empty()
2592 ) {
2593 crate::supervise::notify_registration_release();
2594 }
2595 return Ok(vec![control_error_frame(
2596 &frame,
2597 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2598 "invalid_hello"
2599 } else {
2600 forwarding_error_code(&err)
2601 },
2602 err.to_string(),
2603 )?]);
2604 }
2605 Vec::new()
2606 } else {
2607 vec![hello_ack]
2610 };
2611
2612 if !self
2617 .registry
2618 .publish_registration(connection_id)
2619 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2620 {
2621 debug!(
2622 module_id = %registration.manifest.module_id,
2623 connection_id = connection_id.get(),
2624 "registration was removed before it could be published"
2625 );
2626 }
2627
2628 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2637 info!(
2638 module_id = %registration.manifest.module_id,
2639 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2640 );
2641 }
2642
2643 self.apply_registration_capabilities(®istration);
2644
2645 self.supervisor
2646 .record_activity_registration(®istration.manifest.module_id);
2647
2648 info!(
2649 module_id = %registration.manifest.module_id,
2650 module_version = %registration.manifest.module_version,
2651 negotiated_ver,
2652 routable_provider = manifest_provides_routable_role(®istration.manifest),
2653 connection_id = connection_id.get(),
2654 "module registered"
2655 );
2656
2657 Ok(reply)
2658 }
2659
2660 #[allow(clippy::too_many_arguments)]
2677 fn register_swap_candidate(
2678 &self,
2679 connection_id: ConnectionId,
2680 sink: Option<crate::FrameSink>,
2681 frame: &Frame,
2682 manifest: ModuleManifest,
2683 negotiated_ver: u8,
2684 control_ops: Vec<String>,
2685 hello_ack: Frame,
2686 ) -> Result<Vec<Frame>, RouterError> {
2687 let module_id = manifest.module_id.clone();
2688 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2689 manifest,
2690 negotiated_ver,
2691 connection_id,
2692 control_ops,
2693 ) {
2694 Ok(registration) => registration,
2695 Err(RegistryError::DuplicateModuleId { module_id }) => {
2696 return Ok(vec![control_error_frame(
2697 frame,
2698 "duplicate_module_id",
2699 format!(
2700 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2701 ),
2702 )?])
2703 }
2704 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2705 return Ok(vec![control_error_frame(
2706 frame,
2707 "invalid_module_id",
2708 err.to_string(),
2709 )?])
2710 }
2711 Err(err) => {
2712 return Ok(vec![control_error_frame(
2713 frame,
2714 "registry_error",
2715 err.to_string(),
2716 )?])
2717 }
2718 };
2719 #[cfg(test)]
2720 self.hold_between_registry_and_forwarding();
2721 let reply = if let Some(sink) = sink {
2722 let concurrency = manifest_concurrency(®istration.manifest);
2726 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2727 connection_id,
2728 module_id.clone(),
2729 negotiated_ver,
2730 concurrency,
2731 sink,
2732 hello_ack,
2733 ) {
2734 if matches!(
2735 self.deregister_connection(
2736 connection_id,
2737 RegistrationEndReason::RegistrationFailed,
2738 ),
2739 Ok(r) if !r.is_empty()
2740 ) {
2741 crate::supervise::notify_registration_release();
2742 }
2743 return Ok(vec![control_error_frame(
2744 frame,
2745 forwarding_error_code(&err),
2746 err.to_string(),
2747 )?]);
2748 }
2749 Vec::new()
2750 } else {
2751 vec![hello_ack]
2752 };
2753 if !self
2754 .registry
2755 .publish_registration(connection_id)
2756 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2757 {
2758 debug!(
2759 module_id = %module_id,
2760 connection_id = connection_id.get(),
2761 "swap candidate registration was removed before it could be published"
2762 );
2763 }
2764 self.supervisor.mark_swap_candidate_admitted(&module_id);
2765 info!(
2766 module_id = %module_id,
2767 module_version = %registration.manifest.module_version,
2768 negotiated_ver,
2769 ready = registration.ready,
2770 connection_id = connection_id.get(),
2771 "swap candidate registered; not routable until cutover"
2772 );
2773 Ok(reply)
2774 }
2775
2776 fn build_hello_ack(
2777 &self,
2778 frame: &Frame,
2779 negotiated_ver: u8,
2780 module_id: &str,
2781 ) -> Result<Frame, RouterError> {
2782 let ack = ModuleHelloAckBody {
2783 negotiated_ver,
2784 subc_ops: module_subc_ops(),
2785 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2786 storage: self
2787 .storage_config
2788 .as_ref()
2789 .map(|cfg| cfg.descriptor_for(module_id)),
2790 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2791 };
2792 let body = serde_json::to_vec(&ack).map_err(|err| {
2793 RouterError::backend(
2794 0,
2795 frame.header.corr,
2796 format!("failed to encode HELLO_ACK: {err}"),
2797 )
2798 })?;
2799
2800 Frame::build_with_version(
2801 negotiated_ver,
2802 FrameType::HelloAck,
2803 control_flags(),
2804 0,
2805 0,
2806 frame.header.corr,
2807 body,
2808 )
2809 .map_err(RouterError::FrameBuild)
2810 }
2811
2812 async fn handle_client_control_request(
2813 &self,
2814 ctx: &RouteCtx,
2815 frame: Frame,
2816 request: ClientControlRequest,
2817 route_open_timings: &mut Option<RouteOpenTimings>,
2818 ) -> Result<Vec<Frame>, RouterError> {
2819 if let Some(operation) = privileged_operation(&request) {
2820 let caller = self.resolve_caller(ctx.connection_id).await;
2821 let op = client_control_request_op(&request);
2822 let text = prompt_text(operation, &caller);
2823 match self.privileged_operator_prompt {
2824 PrivilegedOperatorPrompt::Log => {
2825 info!(target: "subc_daemon::privileged_prompt", operation = op,
2826 target = ?operation, caller = %caller.description(), summary = %text,
2827 "privileged request would prompt the operator");
2828 }
2829 PrivilegedOperatorPrompt::Enforce => {
2830 let key =
2831 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2832 let outcome = self
2833 .forwarding
2834 .operator_confirms()
2835 .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2836 .await;
2837 if outcome.code != "confirmed" {
2838 return Ok(vec![outcome.frame(&frame)]);
2839 }
2840 }
2841 }
2842 }
2843 match request {
2844 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2845 ClientControlRequest::CatalogList { module_id } => {
2846 self.handle_catalog_list(frame, module_id)
2847 }
2848 ClientControlRequest::RouteOpen {
2849 target,
2850 identity,
2851 consumer_identity,
2852 consumer_capabilities,
2853 role_versions,
2854 admission_facts,
2855 scope,
2856 } => {
2857 self.handle_route_open(
2858 ctx,
2859 frame,
2860 RouteOpenRequest {
2861 target,
2862 identity,
2863 consumer_identity,
2864 consumer_capabilities,
2865 role_versions,
2866 admission_facts,
2867 scope,
2868 },
2869 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2870 )
2871 .await
2872 }
2873 ClientControlRequest::RoutePoll {
2874 route_channel,
2875 route_epoch,
2876 kind,
2877 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2878 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2879 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2880 self.handle_supervisor_spawn_snapshot(frame)
2881 }
2882 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2883 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2884 }
2885 ClientControlRequest::SupervisorRestart {
2886 module_id,
2887 drain_timeout_ms,
2888 } => {
2889 self.log_supervisor_request_received(
2890 ctx,
2891 frame.header.corr,
2892 ops::SUPERVISOR_RESTART,
2893 Some(&module_id),
2894 None,
2895 )?;
2896 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2897 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2898 .await
2899 }
2900 ClientControlRequest::SupervisorSwap {
2901 module_id,
2902 ready_timeout_ms,
2903 } => {
2904 self.log_supervisor_request_received(
2905 ctx,
2906 frame.header.corr,
2907 ops::SUPERVISOR_SWAP,
2908 Some(&module_id),
2909 None,
2910 )?;
2911 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2912 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2913 .await
2914 }
2915 ClientControlRequest::SupervisorReload { module_id } => {
2916 self.log_supervisor_request_received(
2917 ctx,
2918 frame.header.corr,
2919 ops::SUPERVISOR_RELOAD,
2920 Some(&module_id),
2921 None,
2922 )?;
2923 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2924 self.handle_supervisor_reload(frame, module_id, requester)
2925 .await
2926 }
2927 ClientControlRequest::SupervisorRescan { preview } => {
2928 if !preview {
2929 self.log_supervisor_request_received(
2930 ctx,
2931 frame.header.corr,
2932 ops::SUPERVISOR_RESCAN,
2933 None,
2934 None,
2935 )?;
2936 }
2937 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2938 self.handle_supervisor_rescan(
2939 frame,
2940 if preview {
2941 RescanMode::Preview
2942 } else {
2943 RescanMode::Execute
2944 },
2945 requester,
2946 )
2947 .await
2948 }
2949 ClientControlRequest::SupervisorRescanChecked {
2950 preview,
2951 candidate,
2952 expect,
2953 } => {
2954 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
2955 .await
2956 }
2957 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2958 self.log_supervisor_request_received(
2959 ctx,
2960 frame.header.corr,
2961 ops::SUPERVISOR_RELEASE_RESERVED,
2962 Some(&module_id),
2963 None,
2964 )?;
2965 self.handle_supervisor_release_reserved(frame, module_id)
2966 .await
2967 }
2968 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2969 self.log_supervisor_request_received(
2970 ctx,
2971 frame.header.corr,
2972 ops::SUPERVISOR_SET_ENABLED,
2973 Some(&module_id),
2974 Some(enabled),
2975 )?;
2976 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2977 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2978 .await
2979 }
2980 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2981 self.handle_supervisor_health_probe(frame, module_id).await
2982 }
2983 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2984 ClientControlRequest::SupervisorRoutes { module_id } => {
2985 self.handle_supervisor_routes(frame, module_id)
2986 }
2987 ClientControlRequest::SupervisorProvenance { module_id } => {
2988 self.handle_supervisor_provenance(frame, module_id).await
2989 }
2990 ClientControlRequest::SupervisorStderrTail {
2991 module_id,
2992 max_lines,
2993 max_bytes,
2994 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2995 ClientControlRequest::SupervisorTerminals { module_id } => {
2996 self.handle_supervisor_terminals(frame, module_id).await
2997 }
2998 }
2999 }
3000
3001 fn log_supervisor_request_received(
3002 &self,
3003 ctx: &RouteCtx,
3004 corr: u64,
3005 op: &'static str,
3006 module_id: Option<&str>,
3007 enabled: Option<bool>,
3008 ) -> Result<(), RouterError> {
3009 let caller = self.supervisor_requester(ctx, corr)?;
3010 let caller = principal_label(&caller);
3011 info!(
3014 target: "control",
3015 op,
3016 module_id,
3017 enabled,
3018 connection_id = ctx.connection_id.get(),
3019 caller = %caller,
3020 "supervisor request received"
3021 );
3022 Ok(())
3023 }
3024
3025 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3026 Ok(self
3027 .registry
3028 .get_module_by_connection(ctx.connection_id)
3029 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3030 .map(|registration| Principal::Reserved {
3031 module_id: registration.manifest.module_id,
3032 })
3033 .unwrap_or(Principal::Direct))
3034 }
3035
3036 fn handle_module_control_request(
3037 &self,
3038 connection_id: ConnectionId,
3039 frame: Frame,
3040 request: ModuleControlRequestFromModule,
3041 ) -> Result<Vec<Frame>, RouterError> {
3042 match request {
3043 ModuleControlRequestFromModule::CatalogUpdate {
3044 provides,
3045 capabilities,
3046 ready,
3047 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3048 ModuleControlRequestFromModule::LiveRoots {} => {
3049 let registered = self
3050 .registry
3051 .get_module_by_connection(connection_id)
3052 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3053 let Some(registration) = registered else {
3054 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3055 };
3056 let response = self
3057 .forwarding
3058 .live_roots(®istration.manifest.module_id)
3059 .map_err(RouterError::Forwarding)?;
3060 Ok(vec![control_response_body_frame(
3061 &frame,
3062 &response,
3063 "ModuleControlResponseToModule::LiveRoots",
3064 )?])
3065 }
3066 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3067 self.handle_scope_sync(connection_id, frame, generation, scopes)
3068 }
3069 ModuleControlRequestFromModule::ScopeApply {
3070 generation,
3071 upsert,
3072 end,
3073 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3074 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3075 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3076 }
3077 }
3078 }
3079
3080 fn handle_scope_served_by(
3081 &self,
3082 connection_id: ConnectionId,
3083 frame: Frame,
3084 ) -> Result<Vec<Frame>, RouterError> {
3085 let registered = self
3086 .registry
3087 .get_module_by_connection(connection_id)
3088 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3089 let Some(registered) = registered else {
3090 return Ok(vec![control_error_frame(
3091 &frame,
3092 "not_registered",
3093 "scope.served_by requires an active module registration owned by this connection",
3094 )?]);
3095 };
3096 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3097 {
3098 Ok(request) => request,
3099 Err(err) => {
3100 return Ok(vec![control_error_frame(
3101 &frame,
3102 "invalid_control_body",
3103 format!("malformed scope.served_by body: {err}"),
3104 )?])
3105 }
3106 };
3107 let answer = self
3108 .scopes
3109 .read()
3110 .unwrap_or_else(|poisoned| poisoned.into_inner())
3111 .served_by(
3112 ®istered.manifest.module_id,
3113 &request.scope_ref,
3114 request.scope_epoch,
3115 &request.module_id,
3116 );
3117 let reply = subc_control::ScopeServedByReply::new(
3118 answer,
3119 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3120 );
3121 Ok(vec![control_response_body_frame(
3122 &frame,
3123 &reply,
3124 "ScopeServedByReply",
3125 )?])
3126 }
3127
3128 fn handle_operator_confirm(
3129 &self,
3130 ctx: &RouteCtx,
3131 frame: Frame,
3132 ) -> Result<Vec<Frame>, RouterError> {
3133 use crate::operator_confirm::{audit, Outcome};
3134 let registration = self
3135 .registry
3136 .get_module_by_connection(ctx.connection_id)
3137 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3138 let Some(registration) = registration else {
3139 let outcome = Outcome::refusal("not_registered");
3140 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3141 return Ok(vec![outcome.frame(&frame)]);
3142 };
3143 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3144 Ok(request) => request,
3145 Err(_) => {
3146 let outcome = Outcome::refusal("invalid_control_body");
3147 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3148 return Ok(vec![outcome.frame(&frame)]);
3149 }
3150 };
3151 let module_id = registration.manifest.module_id;
3152 let nonce = self
3156 .hello_launch_nonces
3157 .lock()
3158 .unwrap_or_else(|p| p.into_inner())
3159 .nonce(ctx.connection_id)
3160 .map(str::to_owned);
3161 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3162 self.supervisor
3163 .spawned_consumer_authorized(&module_id, nonce)
3164 });
3165 let confirms = self.forwarding.operator_confirms();
3166 self.forwarding
3167 .with_operator_route(
3168 ctx.connection_id,
3169 request.route_channel,
3170 request.route_epoch,
3171 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3172 )
3173 .map_err(RouterError::Forwarding)
3174 }
3175
3176 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3179 let mut scopes = ScopeRecords::from(scopes);
3180 for (index, record) in scopes.records.iter_mut().enumerate() {
3181 if let Some(refusal) = Self::canonicalize_workdir(record) {
3182 scopes.refusals.insert(index, refusal);
3183 }
3184 }
3185 scopes
3186 }
3187
3188 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3189 let workdir = record.workdir.as_mut()?;
3190 let path = std::path::Path::new(workdir);
3191 if !path.is_absolute() {
3192 return Some((
3193 error_codes::INVALID_WORKDIR,
3194 "workdir must be absolute".to_string(),
3195 ));
3196 }
3197 match ProjectRootId::from_path_allowing_missing(path) {
3198 Ok(root) => match root.as_path().to_str() {
3199 Some(canonical) => {
3200 *workdir = canonical.to_string();
3201 None
3202 }
3203 None => Some((
3204 error_codes::INVALID_WORKDIR,
3205 "canonical workdir is not UTF-8".to_string(),
3206 )),
3207 },
3208 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3209 }
3210 }
3211
3212 fn handle_scope_sync(
3216 &self,
3217 connection_id: ConnectionId,
3218 frame: Frame,
3219 generation: u64,
3220 scopes: Vec<ScopeRecord>,
3221 ) -> Result<Vec<Frame>, RouterError> {
3222 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3223 }
3224
3225 fn handle_scope_change(
3226 &self,
3227 connection_id: ConnectionId,
3228 frame: Frame,
3229 generation: u64,
3230 scopes: Vec<ScopeRecord>,
3231 end: Option<Vec<ScopeEnd>>,
3232 ) -> Result<Vec<Frame>, RouterError> {
3233 let is_apply = end.is_some();
3234 let Some(registration) = self
3235 .registry
3236 .get_module_by_connection(connection_id)
3237 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3238 else {
3239 return Ok(vec![control_error_frame(
3240 &frame,
3241 "not_registered",
3242 "scope.sync and scope.apply require an active module registration owned by this connection",
3243 )?]);
3244 };
3245 let owner = registration.manifest.module_id;
3246 let scopes = Self::prepare_scope_records(scopes);
3247 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3248 let is_current_launch = |connection: ConnectionId| {
3249 self.hello_launch_nonces
3250 .lock()
3251 .unwrap_or_else(|poisoned| poisoned.into_inner())
3252 .presented(connection, current_nonce.as_deref())
3253 };
3254 let mut table = self
3259 .scopes
3260 .write()
3261 .unwrap_or_else(|poisoned| poisoned.into_inner());
3262 let outcome = match end {
3263 Some(end) => table.apply(
3264 &owner,
3265 connection_id,
3266 is_current_launch,
3267 generation,
3268 scopes,
3269 end,
3270 ),
3271 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3272 };
3273 let drained = match &outcome {
3274 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3275 Err(_) => Vec::new(),
3276 };
3277 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3278 drop(table);
3279 if has_deadlines {
3280 self.scope_deadline_added.notify_one();
3281 }
3282 match outcome {
3283 Ok(applied) => {
3284 let counts = ScopeOutcomeCounts::of(&applied.results);
3285 info!(
3286 owner = %owner,
3287 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3288 generation,
3289 records = applied.results.len(),
3290 created = counts.created,
3291 replaced = counts.replaced,
3292 updated = counts.updated,
3293 unchanged = counts.unchanged,
3294 refused = counts.refused,
3295 ended = applied.ended.len(),
3296 tag_changes = applied.tag_changes.len(),
3297 routes_closed = drained.len(),
3298 "scope change accepted"
3299 );
3300 for refused in applied
3306 .results
3307 .iter()
3308 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3309 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3310 {
3311 warn!(
3312 owner = %owner,
3313 generation,
3314 scope_ref = %refused.scope_ref,
3315 scope_epoch = refused.scope_epoch,
3316 code = refused.code.as_deref().unwrap_or(""),
3317 "scope record refused"
3318 );
3319 }
3320 self.close_scope_drained_routes(drained);
3321 let response = if is_apply {
3322 ModuleControlResponseToModule::ScopeApply {
3323 generation,
3324 results: applied.results,
3325 end_results: applied.end_results,
3326 ended: applied.ended,
3327 }
3328 } else {
3329 ModuleControlResponseToModule::ScopeSync {
3330 generation,
3331 results: applied.results,
3332 ended: applied.ended,
3333 }
3334 };
3335 Ok(vec![control_response_body_frame(
3336 &frame,
3337 &response,
3338 "scope change response",
3339 )?])
3340 }
3341 Err(refusal) => {
3342 info!(
3343 owner = %owner,
3344 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3345 generation,
3346 code = refusal.code,
3347 "scope change refused"
3348 );
3349 Ok(vec![control_error_frame(
3350 &frame,
3351 refusal.code,
3352 refusal.message,
3353 )?])
3354 }
3355 }
3356 }
3357
3358 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3364 if drained.is_empty() {
3365 return;
3366 }
3367 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3368 BTreeMap::new();
3369 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3370 for route in drained {
3371 warn!(
3372 module_id = %route.module_id,
3373 reason = ?route.reason,
3374 client_connection_id = route.client.connection_id.get(),
3375 route_channel = route.client.channel,
3376 "closing route because its scope changed"
3377 );
3378 pushes
3379 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3380 .or_insert_with(|| (route.reason, Vec::new()))
3381 .1
3382 .push(EndpointRoute {
3383 goodbye_target: route.client.clone(),
3384 principal: Principal::Unverified,
3385 bound_at: Instant::now(),
3386 draining: false,
3387 drain_reason: None,
3388 });
3389 goodbyes.push(route.module);
3390 goodbyes.push(route.client);
3391 }
3392 for ((module_id, _), (reason, routes)) in pushes {
3393 send_route_control_pushes(
3394 &self.forwarding,
3395 routes,
3396 ClientControlPush::RouteClosed {
3397 module_id,
3398 channels: Vec::new(),
3399 reason,
3400 drained: false,
3401 abandoned: 0,
3402 excluded_subscriptions: 0,
3403 terminal: Some(false),
3404 },
3405 );
3406 }
3407 self.emit_route_goodbyes(goodbyes);
3408 }
3409
3410 fn handle_scope_describe(
3413 &self,
3414 connection_id: ConnectionId,
3415 frame: Frame,
3416 owner: Principal,
3417 scope_ref: String,
3418 ) -> Result<Vec<Frame>, RouterError> {
3419 let registered = self
3420 .registry
3421 .get_module_by_connection(connection_id)
3422 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3423 let Some(registered) = registered else {
3424 return Ok(vec![control_error_frame(
3425 &frame,
3426 "not_registered",
3427 "scope.describe requires an active module registration owned by this connection",
3428 )?]);
3429 };
3430 let mut description = self
3431 .scopes
3432 .read()
3433 .unwrap_or_else(|poisoned| poisoned.into_inner())
3434 .describe(&owner, &scope_ref);
3435 if !registered
3438 .manifest
3439 .capabilities
3440 .as_ref()
3441 .is_some_and(|caps| {
3442 caps.provides
3443 .iter()
3444 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3445 })
3446 {
3447 if let Some(stamp) = description.stamp.as_mut() {
3448 stamp.attributes.on_behalf_of_agent = None;
3449 }
3450 }
3451 if !registered
3452 .manifest
3453 .capabilities
3454 .as_ref()
3455 .is_some_and(|caps| {
3456 caps.provides
3457 .iter()
3458 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
3459 })
3460 {
3461 if let Some(stamp) = description.stamp.as_mut() {
3462 stamp.attributes.flow_version = None;
3463 stamp.attributes.flow_code_hash = None;
3464 stamp.attributes.flow_install_id = None;
3465 }
3466 }
3467 let owner_configured = match &owner {
3468 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3473 _ => false,
3474 };
3475 let response = ModuleControlResponseToModule::ScopeDescribe {
3476 status: description.status,
3477 scope_epoch: description.scope_epoch,
3478 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3479 owner_synced: description.owner_synced,
3480 owner_configured,
3481 scope: description.stamp,
3482 };
3483 Ok(vec![control_response_body_frame(
3484 &frame,
3485 &response,
3486 "ModuleControlResponseToModule::ScopeDescribe",
3487 )?])
3488 }
3489
3490 fn handle_catalog_update(
3491 &self,
3492 connection_id: ConnectionId,
3493 frame: Frame,
3494 provides: Vec<ProviderRole>,
3495 capabilities: Option<CapabilityDeclarations>,
3496 ready: Option<bool>,
3497 ) -> Result<Vec<Frame>, RouterError> {
3498 self.refresh_capability_requirements();
3499 let Some(registration) = self
3500 .registry
3501 .get_module_by_connection(connection_id)
3502 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3503 else {
3504 return Ok(vec![control_error_frame(
3505 &frame,
3506 "not_registered",
3507 "catalog.update requires an active module registration owned by this connection",
3508 )?]);
3509 };
3510
3511 if let Some(message) =
3512 catalog_update_frozen_field_message(®istration.manifest, &provides)
3513 {
3514 return Ok(vec![control_error_frame(
3515 &frame,
3516 "catalog_update_frozen_field",
3517 message,
3518 )?]);
3519 }
3520
3521 let mut candidate = registration.manifest.clone();
3522 candidate.provides = provides.clone();
3523 candidate.capabilities = capabilities
3524 .clone()
3525 .or_else(|| registration.manifest.capabilities.clone());
3526 if let Some(rich_ui) = &candidate.rich_ui {
3527 if let Err(err) = rich_ui.validate() {
3528 return Ok(vec![control_error_frame(
3529 &frame,
3530 "invalid_manifest",
3531 err.to_string(),
3532 )?]);
3533 }
3534 }
3535 if let Err(err) = candidate.validate_capability_grammar() {
3536 return Ok(vec![control_error_frame(
3537 &frame,
3538 "invalid_capability_grammar",
3539 err.to_string(),
3540 )?]);
3541 }
3542
3543 let mut conflicts = self
3546 .capability_evaluator
3547 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3548 if let Some(conflict) = conflicts.first() {
3549 let message = format!(
3550 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3551 conflict.capability, conflict.claimants[0], candidate.module_id
3552 );
3553 for conflict in &mut conflicts {
3554 conflict.source = DuplicateClaimSource::CatalogUpdate;
3555 }
3556 log_duplicate_claim_events(conflicts);
3557 return Ok(vec![control_error_frame(
3558 &frame,
3559 "reserved_capability",
3560 message,
3561 )?]);
3562 }
3563
3564 let updated = self
3565 .registry
3566 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3567 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3568 if updated.is_none() {
3569 return Ok(vec![control_error_frame(
3570 &frame,
3571 "not_registered",
3572 "catalog.update requires an active module registration owned by this connection",
3573 )?]);
3574 }
3575 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3576 log_duplicate_claim_events(
3577 self.capability_evaluator
3578 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3579 );
3580 }
3581 if capability_census_trigger(
3582 registration.manifest.capabilities.as_ref(),
3583 updated
3584 .as_ref()
3585 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3586 ) {
3587 self.enforce_capability_denies();
3588 }
3589 self.refresh_capability_requirements();
3590
3591 let response = ModuleControlResponseToModule::CatalogUpdate {};
3592 control_response_body_frame(
3593 &frame,
3594 &response,
3595 "ModuleControlResponseToModule::CatalogUpdate",
3596 )
3597 .map(|frame| vec![frame])
3598 }
3599
3600 pub(crate) fn with_activity_ledger_config(
3601 mut self,
3602 run_dir: &Path,
3603 incarnation: String,
3604 config: crate::activity::ActivityConfig,
3605 ) -> Self {
3606 self.activity = Some(crate::activity::ActivityLedger::open_config(
3607 run_dir.join("activity.jsonl"),
3608 incarnation,
3609 config,
3610 ));
3611 self.supervisor.set_activity(self.activity.clone());
3612 self
3613 }
3614
3615 pub(crate) async fn configure_activity(
3616 &self,
3617 config: crate::activity::ActivityConfig,
3618 ) -> Result<(), &'static str> {
3619 match &self.activity {
3620 Some(ledger) => ledger.configure(config).await,
3621 None => Ok(()),
3622 }
3623 }
3624
3625 fn cancel_activity_subscription(
3626 &self,
3627 ctx: &RouteCtx,
3628 frame: &Frame,
3629 ) -> Result<bool, RouterError> {
3630 let task = self
3631 .activity_streams
3632 .lock()
3633 .unwrap_or_else(|p| p.into_inner())
3634 .remove(&(ctx.connection_id, frame.header.corr));
3635 let Some(task) = task else {
3636 return Ok(false);
3637 };
3638 task.abort();
3639 let end = Frame::build_with_version(
3640 response_version(frame),
3641 FrameType::StreamEnd,
3642 control_flags(),
3643 0,
3644 0,
3645 frame.header.corr,
3646 Vec::new(),
3647 )
3648 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3649 let sink = ctx.egress.clone();
3650 tokio::spawn(async move {
3651 let _ = sink.send(end).await;
3652 });
3653 Ok(true)
3654 }
3655
3656 async fn handle_activity_read(
3657 &self,
3658 ctx: &RouteCtx,
3659 frame: Frame,
3660 ) -> Result<Vec<Frame>, RouterError> {
3661 use subc_control::{
3662 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3663 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3664 };
3665 #[derive(Deserialize)]
3666 struct Identity {
3667 #[serde(default)]
3668 consumer_identity: Option<ConsumerIdentity>,
3669 }
3670 let principal = match self
3674 .forwarding
3675 .module_id_for_connection(ctx.connection_id)
3676 .map_err(RouterError::Forwarding)?
3677 {
3678 Some(module_id) => Ok(Principal::Reserved { module_id }),
3679 None => {
3680 let identity: Identity = match serde_json::from_slice(&frame.body) {
3681 Ok(identity) => identity,
3682 Err(_) => {
3683 return Ok(vec![control_error_frame(
3684 &frame,
3685 ACTIVITY_MALFORMED,
3686 "malformed activity read",
3687 )?])
3688 }
3689 };
3690 self.route_open_principal(&frame, identity.consumer_identity)?
3691 }
3692 };
3693 let admitted = matches!(principal, Ok(Principal::Direct))
3694 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3695 if !admitted {
3696 return Ok(vec![control_error_frame(
3697 &frame,
3698 ACTIVITY_READ_FORBIDDEN,
3699 "activity reads require an operator or trusted mirror principal",
3700 )?]);
3701 }
3702 let Some(ledger) = &self.activity else {
3703 return Ok(vec![control_error_frame(
3704 &frame,
3705 ACTIVITY_UNAVAILABLE,
3706 "activity ledger unavailable",
3707 )?]);
3708 };
3709 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3710 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3711 {
3712 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3713 Ok(request) => ledger.query(request).await,
3714 Err(_) => Err(ACTIVITY_MALFORMED),
3715 };
3716 return match result {
3717 Ok(reply) => Ok(vec![control_response_body_frame(
3718 &frame,
3719 &reply,
3720 ops::ACTIVITY_QUERY,
3721 )?]),
3722 Err(code) => Ok(vec![control_error_frame(
3723 &frame,
3724 code,
3725 "activity query refused",
3726 )?]),
3727 };
3728 }
3729 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3730 Ok(request) => request,
3731 Err(_) => {
3732 return Ok(vec![control_error_frame(
3733 &frame,
3734 ACTIVITY_MALFORMED,
3735 "malformed activity subscription",
3736 )?])
3737 }
3738 };
3739 let mut subscription = match ledger.subscribe(request.since).await {
3740 Ok(subscription) => subscription,
3741 Err(code) => {
3742 return Ok(vec![control_error_frame(
3743 &frame,
3744 code,
3745 "activity subscription refused",
3746 )?])
3747 }
3748 };
3749 let key = (ctx.connection_id, frame.header.corr);
3750 let sink = ctx.egress.clone();
3751 let mut streams = self
3752 .activity_streams
3753 .lock()
3754 .unwrap_or_else(|p| p.into_inner());
3755 streams.retain(|_, task| !task.is_finished());
3756 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3757 return Ok(vec![control_error_frame(
3758 &frame,
3759 "duplicate_subscription",
3760 "activity subscription correlation id already in use",
3761 )?]);
3762 }
3763 let task = tokio::spawn(async move {
3764 loop {
3765 let entry = if let Some(entry) = subscription.replay.pop_front() {
3766 Some(entry)
3767 } else {
3768 subscription.receiver.recv().await
3769 };
3770 let Some(entry) = entry else {
3771 break;
3772 };
3773 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3774 return;
3775 };
3776 let Ok(next) = Frame::build_with_version(
3777 response_version(&frame),
3778 FrameType::StreamData,
3779 control_flags(),
3780 0,
3781 0,
3782 frame.header.corr,
3783 body,
3784 ) else {
3785 return;
3786 };
3787 if sink.send(next).await.is_err() {
3788 return;
3789 }
3790 }
3791 if let Some(code) = subscription.terminal_code() {
3794 if let Ok(end) =
3795 control_error_frame(&frame, code, "activity subscriber fell behind")
3796 {
3797 let _ = sink.send(end).await;
3798 }
3799 }
3800 });
3801 streams.insert(key, task.abort_handle());
3802 Ok(Vec::new())
3803 }
3804
3805 async fn handle_activity_record(
3806 &self,
3807 ctx: &RouteCtx,
3808 frame: Frame,
3809 ) -> Result<Vec<Frame>, RouterError> {
3810 use subc_control::{
3811 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3812 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3813 };
3814 let result = async {
3815 let module = self
3816 .forwarding
3817 .module_id_for_connection(ctx.connection_id)
3818 .ok()
3819 .flatten()
3820 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3821 let request: ActivityRecordRequest =
3822 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3823 crate::activity::validate(&module, &request)?;
3824 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3825 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3826 let at_ms = std::time::SystemTime::now()
3827 .duration_since(std::time::UNIX_EPOCH)
3828 .unwrap_or_default()
3829 .as_millis() as u64;
3830 ledger.record(module, request, actor, at_ms).await
3831 }
3832 .await;
3833 match result {
3834 Ok(entry) => Ok(vec![control_response_body_frame(
3835 &frame,
3836 &ActivityRecordResponse { seq: entry.seq },
3837 "activity.record",
3838 )?]),
3839 Err(code) => Ok(vec![control_error_frame(
3840 &frame,
3841 code,
3842 "activity recording refused",
3843 )?]),
3844 }
3845 }
3846
3847 fn activity_actor(
3848 &self,
3849 connection: ConnectionId,
3850 module: &str,
3851 on: Option<&subc_control::ActivityOn>,
3852 ) -> Result<subc_control::ActivityActor, &'static str> {
3853 use crate::forwarding::{DataRoute, DataRouteState};
3854 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3855 let principal = |id: &str| Principal::Reserved {
3856 module_id: id.to_owned(),
3857 };
3858 let mut actor = ActivityActor {
3859 principal: principal(module),
3860 scope_owner: None,
3861 agent_id: None,
3862 run_id: None,
3863 flow_id: None,
3864 project_root: None,
3865 workdir: None,
3866 project_id: None,
3867 };
3868 let scope = match on {
3869 None => return Ok(actor),
3870 Some(ActivityOn::Scope(scope)) => {
3871 let selector = subc_protocol::scope::ScopeSelector::new(
3872 principal(module),
3873 scope.scope_ref.clone(),
3874 Some(scope.epoch),
3875 );
3876 self.scopes
3877 .read()
3878 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3879 .attribute(&principal(module), module, &selector, true)
3880 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3881 }
3882 Some(ActivityOn::Route(route)) => {
3883 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3884 .forwarding
3885 .lookup_data_route(connection, route.channel, route.epoch)
3886 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3887 else {
3888 return Err(ACTIVITY_ACTOR_UNKNOWN);
3889 };
3890 if binding.module_id != module {
3891 return Err(ACTIVITY_ACTOR_UNKNOWN);
3892 }
3893 actor.principal = binding.principal.clone();
3894 actor.project_root = binding
3895 .project_root
3896 .as_ref()
3897 .map(|root| root.as_path().to_path_buf());
3898 let Some(bound) = &binding.scope else {
3899 return Ok(actor);
3900 };
3901 let owner = principal(&bound.owner);
3902 let selector = subc_protocol::scope::ScopeSelector::new(
3903 owner.clone(),
3904 bound.scope_ref.clone(),
3905 Some(bound.tag.scope_epoch),
3906 );
3907 let admitted = self
3908 .scopes
3909 .read()
3910 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3911 .attribute(&owner, module, &selector, true)
3912 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3913 if admitted.tag != bound.tag {
3914 return Err(ACTIVITY_ACTOR_UNKNOWN);
3915 }
3916 admitted
3917 }
3918 };
3919 actor.scope_owner = Some(scope.stamp.owner);
3920 actor.agent_id = scope.stamp.attributes.agent_id;
3921 actor.run_id = scope.stamp.attributes.run_id;
3922 actor.flow_id = scope.stamp.attributes.flow_id;
3923 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
3924 actor.project_id = scope.stamp.project_id;
3925 Ok(actor)
3926 }
3927
3928 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3929 self.refresh_capability_requirements();
3930 let mut counters = self.counters.snapshot();
3936 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3937 obj.insert(
3938 "activity_unparseable_lines".into(),
3939 ledger.read_errors().into(),
3940 );
3941 obj.insert(
3942 "activity_lifecycle_append_failures".into(),
3943 ledger.lifecycle_failures().into(),
3944 );
3945 obj.insert(
3946 "activity_started_digest_skips".into(),
3947 ledger.lifecycle_digest_skips().into(),
3948 );
3949 }
3950 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3951 self.forwarding.client_route_concentration(),
3952 counters.as_object_mut(),
3953 ) {
3954 obj.insert(
3955 "client_connections_with_routes".into(),
3956 connections_with_routes.into(),
3957 );
3958 obj.insert("max_routes_on_one_connection".into(), max.into());
3959 }
3960 if let (Some(open_breakers), Some(obj)) = (
3966 self.route_bind_breakers.open_snapshot(),
3967 counters.as_object_mut(),
3968 ) {
3969 obj.insert("route_bind_breakers_open".into(), open_breakers);
3970 }
3971 let response = ClientControlResponse::ServerDescribe {
3972 protocol_ver: PROTOCOL_VERSION,
3973 subc_ops: subc_ops(),
3974 capabilities: self.describe_capabilities(),
3975 connected_clients: self.connected_clients.count(),
3976 counters: Some(counters),
3977 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3978 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3979 capability_requirements: self.capability_requirement_statuses(),
3980 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3981 config_path: self
3982 .rescan
3983 .as_ref()
3984 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
3985 };
3986 Ok(vec![control_response_body_frame(
3987 &frame,
3988 &response,
3989 "ClientControlResponse::ServerDescribe",
3990 )?])
3991 }
3992
3993 fn handle_catalog_list(
3994 &self,
3995 frame: Frame,
3996 module_id: Option<String>,
3997 ) -> Result<Vec<Frame>, RouterError> {
3998 let (generation, modules) = self
3999 .registry
4000 .list_modules_with_nonce_verification()
4001 .map_err(|err| {
4002 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4003 })?;
4004 let entries = modules
4005 .into_iter()
4006 .filter(|(registration, _)| {
4007 module_id
4008 .as_deref()
4009 .map(|wanted| registration.manifest.module_id == wanted)
4010 .unwrap_or(true)
4011 })
4012 .map(|(registration, nonce_verified)| {
4013 let not_ready = self.not_ready_reason(®istration);
4014 let roles = registration.manifest.provides;
4015 CatalogEntry::new(
4016 registration.manifest.module_id,
4017 roles,
4018 registration.control_ops,
4019 )
4020 .with_ready(not_ready.is_none())
4021 .with_not_ready(not_ready)
4022 .with_module_version(Some(registration.manifest.module_version))
4023 .with_capabilities(registration.manifest.capabilities)
4024 .with_self_signals(registration.manifest.self_signals)
4025 .with_events(registration.manifest.events)
4026 .with_operation_delivery(registration.manifest.operation_delivery)
4027 .with_operation_payload(registration.manifest.operation_payload)
4028 .with_rich_ui(registration.manifest.rich_ui)
4029 .with_role_declarations(registration.manifest.role_declarations)
4030 .with_nonce_verified(nonce_verified)
4031 })
4032 .collect();
4033 let response = ClientControlResponse::CatalogList {
4034 generation,
4035 modules: entries,
4036 subc_ops: subc_ops(),
4037 };
4038 Ok(vec![control_response_body_frame(
4039 &frame,
4040 &response,
4041 "ClientControlResponse::CatalogList",
4042 )?])
4043 }
4044
4045 fn attest_route_open_consumer(
4050 &self,
4051 consumer_identity: Option<&ConsumerIdentity>,
4052 ) -> Option<Principal> {
4053 let Some(consumer_identity) = consumer_identity else {
4054 return Some(Principal::Direct);
4055 };
4056 self.supervisor
4057 .spawned_consumer_authorized(
4058 &consumer_identity.module_id,
4059 &consumer_identity.launch_nonce,
4060 )
4061 .then(|| Principal::Reserved {
4062 module_id: consumer_identity.module_id.clone(),
4063 })
4064 }
4065
4066 fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4070 let Ok(ClientControlRequest::RouteOpen {
4071 identity,
4072 consumer_identity,
4073 scope,
4074 ..
4075 }) = parse_client_control_request(&frame.body)
4076 else {
4077 return RouteOpenCaller::unknown();
4078 };
4079 let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4080 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4081 }
4082
4083 fn route_open_principal(
4084 &self,
4085 frame: &Frame,
4086 consumer_identity: Option<ConsumerIdentity>,
4087 ) -> Result<Result<Principal, Frame>, RouterError> {
4088 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4089 self.route_open_principal_attested(frame, consumer_identity, attested)
4090 }
4091
4092 fn route_open_principal_attested(
4096 &self,
4097 frame: &Frame,
4098 consumer_identity: Option<ConsumerIdentity>,
4099 attested: Option<Principal>,
4100 ) -> Result<Result<Principal, Frame>, RouterError> {
4101 if let Some(principal) = attested {
4102 return Ok(Ok(principal));
4103 }
4104 let Some(consumer_identity) = consumer_identity else {
4105 return Ok(Ok(Principal::Direct));
4106 };
4107
4108 Ok(Err(control_error_frame(
4109 frame,
4110 "bad_consumer_identity",
4111 format!(
4112 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4113 consumer_identity.module_id
4114 ),
4115 )?))
4116 }
4117
4118 #[allow(clippy::too_many_arguments)]
4125 fn route_open_refusal_frame(
4126 &self,
4127 ctx: &RouteCtx,
4128 caller: &RouteOpenCaller,
4129 frame: &Frame,
4130 module_id: &str,
4131 reason: &'static str,
4132 code: &'static str,
4133 message: impl Into<String>,
4134 ) -> Result<Frame, RouterError> {
4135 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4136 control_error_frame(frame, code, message.into())
4137 }
4138
4139 #[allow(clippy::too_many_arguments)]
4157 fn route_open_breaker_refusal_frame(
4158 &self,
4159 ctx: &RouteCtx,
4160 caller: &RouteOpenCaller,
4161 frame: &Frame,
4162 module_id: &str,
4163 consecutive_timeouts: u32,
4164 retry_in: Duration,
4165 probe_in_flight: bool,
4166 ) -> Result<Frame, RouterError> {
4167 self.counters
4168 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4169 debug!(
4170 target: "control",
4171 code = "module_timeout",
4172 module_id = ?module_id,
4173 connection_id = ctx.connection_id.get(),
4174 principal = %caller.principal,
4175 scope = %caller.scope,
4176 harness = %caller.harness,
4177 consecutive_timeouts,
4178 retry_in_ms = retry_in.as_millis() as u64,
4179 probe_in_flight,
4180 "route.open refused by open bind-relay breaker"
4181 );
4182 let detail = if probe_in_flight {
4189 "one test route is already being tried; retry once it settles".to_string()
4190 } else {
4191 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4192 };
4193 control_error_frame(
4194 frame,
4195 "module_timeout",
4196 format!(
4197 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4198 timed out in a row); {detail}; its established routes are unaffected"
4199 ),
4200 )
4201 }
4202
4203 fn observe_route_open_refusal(
4216 &self,
4217 ctx: &RouteCtx,
4218 caller: &RouteOpenCaller,
4219 module_id: &str,
4220 reason: &'static str,
4221 code: &'static str,
4222 ) {
4223 self.counters.increment_route_open_refused(code);
4224 info!(
4225 target: "control",
4226 code,
4227 reason,
4228 module_id = ?module_id,
4229 connection_id = ctx.connection_id.get(),
4230 principal = %caller.principal,
4231 scope = %caller.scope,
4232 harness = %caller.harness,
4233 "route.open refused"
4234 );
4235 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4236 self.route_outages.record_not_serving(module_id, reason);
4237 }
4238 }
4239
4240 fn observe_route_open_accept(
4285 &self,
4286 ctx: &RouteCtx,
4287 module_id: &str,
4288 principal: &str,
4289 scope: &str,
4290 ) {
4291 self.route_outages.record_accepted(module_id);
4292 self.counters.increment_route_open_accepted(principal);
4293 info!(
4294 target: "control",
4295 principal,
4296 module_id,
4297 connection_id = ctx.connection_id.get(),
4298 scope = %scope,
4299 "route.open accepted"
4300 );
4301 }
4302
4303 fn supervised_absent_route_open_refusal_frame(
4304 &self,
4305 ctx: &RouteCtx,
4306 caller: &RouteOpenCaller,
4307 frame: &Frame,
4308 module_id: &str,
4309 code: &'static str,
4310 status: &crate::supervise::ModuleStatus,
4311 ) -> Result<Frame, RouterError> {
4312 self.counters.increment_route_open_refused(code);
4313 info!(
4314 target: "control",
4315 code,
4316 reason = "supervised_not_registered",
4317 module_id = ?module_id,
4318 connection_id = ctx.connection_id.get(),
4319 principal = %caller.principal,
4320 scope = %caller.scope,
4321 harness = %caller.harness,
4322 state = %status.state,
4323 enabled = status.enabled,
4324 live = status.live,
4325 "route.open refused"
4326 );
4327 self.route_outages
4331 .record_not_serving(module_id, "supervised_not_registered");
4332 let message = if let Some(pid) = status.waiting_for_previous_pid {
4333 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4334 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4335 })
4336 } else {
4337 format!(
4338 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4339 status.state, status.enabled, status.live
4340 )
4341 };
4342 control_error_frame(frame, code, message)
4343 }
4344
4345 async fn handle_route_open(
4346 &self,
4347 ctx: &RouteCtx,
4348 frame: Frame,
4349 request: RouteOpenRequest,
4350 timings: &mut RouteOpenTimings,
4351 ) -> Result<Vec<Frame>, RouterError> {
4352 let RouteOpenRequest {
4353 target,
4354 mut identity,
4355 consumer_identity,
4356 consumer_capabilities,
4357 role_versions,
4358 admission_facts,
4359 scope,
4360 } = request;
4361 let target_module_id = target_module_id(&target).to_string();
4362 timings.module_id.clone_from(&target_module_id);
4363 if self
4364 .registry
4365 .get_module_by_connection(ctx.connection_id)
4366 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4367 .is_some()
4368 {
4369 return Ok(vec![control_error_frame(
4370 &frame,
4371 "invalid_request",
4372 "module connections cannot open client routes",
4373 )?]);
4374 }
4375 debug!(
4376 connection_id = ctx.connection_id.get(),
4377 corr = frame.header.corr,
4378 module_id = %target_module_id,
4379 "handling route.open"
4380 );
4381
4382 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4387 let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4388
4389 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4395 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4396 self.observe_route_open_refusal(
4397 ctx,
4398 &caller,
4399 &target_module_id,
4400 "invalid_role_versions",
4401 error_codes::INVALID_REQUEST,
4402 );
4403 return Ok(vec![control_error_body_frame(
4404 &frame,
4405 ErrorBody {
4406 code: error_codes::INVALID_REQUEST.to_string(),
4407 message: error.to_string(),
4408 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4409 },
4410 )?]);
4411 }
4412
4413 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4414 if status.waiting_for_previous_pid.is_some() {
4415 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4416 ctx,
4417 &caller,
4418 &frame,
4419 &target_module_id,
4420 "module_warming",
4421 &status,
4422 )?]);
4423 }
4424 }
4425
4426 let Some(registration) = self
4445 .registry
4446 .get_module(&target_module_id)
4447 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4448 else {
4449 if let Some((status, warming)) =
4450 self.supervisor_status(&target_module_id, frame.header.corr)?
4451 {
4452 if status.protocol == ModuleProtocol::None {
4459 return Ok(vec![self.route_open_refusal_frame(
4460 ctx,
4461 &caller,
4462 &frame,
4463 &target_module_id,
4464 "protocol_none",
4465 error_codes::MODULE_NO_PROTOCOL,
4466 format!(
4467 "module_id '{target_module_id}' is declared protocol: none; \
4468 it speaks no subc wire and serves no routes"
4469 ),
4470 )?]);
4471 }
4472 let code = if warming {
4473 "module_warming"
4474 } else {
4475 "target_unavailable"
4476 };
4477 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4478 ctx,
4479 &caller,
4480 &frame,
4481 &target_module_id,
4482 code,
4483 &status,
4484 )?]);
4485 }
4486 if let Some(removed_ago_ms) =
4487 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4488 {
4489 return Ok(vec![self.route_open_refusal_frame(
4490 ctx,
4491 &caller,
4492 &frame,
4493 &target_module_id,
4494 "removed",
4495 error_codes::MODULE_REMOVED,
4496 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4497 )?]);
4498 }
4499 return Ok(vec![self.route_open_refusal_frame(
4500 ctx,
4501 &caller,
4502 &frame,
4503 &target_module_id,
4504 "not_registered",
4505 error_codes::UNKNOWN_MODULE,
4506 format!("module_id '{target_module_id}' is not registered"),
4507 )?]);
4508 };
4509
4510 if !registration.ready {
4514 self.counters
4515 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4516 info!(
4517 target: "control",
4518 code = error_codes::MODULE_WARMING,
4519 module_id = ?target_module_id,
4520 connection_id = ctx.connection_id.get(),
4521 principal = %caller.principal,
4522 scope = %caller.scope,
4523 harness = %caller.harness,
4524 reason = "declared_not_ready",
4525 "route.open refused"
4526 );
4527 self.route_outages
4530 .record_not_serving(&target_module_id, "declared_not_ready");
4531 return Ok(vec![control_error_body_frame(
4532 &frame,
4533 ErrorBody {
4534 code: error_codes::MODULE_WARMING.to_string(),
4535 message: format!(
4536 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4537 ),
4538 detail: Some(serde_json::json!({
4539 "reason": "declared_not_ready"
4540 })),
4541 },
4542 )?]);
4543 }
4544
4545 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4564 self.counters
4565 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4566 info!(
4567 target: "control",
4568 code = error_codes::MODULE_WARMING,
4569 module_id = ?target_module_id,
4570 connection_id = ctx.connection_id.get(),
4571 principal = %caller.principal,
4572 scope = %caller.scope,
4573 harness = %caller.harness,
4574 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4575 capability = %capability,
4576 "route.open refused"
4577 );
4578 return Ok(vec![control_error_body_frame(
4579 &frame,
4580 ErrorBody {
4581 code: error_codes::MODULE_WARMING.to_string(),
4582 message: format!(
4583 "module_id '{target_module_id}' requires capability '{capability}', \
4584 which no registered module provides; retry"
4585 ),
4586 detail: Some(serde_json::json!({
4587 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4588 "capability": capability,
4589 })),
4590 },
4591 )?]);
4592 }
4593
4594 if !target_has_required_role(&target, ®istration.manifest.provides) {
4595 return Ok(vec![self.route_open_refusal_frame(
4596 ctx,
4597 &caller,
4598 &frame,
4599 &target_module_id,
4600 "role_not_provided",
4601 "target_unavailable",
4602 format!("module_id '{target_module_id}' does not provide the requested target"),
4603 )?]);
4604 }
4605
4606 if registration.state != ChannelState::Active {
4607 return Ok(vec![self.route_open_refusal_frame(
4608 ctx,
4609 &caller,
4610 &frame,
4611 &target_module_id,
4612 "registration_not_active",
4613 "target_unavailable",
4614 format!("module_id '{target_module_id}' is not active"),
4615 )?]);
4616 }
4617
4618 if self
4619 .forwarding
4620 .module_is_draining(&target_module_id)
4621 .map_err(RouterError::Forwarding)?
4622 {
4623 return Ok(vec![self.route_open_refusal_frame(
4624 ctx,
4625 &caller,
4626 &frame,
4627 &target_module_id,
4628 "reloading",
4629 "module_reloading",
4630 format!("module_id '{target_module_id}' is reloading"),
4631 )?]);
4632 }
4633
4634 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4635 process_liveness.process_live(&target_module_id) == Some(false)
4636 }) {
4637 if process_liveness.process_replacing(&target_module_id) {
4644 return Ok(vec![self.route_open_refusal_frame(
4645 ctx,
4646 &caller,
4647 &frame,
4648 &target_module_id,
4649 "reloading",
4650 "module_reloading",
4651 format!("module_id '{target_module_id}' is reloading"),
4652 )?]);
4653 }
4654 return Ok(vec![self.route_open_refusal_frame(
4655 ctx,
4656 &caller,
4657 &frame,
4658 &target_module_id,
4659 "supervisor_not_live",
4660 "target_unavailable",
4661 format!("module_id '{target_module_id}' is not live"),
4662 )?]);
4663 }
4664
4665 if !self
4666 .forwarding
4667 .has_live_module_connection(&target_module_id)
4668 .map_err(RouterError::Forwarding)?
4669 {
4670 return Ok(vec![self.route_open_refusal_frame(
4671 ctx,
4672 &caller,
4673 &frame,
4674 &target_module_id,
4675 "no_forwarding_connection",
4676 "target_unavailable",
4677 format!("module_id '{target_module_id}' has no live forwarding connection"),
4678 )?]);
4679 }
4680
4681 if let Some(error) =
4682 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4683 {
4684 self.observe_route_open_refusal(
4685 ctx,
4686 &caller,
4687 &target_module_id,
4688 "op_not_allowed",
4689 "op_not_allowed",
4690 );
4691 return Ok(vec![error]);
4692 }
4693
4694 let principal =
4695 match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
4696 Ok(principal) => principal,
4697 Err(error) => {
4698 self.observe_route_open_refusal(
4699 ctx,
4700 &caller,
4701 &target_module_id,
4702 "bad_consumer_identity",
4703 "bad_consumer_identity",
4704 );
4705 return Ok(vec![error]);
4706 }
4707 };
4708 timings.principal = principal_label(&principal);
4709
4710 if let Principal::Reserved {
4714 module_id: opening_module_id,
4715 } = &principal
4716 {
4717 if let Some(opening_registration) = self
4718 .registry
4719 .get_module(opening_module_id)
4720 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4721 {
4722 if let Some(capability) =
4723 denied_capability(&opening_registration.manifest, ®istration.manifest)
4724 {
4725 warn!(
4726 opening_module_id,
4727 target_module_id,
4728 capability,
4729 "refusing route.open because an attested capability deny edge matches"
4730 );
4731 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4732 &frame,
4733 &target_module_id,
4734 "capability_deny_edge",
4735 "capability_forbidden",
4736 format!(
4737 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4738 ),
4739 )?]);
4740 }
4741 }
4742 }
4743
4744 if admission_facts.is_some() {
4745 let carrier_matches = matches!(
4746 &principal,
4747 Principal::Reserved { module_id }
4748 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4749 );
4750 if !carrier_matches {
4751 return Ok(vec![self.route_open_refusal_frame(
4752 ctx,
4753 &caller,
4754 &frame,
4755 &target_module_id,
4756 "admission_facts_carrier_not_permitted",
4757 "admission_facts_not_permitted",
4758 "admission facts may only be carried by the configured reserved module",
4759 )?]);
4760 }
4761
4762 let target_allowed = self
4763 .admission_facts_targets
4764 .as_ref()
4765 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4766 if !target_allowed {
4767 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4768 &frame,
4769 &target_module_id,
4770 "admission_facts_target_not_listed",
4771 "admission_facts_target_not_allowed",
4772 format!(
4773 "admission facts are not permitted for target module_id '{target_module_id}'"
4774 ),
4775 )?]);
4776 }
4777
4778 }
4782
4783 let (bound_scope, mut scope_stamp) = match &scope {
4789 None => (None, None),
4790 Some(selector) => {
4791 let owner_configured = match &selector.owner {
4792 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4798 _ => false,
4799 };
4800 let mut scopes = self
4801 .scopes
4802 .write()
4803 .unwrap_or_else(|poisoned| poisoned.into_inner());
4804 let admitted = scopes
4805 .admit(&principal, &target_module_id, selector, owner_configured)
4806 .and_then(|admission| {
4807 crate::scopes::check_target_flow_support(
4808 &admission.stamp,
4809 &target_module_id,
4810 registration.manifest.capabilities.as_ref(),
4811 )?;
4812 crate::scopes::check_target_agent_run_support(
4813 &admission.stamp,
4814 &target_module_id,
4815 registration.manifest.capabilities.as_ref(),
4816 )?;
4817 let bound = scopes.reserve_bind(
4818 &admission,
4819 &principal,
4820 ctx.connection_id,
4821 selector.hold_presence,
4822 );
4823 Ok((admission, bound))
4824 });
4825 match admitted {
4826 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4827 Err(refusal) => {
4828 return Ok(vec![self.route_open_refusal_frame(
4829 ctx,
4830 &caller,
4831 &frame,
4832 &target_module_id,
4833 refusal.code,
4834 refusal.code,
4835 refusal.message,
4836 )?]);
4837 }
4838 }
4839 }
4840 };
4841 if let Some(stamp) = &scope_stamp {
4844 caller.scope = route_open_scope_label(stamp);
4845 }
4846
4847 timings.enter(RouteOpenPhase::ProjectRoot);
4869 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4870 Ok(project_root) => project_root,
4871 Err(err) => {
4872 return Ok(vec![control_error_frame(
4873 &frame,
4874 "invalid_project_root",
4875 err.to_string(),
4876 )?])
4877 }
4878 };
4879 timings.enter(RouteOpenPhase::Admission);
4880 identity.project_root = project_root.as_path().to_path_buf();
4881
4882 let _concurrency_guard = match self
4893 .route_bind_concurrency
4894 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4895 {
4896 Ok(guard) => guard,
4897 Err(in_flight) => {
4898 return Ok(vec![self.route_open_target_capacity_refusal(
4899 ctx,
4900 &caller,
4901 &frame,
4902 &target_module_id,
4903 in_flight,
4904 )?]);
4905 }
4906 };
4907
4908 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4911 RouteBindAdmission::Admitted { guard, probe } => {
4912 if probe {
4913 info!(
4914 module_id = %target_module_id,
4915 connection_id = ctx.connection_id.get(),
4916 "route.bind breaker half-open: admitting one probe"
4917 );
4918 }
4919 guard
4920 }
4921 RouteBindAdmission::Refused {
4922 consecutive_timeouts,
4923 retry_in,
4924 probe_in_flight,
4925 } => {
4926 return Ok(vec![self.route_open_breaker_refusal_frame(
4927 ctx,
4928 &caller,
4929 &frame,
4930 &target_module_id,
4931 consecutive_timeouts,
4932 retry_in,
4933 probe_in_flight,
4934 )?]);
4935 }
4936 };
4937
4938 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4943 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4944 timings.enter(RouteOpenPhase::EgressReservation);
4945 let pending = match self
4946 .forwarding
4947 .begin_route_bind_relay_for(
4948 ctx.connection_id,
4949 ctx.egress.clone(),
4950 response_version(&frame),
4951 frame.header.corr,
4952 &target_module_id,
4953 principal.clone(),
4954 bound_scope,
4955 Some(project_root),
4956 relay_deadline,
4957 )
4958 .await
4959 {
4960 Ok(pending) => pending,
4961 Err(err) => {
4962 return Ok(vec![self.route_open_refusal_frame(
4963 ctx,
4964 &caller,
4965 &frame,
4966 &target_module_id,
4967 "relay_reservation_failed",
4968 forwarding_error_code(&err),
4969 err.to_string(),
4970 )?])
4971 }
4972 };
4973 timings.enter(RouteOpenPhase::Admission);
4974 let crate::forwarding::PendingRouteBindRelay {
4975 endpoint,
4976 module_sink,
4977 negotiated_ver,
4978 client_channel,
4979 client_epoch,
4980 module_channel,
4981 module_epoch,
4982 corr: relay_corr,
4983 receiver,
4984 } = pending;
4985 let mut reservation =
4986 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4987
4988 let relay_registration = self
4993 .registry
4994 .get_module_by_connection(endpoint.connection_id)
4995 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4996 let relay_capabilities = relay_registration
4997 .as_ref()
4998 .and_then(|registration| registration.manifest.capabilities.as_ref());
4999 if let Some(selector) = &scope {
5000 if let Err(refusal) = self
5003 .scopes
5004 .read()
5005 .unwrap_or_else(|poisoned| poisoned.into_inner())
5006 .check_bind_proof(&principal, selector)
5007 {
5008 reservation.release_and_disarm();
5009 return Ok(vec![self.route_open_refusal_frame(
5010 ctx,
5011 &caller,
5012 &frame,
5013 &target_module_id,
5014 refusal.code,
5015 refusal.code,
5016 refusal.message,
5017 )?]);
5018 }
5019 }
5020 if let Some(stamp) = scope_stamp
5021 .as_ref()
5022 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5023 {
5024 if let Err(refusal) = crate::scopes::check_target_flow_support(
5025 stamp,
5026 &target_module_id,
5027 relay_capabilities,
5028 )
5029 .and_then(|()| {
5030 crate::scopes::check_target_agent_run_support(
5031 stamp,
5032 &target_module_id,
5033 relay_capabilities,
5034 )
5035 }) {
5036 reservation.release_and_disarm();
5037 return Ok(vec![self.route_open_refusal_frame(
5038 ctx,
5039 &caller,
5040 &frame,
5041 &target_module_id,
5042 refusal.code,
5043 refusal.code,
5044 refusal.message,
5045 )?]);
5046 }
5047 }
5048
5049 if !relay_capabilities.is_some_and(|caps| {
5052 caps.provides
5053 .iter()
5054 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5055 }) {
5056 if let Some(stamp) = scope_stamp.as_mut() {
5057 stamp.workdir = None;
5058 stamp.project_id = None;
5059 }
5060 }
5061 let worker_run = scope_stamp
5066 .as_ref()
5067 .is_some_and(crate::scopes::is_worker_run)
5068 && relay_capabilities.is_some_and(|caps| {
5069 caps.provides
5070 .iter()
5071 .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5072 });
5073 if !worker_run
5074 && !relay_capabilities.is_some_and(|caps| {
5075 caps.provides
5076 .iter()
5077 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5078 })
5079 {
5080 if let Some(stamp) = scope_stamp.as_mut() {
5081 stamp.attributes.on_behalf_of_agent = None;
5082 }
5083 }
5084 if !relay_capabilities.is_some_and(|caps| {
5085 caps.provides
5086 .iter()
5087 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5088 }) {
5089 if let Some(stamp) = scope_stamp.as_mut() {
5090 stamp.attributes.flow_version = None;
5091 stamp.attributes.flow_code_hash = None;
5092 stamp.attributes.flow_install_id = None;
5093 }
5094 }
5095 let call_mode = if worker_run
5096 || relay_capabilities
5097 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5098 {
5099 Some(derive_call_mode(scope_stamp.as_ref()))
5100 } else {
5101 if let Some(stamp) = scope_stamp.as_mut() {
5102 stamp.expires_at_ms = None;
5103 }
5104 None
5105 };
5106
5107 debug!(
5108 connection_id = ctx.connection_id.get(),
5109 client_channel,
5110 client_epoch,
5111 module_channel,
5112 module_epoch,
5113 "reserved route handle pair"
5114 );
5115 let route_principal_label = principal_label(&principal);
5118 let relay = ModuleControlRequest::RouteBind {
5119 route_channel: module_channel,
5120 epoch: module_epoch,
5121 target,
5122 identity,
5123 principal: Some(principal),
5124 consumer_capabilities,
5125 role_versions,
5126 admission_facts,
5127 scope: scope_stamp,
5128 call_mode,
5129 };
5130 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
5131 RouterError::backend(
5132 0,
5133 frame.header.corr,
5134 format!("failed to encode route.bind request: {err}"),
5135 )
5136 })?;
5137 let relay_frame = Frame::build_with_version(
5138 negotiated_ver,
5139 FrameType::Request,
5140 control_flags(),
5141 0,
5142 0,
5143 relay_corr,
5144 relay_body,
5145 )
5146 .map_err(RouterError::FrameBuild)?;
5147
5148 timings.enter(RouteOpenPhase::BindSend);
5149 let sent = module_sink.send(relay_frame).await;
5150 timings.enter(RouteOpenPhase::Admission);
5151 if let Err(err) = sent {
5152 reservation.release_and_disarm();
5153 return Ok(vec![self.route_open_refusal_frame(
5154 ctx,
5155 &caller,
5156 &frame,
5157 &target_module_id,
5158 "relay_send_failed",
5159 "target_unavailable",
5160 err.to_string(),
5161 )?]);
5162 }
5163
5164 #[cfg(test)]
5165 if let Some(hold) = &self.route_bind_relay_sent_hold {
5166 hold.wait().await;
5170 hold.wait().await;
5171 }
5172
5173 if self
5178 .forwarding
5179 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5180 .map_err(RouterError::Forwarding)?
5181 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5182 {
5183 self.send_abandoned_route_bind_goodbye(
5184 &module_sink,
5185 negotiated_ver,
5186 module_channel,
5187 module_epoch,
5188 );
5189 }
5190
5191 timings.enter(RouteOpenPhase::BindAckWait);
5192 let outcome = timeout_at(relay_deadline, receiver).await;
5193 timings.enter(RouteOpenPhase::Admission);
5194 match outcome {
5195 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5196 reservation.disarm();
5197 if breaker.record_accepted() {
5198 info!(
5199 module_id = %target_module_id,
5200 "route.bind breaker closed: the probe was accepted"
5201 );
5202 }
5203 self.observe_route_open_accept(
5204 ctx,
5205 &target_module_id,
5206 &route_principal_label,
5207 &caller.scope,
5208 );
5209 Ok(Vec::new())
5210 }
5211 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5212 reservation.release_and_disarm();
5213 breaker.record_inconclusive();
5217 let scope_code = match body.code.as_str() {
5221 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
5222 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
5223 _ => None,
5224 };
5225 if let Some(code) = scope_code {
5226 self.observe_route_open_refusal(
5227 ctx,
5228 &caller,
5229 &target_module_id,
5230 "scope_changed_before_commit",
5231 code,
5232 );
5233 return Ok(vec![control_error_body_frame(&frame, body)?]);
5234 }
5235 self.counters
5236 .increment_route_open_refused("module_rejected");
5237 info!(
5238 target: "control",
5239 code = "module_rejected",
5240 module_code = ?body.code,
5241 module_id = ?target_module_id,
5242 connection_id = ctx.connection_id.get(),
5243 principal = %caller.principal,
5244 scope = %caller.scope,
5245 harness = %caller.harness,
5246 "route.open refused"
5247 );
5248 Ok(vec![control_error_body_frame(&frame, body)?])
5249 }
5250 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5251 reservation.release_and_disarm();
5252 breaker.record_inconclusive();
5253 tracing::warn!(
5260 module_id = %target_module_id,
5261 "route.bind relay abandoned: {message}"
5262 );
5263 Ok(vec![self.route_open_refusal_frame(
5264 ctx,
5265 &caller,
5266 &frame,
5267 &target_module_id,
5268 "relay_abandoned",
5269 "target_unavailable",
5270 message,
5271 )?])
5272 }
5273 Ok(Err(_)) => {
5274 reservation.release_and_disarm();
5275 breaker.record_inconclusive();
5276 Ok(vec![self.route_open_refusal_frame(
5277 ctx,
5278 &caller,
5279 &frame,
5280 &target_module_id,
5281 "relay_waiter_canceled",
5282 "target_unavailable",
5283 "route.bind relay waiter was canceled before the module responded",
5284 )?])
5285 }
5286 Err(_) => {
5287 reservation.release_and_disarm();
5288 if let Some(opened) = breaker.record_timeout(
5292 self.route_bind_breaker_threshold,
5293 self.route_bind_breaker_cooldown,
5294 ) {
5295 warn!(
5296 module_id = %target_module_id,
5297 consecutive_timeouts = opened.consecutive_timeouts,
5298 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5299 reopened_after_probe = opened.reopened_after_probe,
5300 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5301 );
5302 }
5303 tracing::warn!(
5311 module_id = %target_module_id,
5312 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5313 "route.bind relay timed out: module did not ack within budget"
5314 );
5315 Ok(vec![self.route_open_refusal_frame(
5316 ctx,
5317 &caller,
5318 &frame,
5319 &target_module_id,
5320 "relay_timed_out",
5321 "module_timeout",
5322 format!(
5323 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5324 route_bind_relay_timeout
5325 ),
5326 )?])
5327 }
5328 }
5329 }
5330
5331 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5332 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5333 snapshot: self.supervisor.spawn_snapshot(),
5334 };
5335 Ok(vec![control_response_body_frame(
5336 &frame,
5337 &response,
5338 "ClientControlResponse::SupervisorSpawnSnapshot",
5339 )?])
5340 }
5341
5342 fn handle_supervisor_spawn_subscribe(
5343 &self,
5344 ctx: &RouteCtx,
5345 frame: Frame,
5346 since: Option<SpawnCursor>,
5347 ) -> Result<Vec<Frame>, RouterError> {
5348 match self.supervisor.subscribe_spawns(
5349 ctx.connection_id,
5350 frame.header.corr,
5351 response_version(&frame),
5352 since,
5353 ctx.egress.clone(),
5354 ) {
5355 Ok(()) => Ok(Vec::new()),
5356 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5357 Ok(vec![control_error_body_frame(
5358 &frame,
5359 ErrorBody {
5360 code: "spawn_cursor_incarnation_mismatch".to_string(),
5361 message: "spawn cursor belongs to a different daemon incarnation"
5362 .to_string(),
5363 detail: Some(serde_json::json!({
5364 "current_daemon_incarnation": current
5365 })),
5366 },
5367 )?])
5368 }
5369 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5370 &frame,
5371 ErrorBody {
5372 code: "spawn_cursor_too_old".to_string(),
5373 message: "spawn cursor predates the retained event ring".to_string(),
5374 detail: Some(serde_json::json!({
5375 "oldest_retained_cursor": oldest
5376 })),
5377 },
5378 )?]),
5379 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5380 0,
5381 frame.header.corr,
5382 format!("failed to open supervisor spawn subscription: {error}"),
5383 )),
5384 }
5385 }
5386
5387 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5388 let generation = self
5389 .registry
5390 .generation()
5391 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5392 let mut modules = Vec::new();
5393 for module in self.supervisor.list() {
5394 let status = module.status_for_control("list").map_err(|err| {
5395 RouterError::backend(
5396 0,
5397 frame.header.corr,
5398 format!("failed to read supervisor status: {err}"),
5399 )
5400 })?;
5401 let (configured, _) = module.configuration().map_err(|err| {
5402 RouterError::backend(
5403 0,
5404 frame.header.corr,
5405 format!("failed to read module configuration: {err}"),
5406 )
5407 })?;
5408 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
5409 RouterError::backend(
5410 0,
5411 frame.header.corr,
5412 format!("failed to read retry time: {err}"),
5413 )
5414 })?;
5415 let image = module.running_image_agreement().await;
5417 let resources = Some(module.child_resource_usage());
5420 let mut verdict =
5421 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5422 verdict.spec = Some(launch_spec_agreement(
5423 &configured,
5424 status.process_alive,
5425 status.launch_spec_digest.as_ref(),
5426 ));
5427 let pending_reload = Some(verdict);
5428 let configured_enabled = self
5429 .capability_evaluator
5430 .configured_enabled(&status.module_id);
5431 modules.push(
5434 SupervisorEntry::new(
5435 status.module_id,
5436 status.state.to_string(),
5437 status.enabled,
5438 status.live,
5439 status.health.status,
5440 )
5441 .with_configured_enabled(configured_enabled)
5442 .with_next_retry_at_ms(next_retry_at_ms)
5443 .with_launch_nonce_env(Some(!cfg!(unix)))
5444 .with_launch_nonce_source(status.launch_nonce_source)
5445 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5446 .with_protocol(status.protocol)
5447 .with_pending_reload(pending_reload)
5448 .with_last_probe_ms(status.health.last_probe_ms)
5449 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5450 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5451 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5452 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5453 .with_restart_count(Some(status.restart_count))
5454 .with_max_restarts(Some(status.max_restarts))
5455 .with_lifetime_restarts(Some(status.lifetime_restarts))
5456 .with_spawn_generation(Some(status.spawn_generation))
5457 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5458 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5459 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5460 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5461 .with_resources(resources),
5462 );
5463 }
5464 let response = ClientControlResponse::SupervisorList {
5465 generation,
5466 modules,
5467 };
5468 Ok(vec![control_response_body_frame(
5469 &frame,
5470 &response,
5471 "ClientControlResponse::SupervisorList",
5472 )?])
5473 }
5474
5475 fn handle_supervisor_stderr_tail(
5476 &self,
5477 frame: Frame,
5478 module_id: String,
5479 max_lines: Option<u32>,
5480 max_bytes: Option<u32>,
5481 ) -> Result<Vec<Frame>, RouterError> {
5482 let Some(module) = self.supervisor.get(&module_id) else {
5483 return Ok(vec![control_error_frame(
5484 &frame,
5485 "unknown_module",
5486 format!("module_id '{module_id}' is not supervised"),
5487 )?]);
5488 };
5489
5490 let snapshot = module.stderr_tail(
5491 max_lines.map(|value| value as usize),
5492 max_bytes.map(|value| value as usize),
5493 );
5494
5495 let response = ClientControlResponse::SupervisorStderrTail {
5496 module_id,
5497 tail: StderrTail {
5498 capture: match snapshot.capture {
5499 CaptureState::Captured => StderrCaptureState::Captured,
5500 CaptureState::Incomplete { reason } => {
5501 StderrCaptureState::Incomplete { reason }
5502 }
5503 CaptureState::NotCaptured { reason } => {
5504 StderrCaptureState::NotCaptured { reason }
5505 }
5506 },
5507 entries: snapshot
5508 .entries
5509 .into_iter()
5510 .map(|entry| match entry {
5511 TailEntry::Line {
5512 text,
5513 truncated,
5514 at_ms,
5515 } => StderrTailEntry::Line {
5516 text,
5517 truncated,
5518 at_ms,
5519 },
5520 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5521 })
5522 .collect(),
5523 dropped_lines: snapshot.dropped_lines,
5524 },
5525 };
5526 Ok(vec![control_response_body_frame(
5527 &frame,
5528 &response,
5529 "ClientControlResponse::SupervisorStderrTail",
5530 )?])
5531 }
5532
5533 async fn handle_supervisor_terminals(
5534 &self,
5535 frame: Frame,
5536 module_id: String,
5537 ) -> Result<Vec<Frame>, RouterError> {
5538 let Some(module) = self.supervisor.get(&module_id) else {
5539 return Ok(vec![control_error_frame(
5540 &frame,
5541 "unknown_module",
5542 format!("module_id '{module_id}' is not supervised"),
5543 )?]);
5544 };
5545
5546 let terminals = module
5549 .read_durable_terminal_history()
5550 .await
5551 .map_err(|error| {
5552 RouterError::backend(
5553 0,
5554 frame.header.corr,
5555 format!("failed to read terminal history: {error}"),
5556 )
5557 })?;
5558 let response = ClientControlResponse::SupervisorTerminals {
5559 module_id,
5560 terminals,
5561 };
5562 Ok(vec![control_response_body_frame(
5563 &frame,
5564 &response,
5565 "ClientControlResponse::SupervisorTerminals",
5566 )?])
5567 }
5568
5569 fn handle_supervisor_routes(
5570 &self,
5571 frame: Frame,
5572 module_id: Option<String>,
5573 ) -> Result<Vec<Frame>, RouterError> {
5574 let modules = self
5575 .forwarding
5576 .route_census(module_id.as_deref())
5577 .map_err(RouterError::Forwarding)?
5578 .into_iter()
5579 .map(|(module_id, routes)| SupervisorRouteModule {
5580 module_id,
5581 routes: routes
5582 .into_iter()
5583 .map(|route| SupervisorRoute {
5584 consumer: match route.principal {
5585 Principal::Reserved { module_id } => {
5586 SupervisorRouteConsumer::Reserved { module_id }
5587 }
5588 Principal::Direct | Principal::Unverified => {
5589 SupervisorRouteConsumer::Direct {
5590 connection_id: route.goodbye_target.connection_id.get(),
5591 }
5592 }
5593 },
5594 age_ms: Instant::now()
5595 .saturating_duration_since(route.bound_at)
5596 .as_millis()
5597 .try_into()
5598 .unwrap_or(u64::MAX),
5599 draining: route.draining,
5600 drain_reason: route.drain_reason,
5601 })
5602 .collect(),
5603 })
5604 .collect();
5605 let response = ClientControlResponse::SupervisorRoutes { modules };
5606 Ok(vec![control_response_body_frame(
5607 &frame,
5608 &response,
5609 "ClientControlResponse::SupervisorRoutes",
5610 )?])
5611 }
5612
5613 async fn handle_supervisor_provenance(
5614 &self,
5615 frame: Frame,
5616 module_id: Option<String>,
5617 ) -> Result<Vec<Frame>, RouterError> {
5618 let mut selected = if let Some(module_id) = module_id {
5619 let Some(module) = self.supervisor.get(&module_id) else {
5620 return Ok(vec![control_error_frame(
5621 &frame,
5622 "unknown_module",
5623 format!("module_id '{module_id}' is not supervised"),
5624 )?]);
5625 };
5626 vec![module]
5627 } else {
5628 self.supervisor.list()
5629 };
5630
5631 let mut modules = Vec::with_capacity(selected.len());
5632 for module in selected.drain(..) {
5633 let (status, observed_image) = module
5634 .status_and_running_image_agreement()
5635 .await
5636 .map_err(|err| {
5637 RouterError::backend(
5638 0,
5639 frame.header.corr,
5640 format!("failed to read supervisor status: {err}"),
5641 )
5642 })?;
5643 let module_declared = self
5644 .registry
5645 .get_module(&status.module_id)
5646 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5647 .and_then(|registration| registration.manifest.provenance)
5648 .map(|build| ModuleDeclaredProvenance::Reported { build })
5649 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5650 #[cfg(test)]
5651 let running_image = match &self.provenance_probe_override {
5652 Some(result) => result.clone(),
5653 None => observed_image,
5654 };
5655 #[cfg(not(test))]
5656 let running_image = observed_image;
5657 modules.push(SupervisorModuleProvenance {
5658 module_id: status.module_id,
5659 module_declared,
5660 daemon_observed: SupervisorObservedProcess {
5661 launch_nonce_source: status.launch_nonce_source,
5662 pid: status.pid,
5663 spawned_at_ms: status.spawned_at_ms,
5664 spawned_from: status.spawned_from,
5665 running_image,
5666 },
5667 });
5668 }
5669 let daemon = SupervisorDaemonProvenance {
5670 daemon_build: self.daemon_provenance.build.clone(),
5671 daemon_observed: DaemonObservedProcess {
5672 pid: self.daemon_provenance.pid,
5673 started_at_ms: self
5674 .daemon_provenance
5675 .start_clock
5676 .map(|clock| clock.started_at_ms())
5677 .or(self.daemon_provenance.started_at_ms),
5678 running_image: self
5679 .daemon_provenance
5680 .probe
5681 .observe(
5682 self.daemon_provenance.pid,
5683 self.daemon_provenance.executable_path.as_deref(),
5684 self.daemon_provenance.executable_identity.clone(),
5685 self.daemon_provenance.process_start_time,
5686 )
5687 .await,
5688 },
5689 };
5690 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5691 Ok(vec![control_response_body_frame(
5692 &frame,
5693 &response,
5694 "ClientControlResponse::SupervisorProvenance",
5695 )?])
5696 }
5697
5698 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5699 self.refresh_capability_requirements();
5700 let generation = self
5701 .registry
5702 .generation()
5703 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5704 let modules = self
5705 .supervisor
5706 .list()
5707 .into_iter()
5708 .map(|module| {
5709 let status = module.status_for_control("health").map_err(|err| {
5710 RouterError::backend(
5711 0,
5712 frame.header.corr,
5713 format!("failed to read supervisor health: {err}"),
5714 )
5715 })?;
5716 let module_id = status.module_id;
5717 let capability_detail = self
5718 .capability_evaluator
5719 .required_problem_detail(&module_id);
5720 Ok(SupervisorHealthEntry {
5721 module_id,
5722 status: status.health.status,
5723 detail: append_capability_problem_detail(
5724 status.health.detail,
5725 capability_detail,
5726 ),
5727 metrics: status.health.metrics,
5728 consecutive_failures: status.health.consecutive_failures,
5729 late_answer_count: status.health.late_answer_count,
5730 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5731 last_action: status.health.last_action,
5732 last_action_ms: status.health.last_action_ms,
5733 last_probe_ms: status.health.last_probe_ms,
5734 })
5735 })
5736 .collect::<Result<Vec<_>, RouterError>>()?;
5737 let response = ClientControlResponse::SupervisorHealth {
5738 generation,
5739 modules,
5740 };
5741 Ok(vec![control_response_body_frame(
5742 &frame,
5743 &response,
5744 "ClientControlResponse::SupervisorHealth",
5745 )?])
5746 }
5747
5748 async fn handle_supervisor_restart(
5749 &self,
5750 frame: Frame,
5751 module_id: String,
5752 drain_timeout_ms: Option<u64>,
5753 requester: Principal,
5754 ) -> Result<Vec<Frame>, RouterError> {
5755 let operation_lock = self.supervisor.operation_lock();
5756 let _operation_guard = operation_lock.lock().await;
5757 let Some(module) = self.supervisor.get(&module_id) else {
5758 return Ok(vec![control_error_frame(
5759 &frame,
5760 "unknown_module",
5761 format!("module_id '{module_id}' is not supervised"),
5762 )?]);
5763 };
5764
5765 self.route_outages.mark_operator_action(&module_id);
5766 if let Err(err) = module
5767 .restart_requested(drain_timeout_ms, Some(requester))
5768 .await
5769 {
5770 self.route_outages
5771 .operator_action_ended_unrefused(&module_id);
5772 let (code, message) = match err {
5773 crate::supervise::SuperviseError::Disabled { .. } => {
5774 ("module_disabled", err.to_string())
5775 }
5776 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5777 ("swap_in_progress", err.to_string())
5778 }
5779 _ => (
5780 "target_unavailable",
5781 format!("failed to restart module_id '{module_id}': {err}"),
5782 ),
5783 };
5784 return Ok(vec![control_error_frame(&frame, code, message)?]);
5785 }
5786
5787 let response = ClientControlResponse::SupervisorAck {
5788 module_id,
5789 applied: true,
5790 };
5791 Ok(vec![control_response_body_frame(
5792 &frame,
5793 &response,
5794 "ClientControlResponse::SupervisorAck",
5795 )?])
5796 }
5797
5798 async fn handle_supervisor_swap(
5802 &self,
5803 frame: Frame,
5804 module_id: String,
5805 ready_timeout_ms: Option<u64>,
5806 requester: Principal,
5807 ) -> Result<Vec<Frame>, RouterError> {
5808 let module = {
5815 let operation_lock = self.supervisor.operation_lock();
5816 let _operation_guard = operation_lock.lock().await;
5817 self.supervisor.get(&module_id)
5818 };
5819 let Some(module) = module else {
5820 return Ok(vec![control_error_frame(
5821 &frame,
5822 "unknown_module",
5823 format!("module_id '{module_id}' is not supervised"),
5824 )?]);
5825 };
5826
5827 self.route_outages.mark_operator_action(&module_id);
5828 if let Err(err) = module
5829 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5830 .await
5831 {
5832 self.route_outages
5833 .operator_action_ended_unrefused(&module_id);
5834 use crate::supervise::SuperviseError;
5835 let message = err.to_string();
5836 let error = match err {
5837 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5838 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5839 code: "swap_refused".to_string(),
5840 message,
5841 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5842 },
5843 SuperviseError::SwapFailed {
5844 arm,
5845 candidate_exit,
5846 ..
5847 } => ErrorBody {
5848 code: "swap_failed".to_string(),
5849 message,
5850 detail: Some(serde_json::json!({
5851 "arm": arm.as_str(),
5852 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5853 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5854 })),
5855 },
5856 _ => ErrorBody::new(
5857 "target_unavailable",
5858 format!("failed to swap module_id '{module_id}': {message}"),
5859 ),
5860 };
5861 return Ok(vec![control_error_body_frame(&frame, error)?]);
5862 }
5863 self.route_outages
5867 .operator_action_ended_unrefused(&module_id);
5868
5869 let response = ClientControlResponse::SupervisorAck {
5870 module_id,
5871 applied: true,
5872 };
5873 Ok(vec![control_response_body_frame(
5874 &frame,
5875 &response,
5876 "ClientControlResponse::SupervisorAck",
5877 )?])
5878 }
5879
5880 async fn handle_supervisor_reload(
5881 &self,
5882 frame: Frame,
5883 module_id: String,
5884 requester: Principal,
5885 ) -> Result<Vec<Frame>, RouterError> {
5886 let operation_lock = self.supervisor.operation_lock();
5887 let _operation_guard = operation_lock.lock().await;
5888 let Some(module) = self.supervisor.get(&module_id) else {
5889 return Ok(vec![control_error_frame(
5890 &frame,
5891 "unknown_module",
5892 format!("module_id '{module_id}' is not supervised"),
5893 )?]);
5894 };
5895
5896 self.route_outages.mark_operator_action(&module_id);
5897 if let Err(err) = module.reload_requested(Some(requester)).await {
5898 self.route_outages
5899 .operator_action_ended_unrefused(&module_id);
5900 let (code, message) = match err {
5901 crate::supervise::SuperviseError::Disabled { .. } => {
5902 ("module_disabled", err.to_string())
5903 }
5904 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5905 ("swap_in_progress", err.to_string())
5906 }
5907 _ => (
5908 "reload_failed",
5909 format!("failed to reload module_id '{module_id}': {err}"),
5910 ),
5911 };
5912 return Ok(vec![control_error_frame(&frame, code, message)?]);
5913 }
5914
5915 let response = ClientControlResponse::SupervisorAck {
5916 module_id,
5917 applied: true,
5918 };
5919 Ok(vec![control_response_body_frame(
5920 &frame,
5921 &response,
5922 "ClientControlResponse::SupervisorAck",
5923 )?])
5924 }
5925
5926 async fn handle_supervisor_rescan(
5930 &self,
5931 frame: Frame,
5932 mode: RescanMode,
5933 requester: Principal,
5934 ) -> Result<Vec<Frame>, RouterError> {
5935 let preview = mode.previews();
5936 let Some(context) = self.rescan.clone() else {
5937 return Ok(vec![control_error_frame(
5938 &frame,
5939 "rescan_unavailable",
5940 "the daemon was not started with a reloadable config path".to_string(),
5941 )?]);
5942 };
5943
5944 let operation_lock = self.supervisor.operation_lock();
5945 let _operation_guard = operation_lock.lock().await;
5946 let loaded = match self.load_rescan_config(&mode, &context) {
5950 Ok(config) => config,
5951 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5952 };
5953 let Some((config, activity_config)) = loaded else {
5961 return Ok(vec![control_error_frame(
5962 &frame,
5963 "invalid_daemon_config",
5964 format!(
5965 "daemon config not found at {}; refusing to rescan (an absent config would \
5966 retire every supervised module)",
5967 context.config_path.display()
5968 ),
5969 )?]);
5970 };
5971 let (
5972 configured_port,
5973 storage_config,
5974 admission_facts_carrier_module_id,
5975 admission_facts_targets,
5976 scope_authority_owners,
5977 privileged_operator_prompt,
5978 modules,
5979 reserved_capabilities,
5980 ) = (
5981 config.port,
5982 config.storage,
5983 config.admission_facts_carrier_module_id,
5984 config.admission_facts_targets,
5985 config.scope_authority_owners,
5986 config.privileged_operator_prompt,
5987 config.modules,
5988 config.reserved_capabilities,
5989 );
5990
5991 let mut restart_required = Vec::new();
6000 for section in RestartRequiredSection::ALL {
6001 let changed = match section {
6002 RestartRequiredSection::Port => configured_port != context.configured_port,
6003 RestartRequiredSection::Storage => storage_config != context.storage_config,
6004 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6005 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6006 }
6007 RestartRequiredSection::AdmissionFactsTargets => {
6008 admission_facts_targets != context.admission_facts_targets
6009 }
6010 RestartRequiredSection::ScopeAuthorityOwners => {
6011 scope_authority_owners != context.scope_authority_owners
6012 }
6013 RestartRequiredSection::PrivilegedOperatorPrompt => {
6014 privileged_operator_prompt != context.privileged_operator_prompt
6015 }
6016 };
6017 if changed {
6018 restart_required.push(section.label().to_string());
6019 }
6020 }
6021 if !restart_required.is_empty() {
6022 warn!(
6023 config_path = %context.config_path.display(),
6024 sections = %restart_required.join(", "),
6025 "daemon config changed outside the modules section; restart the daemon to apply those changes"
6026 );
6027 }
6028
6029 for configured in &modules {
6030 if let Err(err) = validate_spec(&configured.module_spec()) {
6031 return Ok(vec![control_error_frame(
6032 &frame,
6033 "invalid_daemon_config",
6034 format!("supervisor rescan rejected daemon config: {err}"),
6035 )?]);
6036 }
6037 }
6038
6039 let configured_capabilities = modules
6040 .iter()
6041 .map(|module| (module.module_id.clone(), module.enabled))
6042 .collect::<Vec<_>>();
6043 let preview_capability_warnings = if preview {
6044 let (_, registrations) = self.runtime_capability_snapshot()?;
6045 let current_modules = self
6046 .supervisor
6047 .list()
6048 .into_iter()
6049 .map(|module| module.module_id().to_string())
6050 .collect::<BTreeSet<_>>();
6051 let resulting_modules = configured_capabilities.clone();
6052 let removed = current_modules
6053 .into_iter()
6054 .filter(|module_id| {
6055 !resulting_modules
6056 .iter()
6057 .any(|(configured_id, _)| configured_id == module_id)
6058 })
6059 .collect::<Vec<_>>();
6060 let disabled = match &mode {
6063 RescanMode::Candidate(_) => resulting_modules
6064 .iter()
6065 .filter(|(_, enabled)| !enabled)
6066 .map(|(module_id, _)| module_id.clone())
6067 .collect::<Vec<_>>(),
6068 _ => Vec::new(),
6069 };
6070 self.capability_evaluator.preview_warnings(
6071 resulting_modules,
6072 &removed,
6073 &disabled,
6074 ®istrations,
6075 )
6076 } else {
6077 Vec::new()
6078 };
6079 let reconciled = match &mode {
6080 RescanMode::Candidate(candidate) => self
6081 .preview_candidate(&context, modules, candidate)
6082 .map(|result| (result, Vec::new())),
6083 _ => {
6084 self.reconcile_supervised_modules(
6085 &context.supervisor,
6086 modules,
6087 preview,
6088 requester.clone(),
6089 )
6090 .await
6091 }
6092 };
6093 let (result, changed) = match reconciled {
6094 Ok(result) => result,
6095 Err(message) => {
6096 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6097 }
6098 };
6099 if !preview {
6100 if let Some(ledger) = &self.activity {
6101 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6102 ledger.config_changed(&result.added, &result.removed, &changed, requester);
6103 }
6104 if let Err(code) = self.configure_activity(activity_config).await {
6105 warn!(
6106 code,
6107 "activity retention could not be applied during rescan"
6108 );
6109 }
6110 self.capability_evaluator
6111 .configure(configured_capabilities, reserved_capabilities);
6112 self.capability_evaluator.wake_deadline_loop();
6113 self.refresh_capability_requirements();
6114 }
6115 let mut result = result;
6116 result.restart_required = restart_required;
6117 result.capability_warnings = preview_capability_warnings;
6118 let response = ClientControlResponse::SupervisorRescan { result };
6119 Ok(vec![control_response_body_frame(
6120 &frame,
6121 &response,
6122 "ClientControlResponse::SupervisorRescan",
6123 )?])
6124 }
6125
6126 async fn handle_supervisor_release_reserved(
6127 &self,
6128 frame: Frame,
6129 module_id: String,
6130 ) -> Result<Vec<Frame>, RouterError> {
6131 let Some(context) = self.rescan.clone() else {
6132 return Ok(vec![control_error_frame(
6133 &frame,
6134 "release_unavailable",
6135 "reserved-id release requires a daemon started with a reloadable config path",
6136 )?]);
6137 };
6138 let operation_lock = self.supervisor.operation_lock();
6139 let _operation_guard = operation_lock.lock().await;
6140 let loaded = match crate::daemon_config::load(&context.config_path) {
6141 Ok(Some(config)) => config,
6142 Ok(None) => {
6143 return Ok(vec![control_error_frame(
6144 &frame,
6145 "invalid_daemon_config",
6146 format!(
6147 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6148 context.config_path.display()
6149 ),
6150 )?])
6151 }
6152 Err(err) => {
6153 return Ok(vec![control_error_frame(
6154 &frame,
6155 "invalid_daemon_config",
6156 format!("unable to verify reserved-id release against daemon config: {err}"),
6157 )?])
6158 }
6159 };
6160 if loaded
6161 .modules
6162 .iter()
6163 .any(|configured| configured.module_id == module_id)
6164 {
6165 return Ok(vec![control_error_frame(
6166 &frame,
6167 "reserved_module_configured",
6168 format!(
6169 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6170 ),
6171 )?]);
6172 }
6173 if !self.supervisor.release_retained_reserved_gate(&module_id) {
6174 return Ok(vec![control_error_frame(
6175 &frame,
6176 "reserved_gate_not_retained",
6177 format!(
6178 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6179 ),
6180 )?]);
6181 }
6182
6183 let response = ClientControlResponse::SupervisorAck {
6184 module_id,
6185 applied: true,
6186 };
6187 Ok(vec![control_response_body_frame(
6188 &frame,
6189 &response,
6190 "ClientControlResponse::SupervisorAck",
6191 )?])
6192 }
6193
6194 fn plan_rescan(
6198 &self,
6199 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6200 ) -> Result<RescanPlan, String> {
6201 let mut current = BTreeMap::new();
6202 for module in self.supervisor.list() {
6203 let (spec, health) = module.configuration().map_err(|err| {
6204 format!(
6205 "failed to read configuration for module_id '{}': {err}",
6206 module.module_id()
6207 )
6208 })?;
6209 let enabled = module
6210 .status()
6211 .map_err(|err| {
6212 format!(
6213 "failed to read status for module_id '{}': {err}",
6214 module.module_id()
6215 )
6216 })?
6217 .enabled;
6218 current.insert(
6219 module.module_id().to_string(),
6220 (module, spec, health, enabled),
6221 );
6222 }
6223 let configured = configured_modules
6224 .into_iter()
6225 .map(|module| (module.module_id.clone(), module))
6226 .collect::<BTreeMap<_, _>>();
6227
6228 let added = configured
6229 .keys()
6230 .filter(|module_id| !current.contains_key(*module_id))
6231 .cloned()
6232 .collect::<Vec<_>>();
6233 let removed = current
6234 .keys()
6235 .filter(|module_id| !configured.contains_key(*module_id))
6236 .cloned()
6237 .collect::<Vec<_>>();
6238 let mut changed_pending_reload = Vec::new();
6239 let mut configuration_changes = BTreeSet::new();
6240 let mut enabled_changes = BTreeSet::new();
6241 let mut unchanged = 0_u32;
6242
6243 for (module_id, configured_module) in &configured {
6244 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6245 else {
6246 continue;
6247 };
6248 let launch_changed = *current_spec != configured_module.module_spec();
6252 let configuration_changed =
6253 launch_changed || *current_health != configured_module.health;
6254 let enabled_changed = *current_enabled != configured_module.enabled;
6255 if configuration_changed {
6256 configuration_changes.insert(module_id.clone());
6257 }
6258 if launch_changed {
6259 changed_pending_reload.push(module_id.clone());
6260 }
6261 if enabled_changed {
6262 enabled_changes.insert(module_id.clone());
6263 }
6264 if !configuration_changed && !enabled_changed {
6265 unchanged = unchanged.saturating_add(1);
6266 }
6267 }
6268 Ok(RescanPlan {
6269 current,
6270 configured,
6271 added,
6272 removed,
6273 changed_pending_reload,
6274 configuration_changes,
6275 enabled_changes,
6276 unchanged,
6277 })
6278 }
6279
6280 async fn reconcile_supervised_modules(
6289 &self,
6290 supervisor: &Supervisor,
6291 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6292 preview: bool,
6293 requester: Principal,
6294 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6295 let plan = self.plan_rescan(configured_modules)?;
6296
6297 if preview {
6303 return Ok((plan.preview_result(), Vec::new()));
6304 }
6305 let changed = plan
6306 .configuration_changes
6307 .union(&plan.enabled_changes)
6308 .cloned()
6309 .collect::<Vec<_>>();
6310 let RescanPlan {
6311 current,
6312 configured,
6313 added,
6314 removed,
6315 changed_pending_reload,
6316 configuration_changes,
6317 enabled_changes,
6318 unchanged,
6319 } = plan;
6320 if !(added.is_empty()
6325 && removed.is_empty()
6326 && configuration_changes.is_empty()
6327 && enabled_changes.is_empty())
6328 {
6329 if let Some(context) = &self.rescan {
6330 context
6331 .generation
6332 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6333 }
6334 }
6335
6336 for module_id in &removed {
6337 let module = ¤t
6338 .get(module_id)
6339 .expect("removed module came from current supervisor state")
6340 .0;
6341 module
6342 .retire_requested(Some(requester.clone()))
6343 .await
6344 .map_err(|err| {
6345 format!("failed to retire module_id '{module_id}' during rescan: {err}")
6346 })?;
6347 self.supervisor.record_rescan_removal(module_id);
6381 self.supervisor.retire(module_id);
6382 self.route_outages.forget(module_id);
6383 }
6384
6385 for module_id in configured.keys() {
6386 let Some((module, _, _, _)) = current.get(module_id) else {
6387 continue;
6388 };
6389 let configured_module = configured
6390 .get(module_id)
6391 .expect("configured module id came from configured map");
6392 if configuration_changes.contains(module_id) {
6393 module
6394 .update_configuration(
6395 configured_module.module_spec(),
6396 configured_module.health.clone(),
6397 configured_module.drain_timeout_ms,
6398 )
6399 .await
6400 .map_err(|err| {
6401 format!(
6402 "failed to update module_id '{module_id}' configuration during rescan: {err}"
6403 )
6404 })?;
6405 }
6406 if enabled_changes.contains(module_id) {
6407 self.route_outages.mark_operator_action(module_id);
6410 module
6411 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6412 .await
6413 .map_err(|err| {
6414 self.route_outages.operator_action_ended_unrefused(module_id);
6415 format!(
6416 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6417 configured_module.enabled
6418 )
6419 })?;
6420 }
6421 }
6422
6423 for module_id in &added {
6424 let configured_module = configured
6425 .get(module_id)
6426 .expect("added module id came from configured map");
6427 supervisor
6428 .supervise_configured_requested(
6429 configured_module.module_spec(),
6430 configured_module.enabled,
6431 configured_module.health.clone(),
6432 configured_module.drain_timeout_ms,
6433 configured_module.restart,
6434 configured_module.register_timeout,
6435 Some(requester.clone()),
6436 )
6437 .map_err(|err| {
6438 format!("failed to add module_id '{module_id}' during rescan: {err}")
6439 })?;
6440 }
6441
6442 Ok((
6443 SupervisorRescanResult {
6444 added,
6445 removed,
6446 changed_pending_reload,
6447 enabled_changes: enabled_changes.iter().cloned().collect(),
6448 unchanged,
6449 preview: false,
6450 restart_required: Vec::new(),
6453 capability_warnings: Vec::new(),
6454 module_actions: Vec::new(),
6455 earlier_pending: Vec::new(),
6456 expect: None,
6457 },
6458 changed,
6459 ))
6460 }
6461
6462 async fn handle_supervisor_set_enabled(
6463 &self,
6464 frame: Frame,
6465 module_id: String,
6466 enabled: bool,
6467 requester: Principal,
6468 ) -> Result<Vec<Frame>, RouterError> {
6469 let operation_lock = self.supervisor.operation_lock();
6470 let _operation_guard = operation_lock.lock().await;
6471 let Some(module) = self.supervisor.get(&module_id) else {
6472 return Ok(vec![control_error_frame(
6473 &frame,
6474 "unknown_module",
6475 format!("module_id '{module_id}' is not supervised"),
6476 )?]);
6477 };
6478
6479 self.route_outages.mark_operator_action(&module_id);
6482 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6483 Ok(applied) => applied,
6484 Err(err) => {
6485 self.route_outages
6486 .operator_action_ended_unrefused(&module_id);
6487 return Ok(vec![control_error_frame(
6488 &frame,
6489 "target_unavailable",
6490 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6491 )?]);
6492 }
6493 };
6494 if !applied {
6495 self.route_outages
6498 .operator_action_ended_unrefused(&module_id);
6499 }
6500
6501 self.capability_evaluator.wake_deadline_loop();
6502 self.refresh_capability_requirements();
6503 let response = ClientControlResponse::SupervisorAck { module_id, applied };
6504 Ok(vec![control_response_body_frame(
6505 &frame,
6506 &response,
6507 "ClientControlResponse::SupervisorAck",
6508 )?])
6509 }
6510
6511 async fn handle_supervisor_health_probe(
6512 &self,
6513 frame: Frame,
6514 module_id: String,
6515 ) -> Result<Vec<Frame>, RouterError> {
6516 self.refresh_capability_requirements();
6517 let Some(registration) = self
6518 .registry
6519 .get_module(&module_id)
6520 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6521 else {
6522 return Ok(vec![control_error_frame(
6523 &frame,
6524 "unknown_module",
6525 format!("module_id '{module_id}' is not registered"),
6526 )?]);
6527 };
6528
6529 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6542 {
6543 return Ok(vec![control_error_frame(
6544 &frame,
6545 "health_not_advertised",
6546 format!("module_id '{module_id}' did not advertise health.check"),
6547 )?]);
6548 }
6549
6550 let deadline = Instant::now() + self.health_probe_timeout;
6551 let pending = match self.forwarding.begin_module_control_rpc_for(
6552 &module_id,
6553 MODULE_CONTROL_OP_HEALTH_CHECK,
6554 deadline,
6555 ) {
6556 Ok(pending) => pending,
6557 Err(err) => {
6558 return Ok(vec![control_error_frame(
6559 &frame,
6560 forwarding_error_code(&err),
6561 err.to_string(),
6562 )?])
6563 }
6564 };
6565
6566 let PendingModuleControlRpc {
6567 endpoint,
6568 module_sink,
6569 negotiated_ver,
6570 corr: probe_corr,
6571 receiver,
6572 } = pending;
6573 let mut guard =
6574 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6575 let probe_body =
6576 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6577 RouterError::backend(
6578 0,
6579 frame.header.corr,
6580 format!("failed to encode health.check request: {err}"),
6581 )
6582 })?;
6583 let probe_frame = Frame::build_with_version(
6584 negotiated_ver,
6585 FrameType::Request,
6586 control_flags(),
6587 0,
6588 0,
6589 probe_corr,
6590 probe_body,
6591 )
6592 .map_err(RouterError::FrameBuild)?;
6593
6594 match timeout_at(deadline, module_sink.send(probe_frame)).await {
6597 Ok(Ok(())) => {}
6598 Ok(Err(err)) => {
6599 return Ok(vec![control_error_frame(
6600 &frame,
6601 "target_unavailable",
6602 err.to_string(),
6603 )?]);
6604 }
6605 Err(_) => {
6606 return Ok(vec![control_error_frame(
6607 &frame,
6608 "module_timeout",
6609 format!(
6610 "module_id '{module_id}' did not answer health.check within {:?}",
6611 self.health_probe_timeout
6612 ),
6613 )?]);
6614 }
6615 }
6616
6617 match timeout_at(deadline, receiver).await {
6618 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6619 guard.disarm();
6620 let Some(report) = response.health_report() else {
6621 return Ok(vec![control_error_frame(
6622 &frame,
6623 "invalid_control_body",
6624 "health.check RPC returned a non-health response",
6625 )?]);
6626 };
6627 let HealthReport {
6632 status,
6633 detail,
6634 metrics,
6635 } = report;
6636 let capability_detail = self
6637 .capability_evaluator
6638 .required_problem_detail(&module_id);
6639 let response = ClientControlResponse::SupervisorHealthProbe {
6640 module_id,
6641 status,
6642 detail: append_capability_problem_detail(detail, capability_detail),
6643 metrics,
6644 };
6645 Ok(vec![control_response_body_frame(
6646 &frame,
6647 &response,
6648 "ClientControlResponse::SupervisorHealthProbe",
6649 )?])
6650 }
6651 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6652 guard.disarm();
6653 Ok(vec![control_error_body_frame(&frame, body)?])
6654 }
6655 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6656 guard.disarm();
6657 Ok(vec![control_error_frame(
6658 &frame,
6659 "target_unavailable",
6660 message,
6661 )?])
6662 }
6663 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6664 guard.disarm();
6665 Ok(vec![control_error_frame(
6666 &frame,
6667 "invalid_control_body",
6668 message,
6669 )?])
6670 }
6671 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6672 guard.disarm();
6673 Ok(vec![control_error_frame(
6674 &frame,
6675 "invalid_control_body",
6676 format!("expected module-control op '{expected}', got '{actual}'"),
6677 )?])
6678 }
6679 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6680 guard.disarm();
6681 Ok(vec![control_error_frame(
6682 &frame,
6683 "module_timeout",
6684 format!(
6685 "module_id '{module_id}' answered health.check after {:?}",
6686 self.health_probe_timeout
6687 ),
6688 )?])
6689 }
6690 Ok(Err(_)) => Ok(vec![control_error_frame(
6691 &frame,
6692 "target_unavailable",
6693 "health.check waiter was canceled before the module responded",
6694 )?]),
6695 Err(_) => Ok(vec![control_error_frame(
6696 &frame,
6697 "module_timeout",
6698 format!(
6699 "module_id '{module_id}' did not answer health.check within {:?}",
6700 self.health_probe_timeout
6701 ),
6702 )?]),
6703 }
6704 }
6705
6706 fn supervisor_status(
6707 &self,
6708 module_id: &str,
6709 corr: u64,
6710 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6711 self.supervisor
6712 .get(module_id)
6713 .map(|module| {
6714 let warming = module.is_warming_for_control("status").map_err(|err| {
6715 RouterError::backend(
6716 0,
6717 corr,
6718 format!(
6719 "failed to read supervisor warming state for module_id '{module_id}': {err}"
6720 ),
6721 )
6722 })?;
6723 module.status_for_control("status").map_err(|err| {
6724 RouterError::backend(
6725 0,
6726 corr,
6727 format!(
6728 "failed to read supervisor status for module_id '{module_id}': {err}"
6729 ),
6730 )
6731 }).map(|status| (status, warming))
6732 })
6733 .transpose()
6734 }
6735
6736 fn guard_module_control_op(
6737 &self,
6738 frame: &Frame,
6739 module_id: &str,
6740 op: &str,
6741 ) -> Result<Option<Frame>, RouterError> {
6742 if self.module_grants_op(module_id, op, frame.header.corr)? {
6743 return Ok(None);
6744 }
6745
6746 Ok(Some(control_error_frame(
6747 frame,
6748 "op_not_allowed",
6749 format!("module_id '{module_id}' did not grant control op '{op}'"),
6750 )?))
6751 }
6752
6753 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6754 let Some(registration) = self
6755 .registry
6756 .get_module(module_id)
6757 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6758 else {
6759 return Ok(false);
6760 };
6761 Ok(module_registration_grants_op(®istration.control_ops, op))
6762 }
6763
6764 fn handle_status_update(
6765 &self,
6766 endpoint: ModuleEndpointId,
6767 frame: Frame,
6768 ) -> Result<Vec<Frame>, RouterError> {
6769 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6770 Ok(update) => update,
6771 Err(err) => {
6772 if is_known_module_push_op(&frame.body) {
6777 return Ok(vec![control_error_frame(
6778 &frame,
6779 "invalid_control_body",
6780 format!("malformed module control push body: {err}"),
6781 )?]);
6782 }
6783 return Ok(Vec::new());
6784 }
6785 };
6786
6787 match update {
6788 ModuleControlPush::RouteStatus {
6789 route_channel,
6790 route_epoch,
6791 status,
6792 } => {
6793 self.forwarding
6794 .cache_status(endpoint, route_channel, route_epoch, status)
6795 .map_err(RouterError::Forwarding)?;
6796 }
6797 }
6798 Ok(Vec::new())
6799 }
6800
6801 fn handle_route_poll(
6802 &self,
6803 ctx: &RouteCtx,
6804 frame: Frame,
6805 route_channel: u16,
6806 route_epoch: u32,
6807 kind: PollKind,
6808 ) -> Result<Vec<Frame>, RouterError> {
6809 let snapshot = self
6810 .forwarding
6811 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6812 .map_err(RouterError::Forwarding)?;
6813 let response = match (kind, snapshot) {
6814 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6815 ClientControlResponse::RoutePoll {
6816 route_channel,
6817 route_epoch,
6818 status,
6819 live: None,
6820 }
6821 }
6822 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6823 route_channel,
6824 route_epoch,
6825 status: None,
6826 live: None,
6827 },
6828 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6829 let live = self
6855 .process_liveness
6856 .as_ref()
6857 .and_then(|source| source.process_live(&module_id))
6858 .unwrap_or(true);
6859 ClientControlResponse::RoutePoll {
6860 route_channel,
6861 route_epoch,
6862 status: None,
6863 live: Some(live),
6864 }
6865 }
6866 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6867 route_channel,
6868 route_epoch,
6869 status: None,
6870 live: Some(false),
6871 },
6872 };
6873
6874 Ok(vec![control_response_body_frame(
6875 &frame,
6876 &response,
6877 "ClientControlResponse::RoutePoll",
6878 )?])
6879 }
6880
6881 pub(crate) fn observe_module_control_completion(
6882 &self,
6883 completion: ModuleControlRpcCompletion,
6884 ) -> bool {
6885 match completion {
6886 ModuleControlRpcCompletion::Unknown => false,
6887 ModuleControlRpcCompletion::Settled => true,
6888 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6889 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6890 info!(
6891 module_id = %module_id,
6892 latency_ms,
6893 "late health.check answer proves the module is alive"
6894 );
6895 match self
6896 .supervisor
6897 .record_late_health_answer(&module_id, latency_ms)
6898 {
6899 Ok(true) => {}
6900 Ok(false) => debug!(
6901 module_id = %module_id,
6902 latency_ms,
6903 "late health.check answer has no active supervisor snapshot"
6904 ),
6905 Err(err) => warn!(
6906 module_id = %module_id,
6907 latency_ms,
6908 error = %err,
6909 "failed to record late health.check answer"
6910 ),
6911 }
6912 true
6913 }
6914 }
6915 }
6916
6917 fn refuse_to_end_module_connection_for_a_client(
6938 &self,
6939 module_connection_id: ConnectionId,
6940 corr: u64,
6941 err: ForwardingError,
6942 ) -> Result<(), RouterError> {
6943 if let ForwardingError::ConnectionClosing { connection_id } = err {
6944 if connection_id != module_connection_id {
6945 warn!(
6946 module_connection_id = module_connection_id.get(),
6947 client_connection_id = connection_id.get(),
6948 corr,
6949 "dropping a route.bind response for a closing client; the module connection keeps serving"
6950 );
6951 return Ok(());
6952 }
6953 }
6954 Err(RouterError::Forwarding(err))
6955 }
6956
6957 fn handle_module_relay_response(
6958 &self,
6959 connection_id: ConnectionId,
6960 frame: Frame,
6961 ) -> Result<Vec<Frame>, RouterError> {
6962 let mut secondary_error = None;
6963 let outcome = match frame.header.ty {
6964 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6965 Ok(probe) if probe.op == "route.bind" => {
6966 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6967 Ok(ModuleControlResponse::RouteBindAck {}) => {
6968 RouteBindRelayOutcome::Accepted
6969 }
6970 Ok(other) => {
6971 let message =
6972 format!("route.bind response carried unexpected body: {other:?}");
6973 secondary_error = Some(control_error_frame(
6974 &frame,
6975 "invalid_control_body",
6976 message.clone(),
6977 )?);
6978 RouteBindRelayOutcome::ModuleGone(message)
6979 }
6980 Err(err) => {
6981 let message = format!("malformed route.bind response body: {err}");
6982 secondary_error = Some(control_error_frame(
6983 &frame,
6984 "invalid_control_body",
6985 message.clone(),
6986 )?);
6987 RouteBindRelayOutcome::ModuleGone(message)
6988 }
6989 }
6990 }
6991 Ok(probe) => {
6992 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6993 {
6994 Ok(response) => ModuleControlRpcOutcome::Response(response),
6995 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6996 "malformed {} response body: {err}",
6997 probe.op
6998 )),
6999 };
7000 let completion = self
7001 .forwarding
7002 .complete_module_control_rpc(
7003 connection_id,
7004 frame.header.corr,
7005 Some(&probe.op),
7006 outcome,
7007 )
7008 .map_err(RouterError::Forwarding)?;
7009 if !self.observe_module_control_completion(completion) {
7010 debug!(
7011 connection_id = connection_id.get(),
7012 corr = frame.header.corr,
7013 op = %probe.op,
7014 "dropping late or unknown module-control RPC response"
7015 );
7016 }
7017 return Ok(Vec::new());
7018 }
7019 Err(err) => {
7020 if let Some(expected_op) = self
7021 .forwarding
7022 .pending_module_control_op(connection_id, frame.header.corr)
7023 .map_err(RouterError::Forwarding)?
7024 {
7025 let completion = self
7026 .forwarding
7027 .complete_module_control_rpc(
7028 connection_id,
7029 frame.header.corr,
7030 None,
7031 ModuleControlRpcOutcome::MalformedResponse(format!(
7032 "malformed {expected_op} response body: {err}"
7033 )),
7034 )
7035 .map_err(RouterError::Forwarding)?;
7036 if !self.observe_module_control_completion(completion) {
7037 debug!(
7038 connection_id = connection_id.get(),
7039 corr = frame.header.corr,
7040 "dropping late malformed module-control RPC response"
7041 );
7042 }
7043 return Ok(Vec::new());
7044 }
7045 let message = format!("malformed route.bind response body: {err}");
7046 secondary_error = Some(control_error_frame(
7047 &frame,
7048 "invalid_control_body",
7049 message.clone(),
7050 )?);
7051 RouteBindRelayOutcome::ModuleGone(message)
7052 }
7053 },
7054 FrameType::Error => {
7055 if self
7056 .forwarding
7057 .pending_module_control_op(connection_id, frame.header.corr)
7058 .map_err(RouterError::Forwarding)?
7059 .is_some()
7060 {
7061 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7062 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7063 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7064 "malformed module-control ERROR body: {err}"
7065 )),
7066 };
7067 let completion = self
7068 .forwarding
7069 .complete_module_control_rpc(
7070 connection_id,
7071 frame.header.corr,
7072 None,
7073 outcome,
7074 )
7075 .map_err(RouterError::Forwarding)?;
7076 if !self.observe_module_control_completion(completion) {
7077 debug!(
7078 connection_id = connection_id.get(),
7079 corr = frame.header.corr,
7080 "dropping late or unknown module-control RPC error"
7081 );
7082 }
7083 return Ok(Vec::new());
7084 }
7085 match serde_json::from_slice::<ErrorBody>(&frame.body) {
7086 Ok(body) => RouteBindRelayOutcome::Rejected(body),
7087 Err(err) => {
7088 let message = format!("malformed route.bind ERROR body: {err}");
7089 secondary_error = Some(control_error_frame(
7090 &frame,
7091 "invalid_control_body",
7092 message.clone(),
7093 )?);
7094 RouteBindRelayOutcome::ModuleGone(message)
7095 }
7096 }
7097 }
7098 ty => {
7099 return Ok(vec![control_error_frame(
7100 &frame,
7101 "unsupported_control_frame",
7102 format!("unsupported module channel-0 frame {ty:?}"),
7103 )?])
7104 }
7105 };
7106
7107 let settled =
7108 self.forwarding
7109 .complete_pending_relay(connection_id, frame.header.corr, outcome);
7110 let completion = match settled {
7111 Ok(completion) => completion,
7112 Err(err) => {
7113 self.refuse_to_end_module_connection_for_a_client(
7114 connection_id,
7115 frame.header.corr,
7116 err,
7117 )?;
7118 return Ok(secondary_error.into_iter().collect());
7119 }
7120 };
7121 if let Some(target) = completion.abandoned.as_ref() {
7122 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7123 }
7124 if !completion.settled {
7125 debug!(
7126 connection_id = connection_id.get(),
7127 corr = frame.header.corr,
7128 frame_type = ?frame.header.ty,
7129 "dropping late or unknown route.bind relay response"
7130 );
7131 }
7132 Ok(secondary_error.into_iter().collect())
7133 }
7134
7135 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7136 debug!(connection_id = connection_id.get(), "handling GOODBYE");
7137 self.cleanup_connection_with_end_reason(
7141 connection_id,
7142 RegistrationEndReason::ExplicitGoodbye,
7143 )
7144 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7145 Ok(Vec::new())
7146 }
7147}
7148
7149impl Default for ControlHandler {
7150 fn default() -> Self {
7151 Self::new(Arc::new(Registry::default()))
7152 }
7153}
7154
7155impl crate::supervise::SwapPromotionObserver for ControlHandler {
7156 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7157 self.apply_registration_capabilities(registration);
7158 }
7159}
7160
7161fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7162 CapabilityRequirementStatus {
7163 consumer: status.consumer,
7164 capability: status.capability,
7165 need: match status.need {
7166 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7167 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7168 },
7169 verdict: status.verdict.as_str().to_string(),
7170 episode_seq: status.episode_seq,
7171 config_satisfiable: status.config_satisfiable,
7172 runtime_available: status.runtime_available,
7173 detail: status.detail,
7174 }
7175}
7176
7177fn append_capability_problem_detail(
7178 detail: Option<String>,
7179 capability_detail: Option<String>,
7180) -> Option<String> {
7181 match (detail, capability_detail) {
7182 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7183 (Some(detail), None) => Some(detail),
7184 (None, Some(capability_detail)) => Some(capability_detail),
7185 (None, None) => None,
7186 }
7187}
7188
7189fn subc_ops() -> Vec<String> {
7190 SUBC_CONTROL_OPS
7191 .iter()
7192 .map(|op| (*op).to_string())
7193 .collect()
7194}
7195
7196fn module_subc_ops() -> Vec<String> {
7197 SUBC_CONTROL_OPS
7198 .iter()
7199 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7200 .map(|op| (*op).to_string())
7201 .collect()
7202}
7203
7204#[cfg(test)]
7205fn module_baseline_control_ops() -> Vec<String> {
7206 MODULE_BASELINE_CONTROL_OPS
7207 .iter()
7208 .map(|op| (*op).to_string())
7209 .collect()
7210}
7211
7212fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7213 let mut seen = HashSet::new();
7214 let mut effective = Vec::new();
7215 for op in MODULE_BASELINE_CONTROL_OPS {
7216 if seen.insert((*op).to_string()) {
7217 effective.push((*op).to_string());
7218 }
7219 }
7220 for op in declared.unwrap_or_default() {
7221 if seen.insert(op.clone()) {
7222 effective.push(op);
7223 }
7224 }
7225 effective
7226}
7227
7228fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7229 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7230}
7231
7232fn target_module_id(target: &RouteTarget) -> &str {
7233 match target {
7234 RouteTarget::ToolProvider { module_id }
7235 | RouteTarget::ManagementSurface { module_id }
7236 | RouteTarget::InternalService { module_id, .. } => module_id,
7237 }
7238}
7239
7240fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7241 roles.iter().any(|role| match (target, role) {
7242 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7243 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7244 (
7245 RouteTarget::InternalService { service_id, .. },
7246 ProviderRole::InternalService {
7247 service_id: provided,
7248 ..
7249 },
7250 ) => service_id == provided,
7251 _ => false,
7252 })
7253}
7254
7255fn is_routable_role(role: &ProviderRole) -> bool {
7256 matches!(
7257 role,
7258 ProviderRole::ToolProvider { .. }
7259 | ProviderRole::ManagementSurface { .. }
7260 | ProviderRole::InternalService { .. }
7261 )
7262}
7263
7264#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7265enum ControlRequestBodyError {
7266 UnknownOp,
7267 InvalidBody,
7268}
7269
7270#[derive(Debug, Deserialize)]
7271struct ControlOpProbe {
7272 op: String,
7273}
7274
7275const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7278
7279fn is_known_module_push_op(body: &[u8]) -> bool {
7280 serde_json::from_slice::<ControlOpProbe>(body)
7281 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7282 .unwrap_or(false)
7283}
7284
7285fn is_known_module_request_op(body: &[u8]) -> bool {
7286 serde_json::from_slice::<ControlOpProbe>(body)
7287 .map(|probe| is_module_to_subc_op(&probe.op))
7288 .unwrap_or(false)
7289}
7290
7291fn is_module_to_subc_op(op: &str) -> bool {
7292 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7293}
7294
7295fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7296 debug!(
7297 op = %op,
7298 connection_id = connection_id.get(),
7299 corr,
7300 "control dispatch"
7301 );
7302}
7303
7304fn log_slow_control_dispatch(
7305 dispatch_started_at: Option<StdInstant>,
7306 op: &'static str,
7307 connection_id: ConnectionId,
7308 corr: u64,
7309 route_open_timings: Option<&RouteOpenTimings>,
7310) {
7311 let Some(dispatch_started_at) = dispatch_started_at else {
7312 return;
7313 };
7314 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7316 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7317 if let Some(timings) = route_open_timings {
7318 warn!(
7319 op = %op,
7320 connection_id = connection_id.get(),
7321 corr,
7322 elapsed_ms = elapsed.as_millis() as u64,
7323 module_id = ?timings.module_id,
7324 principal = %timings.principal,
7325 admission_ms = timings.admission.as_millis() as u64,
7326 project_root_ms = timings.project_root.as_millis() as u64,
7327 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7328 bind_send_ms = timings.bind_send.as_millis() as u64,
7329 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7330 "slow control dispatch"
7331 );
7332 return;
7333 }
7334 warn!(
7335 op = %op,
7336 connection_id = connection_id.get(),
7337 corr,
7338 elapsed_ms = elapsed.as_millis() as u64,
7339 "slow control dispatch"
7340 );
7341 }
7342}
7343
7344#[cfg(test)]
7346type TestCallerLookup =
7347 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7348
7349fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7351 match request {
7352 ClientControlRequest::SupervisorRestart { module_id, .. } => {
7353 Some(PrivilegedOperation::Restart { module_id })
7354 }
7355 ClientControlRequest::SupervisorSwap { module_id, .. } => {
7356 Some(PrivilegedOperation::Swap { module_id })
7357 }
7358 ClientControlRequest::SupervisorReload { module_id } => {
7359 Some(PrivilegedOperation::Reload { module_id })
7360 }
7361 ClientControlRequest::SupervisorRescan { preview } => {
7362 if *preview {
7363 None
7364 } else {
7365 Some(PrivilegedOperation::Rescan)
7366 }
7367 }
7368 ClientControlRequest::SupervisorRescanChecked {
7373 preview,
7374 candidate,
7375 expect,
7376 } => (!*preview && candidate.is_none() && expect.is_some())
7377 .then_some(PrivilegedOperation::Rescan),
7378 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7379 Some(PrivilegedOperation::SetEnabled {
7380 module_id,
7381 enabled: *enabled,
7382 })
7383 }
7384 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7385 Some(PrivilegedOperation::ReleaseReserved { module_id })
7386 }
7387 ClientControlRequest::ServerDescribe {}
7388 | ClientControlRequest::CatalogList { .. }
7389 | ClientControlRequest::RouteOpen { .. }
7390 | ClientControlRequest::RoutePoll { .. }
7391 | ClientControlRequest::SupervisorList {}
7392 | ClientControlRequest::SupervisorHealth {}
7393 | ClientControlRequest::SupervisorHealthProbe { .. }
7394 | ClientControlRequest::SupervisorRoutes { .. }
7395 | ClientControlRequest::SupervisorProvenance { .. }
7396 | ClientControlRequest::SupervisorStderrTail { .. }
7397 | ClientControlRequest::SupervisorTerminals { .. }
7398 | ClientControlRequest::SupervisorSpawnSnapshot {}
7399 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7400 }
7401}
7402
7403fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7404 match request {
7405 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7406 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7407 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7408 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7409 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7410 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7411 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7412 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7413 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7414 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7415 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7416 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7417 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7418 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7419 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7420 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7421 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7422 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7423 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7424 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7425 }
7426}
7427
7428struct RouteOpenCaller {
7436 principal: String,
7440 scope: String,
7443 harness: String,
7446}
7447
7448impl RouteOpenCaller {
7449 fn new(
7450 principal: Option<&Principal>,
7451 scope: Option<&ScopeSelector>,
7452 identity: &BindIdentity,
7453 ) -> Self {
7454 Self {
7455 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
7456 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
7457 harness: bounded_log_text(&identity.harness),
7458 }
7459 }
7460
7461 fn unknown() -> Self {
7464 Self {
7465 principal: "unknown".to_string(),
7466 scope: "unknown".to_string(),
7467 harness: "unknown".to_string(),
7468 }
7469 }
7470}
7471
7472const MAX_LOGGED_TEXT_CHARS: usize = 64;
7475
7476fn bounded_log_text(value: &str) -> String {
7480 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
7481 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
7482 bounded.push('…');
7483 }
7484 format!("{bounded:?}")
7485}
7486
7487fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7493 use subc_protocol::scope::ScopeKind;
7494 let kind = match stamp.kind {
7495 ScopeKind::Head => "head",
7496 ScopeKind::Worker => "worker",
7497 ScopeKind::Ephemeral => "ephemeral",
7498 };
7499 format!(
7500 "{kind}/{}/{}@{}",
7501 principal_label(&stamp.owner),
7502 bounded_log_text(&stamp.scope_ref),
7503 stamp.scope_epoch
7504 )
7505}
7506
7507fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
7514 let epoch = selector
7515 .scope_epoch
7516 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
7517 format!(
7518 "requested/{}/{}@{epoch}",
7519 bounded_log_text(&principal_label(&selector.owner)),
7520 bounded_log_text(&selector.scope_ref),
7521 )
7522}
7523
7524fn principal_label(principal: &Principal) -> String {
7525 match principal {
7526 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7527 Principal::Direct => "direct".to_string(),
7528 other => format!("{other:?}"),
7529 }
7530}
7531
7532fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7533 match request {
7534 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7535 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7536 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7537 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7538 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7539 }
7540}
7541
7542fn parse_client_control_request(
7543 body: &[u8],
7544) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7545 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7546 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7547 Ok(probe)
7548 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7549 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7550 {
7551 ControlRequestBodyError::InvalidBody
7552 }
7553 Ok(_) => ControlRequestBodyError::UnknownOp,
7554 Err(_) => ControlRequestBodyError::InvalidBody,
7555 };
7556 (err, classification)
7557 })
7558}
7559
7560fn parse_module_control_request_from_module(
7561 body: &[u8],
7562) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7563 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7564 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7565 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7566 Ok(_) => ControlRequestBodyError::UnknownOp,
7567 Err(_) => ControlRequestBodyError::InvalidBody,
7568 };
7569 (err, classification)
7570 })
7571}
7572
7573#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7574enum ProviderRoleKind {
7575 ToolProvider,
7576 PipelineStage,
7577 ManagementSurface,
7578 InternalService,
7579}
7580
7581fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7582 match role {
7583 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7584 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7585 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7586 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7587 }
7588}
7589
7590fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7591 roles.iter().map(provider_role_kind).collect()
7592}
7593
7594const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7597
7598#[derive(Debug, Default, PartialEq, Eq)]
7600struct ScopeOutcomeCounts {
7601 created: usize,
7602 replaced: usize,
7603 updated: usize,
7604 unchanged: usize,
7605 refused: usize,
7606}
7607
7608impl ScopeOutcomeCounts {
7609 fn of(results: &[ScopeRecordResult]) -> Self {
7610 let mut counts = Self::default();
7611 for result in results {
7612 let slot = match result.outcome {
7613 ScopeRecordOutcome::Created => &mut counts.created,
7614 ScopeRecordOutcome::Replaced => &mut counts.replaced,
7615 ScopeRecordOutcome::Updated => &mut counts.updated,
7616 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7617 ScopeRecordOutcome::Refused => &mut counts.refused,
7618 };
7619 *slot += 1;
7620 }
7621 counts
7622 }
7623}
7624
7625#[cfg(test)]
7626mod scope_outcome_count_tests {
7627 use super::*;
7628
7629 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7630 ScopeRecordResult {
7631 scope_ref: "r".to_string(),
7632 scope_epoch: 1,
7633 outcome,
7634 code: None,
7635 message: None,
7636 version: None,
7637 parent_state: None,
7638 }
7639 }
7640
7641 #[test]
7644 fn every_outcome_is_counted_in_its_own_field() {
7645 let results = [
7646 result(ScopeRecordOutcome::Created),
7647 result(ScopeRecordOutcome::Created),
7648 result(ScopeRecordOutcome::Replaced),
7649 result(ScopeRecordOutcome::Updated),
7650 result(ScopeRecordOutcome::Unchanged),
7651 result(ScopeRecordOutcome::Refused),
7652 result(ScopeRecordOutcome::Refused),
7653 result(ScopeRecordOutcome::Refused),
7654 ];
7655 assert_eq!(
7656 ScopeOutcomeCounts::of(&results),
7657 ScopeOutcomeCounts {
7658 created: 2,
7659 replaced: 1,
7660 updated: 1,
7661 unchanged: 1,
7662 refused: 3,
7663 }
7664 );
7665 }
7666}
7667
7668fn capability_census_trigger(
7672 old: Option<&CapabilityDeclarations>,
7673 new: Option<&CapabilityDeclarations>,
7674) -> bool {
7675 let old_provides = old
7676 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7677 .unwrap_or_default();
7678 let old_denies = old
7679 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7680 .unwrap_or_default();
7681 let new = new.cloned().unwrap_or(CapabilityDeclarations {
7682 provides: Vec::new(),
7683 requires: Vec::new(),
7684 must_never_reach: Vec::new(),
7685 });
7686
7687 new.provides
7688 .iter()
7689 .any(|capability| !old_provides.contains(capability))
7690 || new
7691 .must_never_reach
7692 .iter()
7693 .any(|capability| !old_denies.contains(capability))
7694}
7695
7696fn denied_capability<'a>(
7699 opening_manifest: &'a ModuleManifest,
7700 target_manifest: &ModuleManifest,
7701) -> Option<&'a str> {
7702 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7703 let target_capabilities = target_manifest.capabilities.as_ref()?;
7704 opening_capabilities
7705 .must_never_reach
7706 .iter()
7707 .find(|denied| {
7708 target_capabilities
7709 .provides
7710 .iter()
7711 .any(|provided| provided == *denied)
7712 })
7713 .map(String::as_str)
7714}
7715
7716fn catalog_update_frozen_field_message(
7717 registered: &ModuleManifest,
7718 provides: &[ProviderRole],
7719) -> Option<String> {
7720 let old_has_provides = !registered.provides.is_empty();
7721 let new_has_provides = !provides.is_empty();
7722 if old_has_provides != new_has_provides {
7723 return Some(format!(
7724 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7725 registered.module_id
7726 ));
7727 }
7728
7729 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
7730 return Some(format!(
7731 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7732 registered.module_id
7733 ));
7734 }
7735
7736 let registered_concurrency = manifest_concurrency(registered);
7737 let mut candidate = registered.clone();
7738 candidate.provides = provides.to_vec();
7739 let candidate_concurrency = manifest_concurrency(&candidate);
7740 if candidate_concurrency != registered_concurrency {
7741 return Some(format!(
7742 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7743 registered.module_id, registered_concurrency, candidate_concurrency
7744 ));
7745 }
7746
7747 None
7750}
7751
7752fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7753 manifest.provides.iter().any(is_routable_role)
7754}
7755
7756fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7762 manifest
7763 .provides
7764 .iter()
7765 .find_map(|provider| match provider {
7766 ProviderRole::ToolProvider { concurrency, .. }
7767 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7768 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7769 })
7770 .unwrap_or(Concurrency::ModuleManaged)
7771}
7772
7773fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7780 let has_management_surface = manifest
7781 .provides
7782 .iter()
7783 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7784 if !has_management_surface {
7785 return false;
7786 }
7787 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7788 return false;
7789 };
7790 let Some(provides) = raw
7791 .get("manifest")
7792 .and_then(|manifest| manifest.get("provides"))
7793 .and_then(serde_json::Value::as_array)
7794 else {
7795 return false;
7796 };
7797 provides.iter().any(|role| {
7802 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7803 && role.get("concurrency").is_none()
7804 })
7805}
7806
7807fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7808 if peer_version != PROTOCOL_VERSION {
7809 return Err(format!(
7810 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7811 ));
7812 }
7813 Ok(PROTOCOL_VERSION)
7814}
7815
7816fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7817 Frame::build_with_version(
7818 response_version(frame),
7819 FrameType::Pong,
7820 frame.header.flags,
7821 0,
7822 0,
7823 frame.header.corr,
7824 Vec::new(),
7825 )
7826 .map_err(RouterError::FrameBuild)
7827}
7828
7829pub(crate) fn control_error_frame(
7830 frame: &Frame,
7831 code: &'static str,
7832 message: impl Into<String>,
7833) -> Result<Frame, RouterError> {
7834 control_error_body_frame(
7835 frame,
7836 ErrorBody {
7837 code: code.to_string(),
7838 message: message.into(),
7839 detail: None,
7840 },
7841 )
7842}
7843
7844fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7845 let body = serde_json::to_vec(&error).map_err(|err| {
7846 RouterError::backend(
7847 0,
7848 frame.header.corr,
7849 format!("failed to encode control ERROR: {err}"),
7850 )
7851 })?;
7852
7853 Frame::build_with_version(
7854 response_version(frame),
7855 FrameType::Error,
7856 control_flags(),
7857 0,
7858 0,
7859 frame.header.corr,
7860 body,
7861 )
7862 .map_err(RouterError::FrameBuild)
7863}
7864
7865fn control_response_body_frame<T: Serialize>(
7866 frame: &Frame,
7867 reply: &T,
7868 label: &'static str,
7869) -> Result<Frame, RouterError> {
7870 let body = serde_json::to_vec(reply).map_err(|err| {
7871 RouterError::backend(
7872 0,
7873 frame.header.corr,
7874 format!("failed to encode {label}: {err}"),
7875 )
7876 })?;
7877
7878 Frame::build_with_version(
7879 response_version(frame),
7880 FrameType::Response,
7881 control_flags(),
7882 0,
7883 0,
7884 frame.header.corr,
7885 body,
7886 )
7887 .map_err(RouterError::FrameBuild)
7888}
7889
7890fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7920 match err {
7921 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7922 ForwardingError::NoModuleConnection => "target_unavailable",
7923 ForwardingError::ModuleReloading { .. } => "module_reloading",
7924 ForwardingError::ClientRouteChannelExhausted { .. }
7925 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7926 ForwardingError::StaleModuleEndpoint
7927 | ForwardingError::UnknownReservation { .. }
7928 | ForwardingError::ConnectionClosing { .. }
7929 | ForwardingError::ClientEgressClosed { .. }
7930 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7931 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7934 ForwardingError::RelayCorrelationExhausted
7935 | ForwardingError::RouteOpenBuild(_)
7936 | ForwardingError::Poisoned => "forwarding_error",
7937 }
7938}
7939
7940fn response_version(frame: &Frame) -> u8 {
7941 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7942 frame.header.ver
7943 } else {
7944 PROTOCOL_VERSION
7945 }
7946}
7947
7948fn control_flags() -> Flags {
7949 Flags::new(false, Priority::Passive, false)
7950}
7951
7952fn send_goodbye_target_best_effort(
7957 counters: &DaemonCounters,
7958 target: &GoodbyeTarget,
7959 context: &'static str,
7960) {
7961 let Ok(frame) = Frame::build_with_version(
7962 target.negotiated_ver,
7963 FrameType::Goodbye,
7964 control_flags(),
7965 target.channel,
7966 target.epoch,
7967 0,
7968 Vec::new(),
7969 ) else {
7970 return;
7971 };
7972 crate::forwarding::send_module_route_goodbye(
7973 counters,
7974 &target.sink,
7975 frame,
7976 target.module_id.as_deref(),
7977 context,
7978 );
7979}
7980
7981pub(crate) fn send_route_control_pushes(
7982 forwarding: &ForwardingTable,
7983 routes: Vec<EndpointRoute>,
7984 push: ClientControlPush,
7985) {
7986 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7987 for route in routes {
7988 let target = route.goodbye_target;
7989 if let Some((existing, channels)) = targets
7990 .iter_mut()
7991 .find(|(existing, _)| existing.connection_id == target.connection_id)
7992 {
7993 debug_assert_eq!(
7994 existing.negotiated_ver, target.negotiated_ver,
7995 "one connection cannot negotiate multiple frame versions"
7996 );
7997 if !channels.contains(&target.channel) {
7998 channels.push(target.channel);
7999 }
8000 continue;
8001 }
8002 let channel = target.channel;
8003 targets.push((target, vec![channel]));
8004 }
8005 for (target, mut channels) in targets {
8006 channels.sort_unstable();
8007 let mut push = push.clone();
8008 match &mut push {
8009 ClientControlPush::RouteClosing {
8010 channels: covered, ..
8011 }
8012 | ClientControlPush::RouteClosed {
8013 channels: covered, ..
8014 } => *covered = channels,
8015 }
8016 let body = match serde_json::to_vec(&push) {
8017 Ok(body) => body,
8018 Err(err) => {
8019 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8020 continue;
8021 }
8022 };
8023 let frame = match Frame::build_with_version(
8024 target.negotiated_ver,
8025 FrameType::Push,
8026 control_flags(),
8027 0,
8028 0,
8029 0,
8030 body.clone(),
8031 ) {
8032 Ok(frame) => frame,
8033 Err(err) => {
8034 warn!(
8035 route_channel = target.channel,
8036 error = %err,
8037 "failed to build route lifecycle control PUSH frame"
8038 );
8039 continue;
8040 }
8041 };
8042 if let Err(err) = target.sink.try_send(frame) {
8043 if target.close_on_delivery_failure() {
8044 warn!(
8045 target_connection_id = target.connection_id.get(),
8046 route_channel = target.channel,
8047 error = %err,
8048 "route lifecycle control PUSH was not delivered to client; closing target connection"
8049 );
8050 let _ = forwarding.escalate_client_delivery_failure(
8051 target.connection_id,
8052 target.channel,
8053 target.epoch,
8054 CloseReason::new(
8055 "route_lifecycle_push_delivery_failed",
8056 format!(
8057 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8058 target.channel
8059 ),
8060 ),
8061 crate::forwarding::UndeliveredFrame {
8062 module_id: target.module_id.as_deref(),
8063 sink: &target.sink,
8064 },
8065 );
8066 }
8067 }
8068 }
8069}
8070
8071#[cfg(test)]
8072mod tests {
8073 mod activity_read {
8074 include!("activity_read_tests.rs");
8075 }
8076
8077 mod lifecycle_activity {
8078 include!("control_lifecycle_tests.rs");
8079 }
8080 #[cfg(unix)]
8081 #[tokio::test]
8082 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8083 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8084 let path = dir.join("subc.jsonc");
8085 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8086 "program":"/bin/sleep","args":["60"],"protocol":"none",
8087 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8088 }}}).to_string()).unwrap();
8089 let mut configured = crate::daemon_config::load(&path)
8090 .unwrap()
8091 .unwrap()
8092 .modules
8093 .pop()
8094 .unwrap();
8095 let registry = std::sync::Arc::new(crate::Registry::default());
8096 let handle = crate::SupervisorHandle::new();
8097 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8098 .with_handle(handle.clone());
8099 let module = supervisor
8100 .supervise_configured_with_health(
8101 configured.module_spec(),
8102 true,
8103 configured.health.clone(),
8104 None,
8105 configured.restart,
8106 )
8107 .unwrap();
8108 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8109 let before = module.status().unwrap().pid;
8110 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8111 configured.health.cadence = std::time::Duration::from_secs(3600);
8112 let health_only = handler
8113 .reconcile_supervised_modules(
8114 &supervisor,
8115 vec![configured.clone()],
8116 false,
8117 Principal::Direct,
8118 )
8119 .await
8120 .unwrap();
8121 assert!(
8122 health_only.0.changed_pending_reload.is_empty(),
8123 "health policy is already applied live"
8124 );
8125 assert_eq!(module.status().unwrap().pid, before);
8126 assert_eq!(
8127 module.configuration().unwrap().1.http,
8128 configured.health.http
8129 );
8130 configured.args = vec!["61".into()];
8131 let launch = handler
8132 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8133 .await
8134 .unwrap();
8135 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8136 assert_eq!(
8137 module.status().unwrap().pid,
8138 before,
8139 "a launch edit is stored until reload"
8140 );
8141 module.drain().await.unwrap();
8142 }
8143 use cortexkit_test_support::ScratchDir;
8144 use std::{
8145 collections::BTreeMap,
8146 fmt,
8147 path::PathBuf,
8148 sync::{Arc, Mutex},
8149 time::Duration,
8150 };
8151
8152 use serde_json::{json, Value};
8153 use subc_protocol::{
8154 manifest::{
8155 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8156 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8157 },
8158 session::HealthStatus,
8159 FrameType,
8160 };
8161
8162 use super::*;
8163 use crate::{
8164 forwarding::{DataRoute, DataRouteState},
8165 registry::ChannelState,
8166 router::FrameSink,
8167 stderr_tail::DEFAULT_MAX_LINE_BYTES,
8168 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8169 RouteCtx, Router,
8170 };
8171 use tokio::{
8172 sync::mpsc,
8173 time::{sleep, Instant},
8174 };
8175 use tracing::{
8176 field::{Field, Visit},
8177 Event, Subscriber,
8178 };
8179 use tracing_subscriber::{layer::Context, prelude::*, Layer};
8180
8181 fn fake_aft_stub_path() -> PathBuf {
8200 let mut path = std::env::current_exe().expect("current_exe available in tests");
8201 path.pop(); path.pop(); path.push(if cfg!(windows) {
8204 "fake-aft-stub.exe"
8205 } else {
8206 "fake-aft-stub"
8207 });
8208 assert!(
8209 path.exists(),
8210 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8211 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8212 path.display()
8213 );
8214 path
8215 }
8216
8217 fn client_retries(code: &str) -> bool {
8222 subc_protocol::error_codes::is_retryable_route_open(code)
8223 }
8224
8225 #[test]
8236 fn retryability_of_forwarding_codes_matches_the_failure() {
8237 let transient = [
8240 ForwardingError::NoModuleConnection,
8241 ForwardingError::ModuleReloading {
8242 module_id: "m".into(),
8243 },
8244 ForwardingError::StaleModuleEndpoint,
8245 ForwardingError::UnknownReservation {
8246 client_channel: 1,
8247 module_channel: 1,
8248 },
8249 ForwardingError::ConnectionClosing {
8250 connection_id: ConnectionId::new(1),
8251 },
8252 ForwardingError::ClientEgressClosed {
8253 connection_id: ConnectionId::new(1),
8254 },
8255 ForwardingError::ModuleEgressUnavailable {
8256 connection_id: ConnectionId::new(1),
8257 },
8258 ];
8259 for err in transient {
8260 let code = forwarding_error_code(&err);
8261 assert!(
8262 client_retries(code),
8263 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8264 );
8265 }
8266
8267 let permanent = [
8272 ForwardingError::ConnectionRoleConflict {
8273 connection_id: ConnectionId::new(1),
8274 },
8275 ForwardingError::ClientRouteChannelExhausted {
8276 connection_id: ConnectionId::new(1),
8277 },
8278 ForwardingError::ModuleRouteChannelExhausted {
8279 endpoint: ModuleEndpointId {
8280 connection_id: ConnectionId::new(1),
8281 generation: 1,
8282 },
8283 },
8284 ForwardingError::RelayCorrelationExhausted,
8285 ForwardingError::RouteOpenBuild("x".into()),
8286 ForwardingError::Poisoned,
8287 ];
8288 for err in permanent {
8289 let code = forwarding_error_code(&err);
8290 assert!(
8291 !client_retries(code),
8292 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8293 );
8294 }
8295 }
8296
8297 #[tokio::test]
8325 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8326 let handler = ControlHandler::default();
8327 let frame =
8328 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8329
8330 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8334 assert_eq!(
8335 stamped,
8336 Principal::Direct,
8337 "a caller that proved nothing must not be stamped as a supervised module"
8338 );
8339
8340 let forged = handler
8345 .route_open_principal(
8346 &frame,
8347 Some(ConsumerIdentity {
8348 module_id: "aft".to_string(),
8349 launch_nonce: "not-a-real-nonce".to_string(),
8350 }),
8351 )
8352 .unwrap();
8353 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8354 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8355 }
8356
8357 #[test]
8374 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8375 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8376 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8377 let ClientControlRequest::RouteOpen {
8378 consumer_identity, ..
8379 } = parsed
8380 else {
8381 panic!("route.open body must parse as RouteOpen");
8382 };
8383 assert_eq!(
8384 consumer_identity,
8385 Some(ConsumerIdentity {
8386 module_id: "aft".to_string(),
8387 launch_nonce: "n".to_string(),
8388 }),
8389 "the wire field name must reach the value route_open_principal reads"
8390 );
8391 }
8392
8393 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8394 ModuleManifest::builder(module_id, "0.1.0")
8395 .protocol_ver(protocol_ver)
8396 .provides(vec![ProviderRole::ToolProvider {
8397 tools: vec![Tool {
8398 name: "read".to_string(),
8399 description: None,
8400 execution_mode: ExecutionMode::Pure,
8401 schema: json!({"type": "object"}),
8402 }],
8403 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8404 concurrency: Concurrency::ModuleManaged,
8405 emits_push: true,
8406 sub_supervises: true,
8407 }])
8408 .build()
8409 }
8410
8411 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8412 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8413 }
8414
8415 fn hello_frame_with_control_ops(
8416 module_id: &str,
8417 protocol_ver: u8,
8418 corr: u64,
8419 control_ops: Option<Vec<String>>,
8420 ) -> Frame {
8421 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8422 }
8423
8424 fn hello_frame_with_nonce(
8425 module_id: &str,
8426 protocol_ver: u8,
8427 corr: u64,
8428 launch_nonce: Option<&str>,
8429 ) -> Frame {
8430 hello_frame_full(
8431 module_id,
8432 protocol_ver,
8433 corr,
8434 None,
8435 launch_nonce.map(ToOwned::to_owned),
8436 )
8437 }
8438
8439 fn hello_frame_full(
8440 module_id: &str,
8441 protocol_ver: u8,
8442 corr: u64,
8443 control_ops: Option<Vec<String>>,
8444 launch_nonce: Option<String>,
8445 ) -> Frame {
8446 let body = serde_json::to_vec(&ModuleHelloBody {
8447 manifest: manifest(module_id, protocol_ver),
8448 protocol_ver,
8449 control_ops,
8450 launch_nonce,
8451 })
8452 .unwrap();
8453 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8454 }
8455
8456 fn hello_frame_with_rich_ui(
8457 module_id: &str,
8458 protocol_ver: u8,
8459 corr: u64,
8460 launch_nonce: Option<&str>,
8461 rich_ui: Value,
8462 ) -> Frame {
8463 let mut body = serde_json::to_value(ModuleHelloBody {
8464 manifest: manifest(module_id, protocol_ver),
8465 protocol_ver,
8466 control_ops: None,
8467 launch_nonce: launch_nonce.map(str::to_string),
8468 })
8469 .unwrap();
8470 body["manifest"]["rich_ui"] = rich_ui;
8471 Frame::build(
8472 FrameType::Hello,
8473 control_flags(),
8474 0,
8475 0,
8476 corr,
8477 serde_json::to_vec(&body).unwrap(),
8478 )
8479 .unwrap()
8480 }
8481
8482 fn non_routable_hello_frame_with_control_ops(
8483 module_id: &str,
8484 corr: u64,
8485 control_ops: Option<Vec<String>>,
8486 ) -> Frame {
8487 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8488 manifest.provides.clear();
8489 let body = serde_json::to_vec(&ModuleHelloBody {
8490 manifest,
8491 protocol_ver: PROTOCOL_VERSION,
8492 control_ops,
8493 launch_nonce: None,
8494 })
8495 .unwrap();
8496 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8497 }
8498
8499 fn capability_grammar_hello_frame(
8500 capabilities: Value,
8501 runtime_computed: Option<Value>,
8502 corr: u64,
8503 ) -> Frame {
8504 let mut body = serde_json::to_value(ModuleHelloBody {
8505 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8506 protocol_ver: PROTOCOL_VERSION,
8507 control_ops: None,
8508 launch_nonce: None,
8509 })
8510 .expect("HELLO body serializes");
8511 body["manifest"]["capabilities"] = capabilities;
8512 if let Some(runtime_computed) = runtime_computed {
8513 body["runtime_computed"] = runtime_computed;
8514 }
8515 Frame::build(
8516 FrameType::Hello,
8517 control_flags(),
8518 0,
8519 0,
8520 corr,
8521 serde_json::to_vec(&body).expect("HELLO body reserializes"),
8522 )
8523 .expect("HELLO frame builds")
8524 }
8525
8526 fn channel_request(channel: u16, corr: u64) -> Frame {
8527 Frame::build(
8528 FrameType::Request,
8529 Flags::new(true, Priority::Interactive, false),
8530 channel,
8531 0,
8532 corr,
8533 b"opaque".to_vec(),
8534 )
8535 .unwrap()
8536 }
8537
8538 fn route_ctx(
8539 connection_id: ConnectionId,
8540 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8541 let (tx, rx) = mpsc::channel(8);
8542 (
8543 RouteCtx {
8544 connection_id,
8545 egress: FrameSink::new(tx),
8546 },
8547 rx,
8548 )
8549 }
8550
8551 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8552 serde_json::from_slice(&frame.body).unwrap()
8553 }
8554
8555 async fn hello_via_sink(
8560 handler: &ControlHandler,
8561 ctx: &RouteCtx,
8562 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8563 hello: Frame,
8564 ) -> Frame {
8565 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8566 assert!(
8567 replies.is_empty(),
8568 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8569 );
8570 let ack = rx
8571 .try_recv()
8572 .expect("HELLO_ACK is queued on the module sink")
8573 .frame;
8574 assert_eq!(ack.header.ty, FrameType::HelloAck);
8575 ack
8576 }
8577
8578 fn parse_error(frame: &Frame) -> Value {
8579 serde_json::from_slice(&frame.body).unwrap()
8580 }
8581
8582 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8583 serde_json::from_slice(&frame.body).unwrap()
8584 }
8585
8586 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8587 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8588 route_channel,
8589 route_epoch: 0,
8590 kind,
8591 })
8592 .unwrap();
8593 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8594 }
8595
8596 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8597 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8598 module_id: module_id.to_string(),
8599 })
8600 .unwrap();
8601 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8602 }
8603
8604 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8605 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8606 }
8607
8608 fn route_open_frame_with_consumer_capabilities(
8609 corr: u64,
8610 module_id: &str,
8611 project_root: ScratchDir,
8612 consumer_capabilities: Option<Vec<String>>,
8613 ) -> Frame {
8614 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8615 target: RouteTarget::ToolProvider {
8616 module_id: module_id.to_string(),
8617 },
8618 identity: BindIdentity::new(
8619 project_root.path().to_path_buf(),
8620 "unit".to_string(),
8621 "session".to_string(),
8622 ),
8623 consumer_identity: None,
8624 consumer_capabilities,
8625 role_versions: None,
8626 admission_facts: None,
8627 scope: None,
8628 })
8629 .unwrap();
8630 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8631 }
8632
8633 fn route_open_frame_with_role_versions(
8634 corr: u64,
8635 module_id: &str,
8636 project_root: ScratchDir,
8637 role_versions: Option<BTreeMap<String, String>>,
8638 ) -> Frame {
8639 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8640 target: RouteTarget::ToolProvider {
8641 module_id: module_id.to_string(),
8642 },
8643 identity: BindIdentity::new(
8644 project_root.path().to_path_buf(),
8645 "unit".to_string(),
8646 format!("session-{corr}"),
8647 ),
8648 consumer_identity: None,
8649 consumer_capabilities: None,
8650 role_versions,
8651 admission_facts: None,
8652 scope: None,
8653 })
8654 .unwrap();
8655 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8656 }
8657
8658 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8659 entries
8660 .iter()
8661 .map(|(role, version)| (role.to_string(), version.to_string()))
8662 .collect()
8663 }
8664
8665 fn route_open_frame_with_admission_facts(
8666 corr: u64,
8667 module_id: &str,
8668 project_root: ScratchDir,
8669 consumer_identity: Option<subc_control::ConsumerIdentity>,
8670 facts: Option<Value>,
8671 ) -> Frame {
8672 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8673 target: RouteTarget::ToolProvider {
8674 module_id: module_id.to_string(),
8675 },
8676 identity: BindIdentity::new(
8677 project_root.path().to_path_buf(),
8678 "unit".to_string(),
8679 format!("session-{corr}"),
8680 ),
8681 consumer_identity,
8682 consumer_capabilities: None,
8683 role_versions: None,
8684 admission_facts: facts,
8685 scope: None,
8686 })
8687 .unwrap();
8688 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8689 }
8690
8691 #[derive(Clone, Default)]
8692 struct EventCapture {
8693 events: Arc<Mutex<Vec<CapturedEvent>>>,
8694 }
8695
8696 #[derive(Clone, Debug)]
8697 struct CapturedEvent {
8698 target: String,
8699 level: tracing::Level,
8700 fields: BTreeMap<String, String>,
8701 }
8702
8703 impl EventCapture {
8704 fn events(&self) -> Vec<CapturedEvent> {
8705 self.events.lock().unwrap().clone()
8706 }
8707 }
8708
8709 impl<S> Layer<S> for EventCapture
8710 where
8711 S: Subscriber,
8712 {
8713 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8714 let mut visitor = EventFieldVisitor::default();
8715 event.record(&mut visitor);
8716 self.events.lock().unwrap().push(CapturedEvent {
8717 target: event.metadata().target().to_string(),
8718 level: *event.metadata().level(),
8719 fields: visitor.fields,
8720 });
8721 }
8722 }
8723
8724 #[derive(Default)]
8725 struct EventFieldVisitor {
8726 fields: BTreeMap<String, String>,
8727 }
8728
8729 impl Visit for EventFieldVisitor {
8730 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8731 self.fields
8732 .insert(field.name().to_string(), format!("{value:?}"));
8733 }
8734 }
8735
8736 fn health_response(corr: u64, status: HealthStatus) -> Frame {
8737 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8738 status,
8739 detail: Some("warming".to_string()),
8740 metrics: Some(json!({"queue_depth": 3})),
8741 })
8742 .unwrap();
8743 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8744 }
8745
8746 fn route_bind_ack(corr: u64) -> Frame {
8747 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8748 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8749 }
8750
8751 fn unique_project_root(label: &str) -> ScratchDir {
8752 ScratchDir::new(label)
8753 }
8754
8755 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8756 match parse_route_poll(frame) {
8757 ClientControlResponse::RoutePoll {
8758 status: None,
8759 live: Some(live),
8760 ..
8761 } => assert_eq!(live, expected_live),
8762 other => panic!("unexpected route.poll response: {other:?}"),
8763 }
8764 }
8765
8766 fn bind_liveness_route(
8767 registry: &Registry,
8768 forwarding: &ForwardingTable,
8769 module_id: &str,
8770 ) -> (RouteCtx, u16, u32) {
8771 let module_connection = ConnectionId::new(101);
8772 let client_connection = ConnectionId::new(202);
8773 let registration = registry
8774 .register_with_control_ops(
8775 manifest(module_id, PROTOCOL_VERSION),
8776 PROTOCOL_VERSION,
8777 module_connection,
8778 module_baseline_control_ops(),
8779 )
8780 .unwrap();
8781 let (module_tx, _module_rx) = mpsc::channel(8);
8782 let endpoint = forwarding
8783 .register_module_connection(
8784 module_connection,
8785 module_id.to_string(),
8786 PROTOCOL_VERSION,
8787 manifest_concurrency(®istration.manifest),
8788 FrameSink::new(module_tx),
8789 )
8790 .unwrap();
8791 let (client_ctx, _client_rx) = route_ctx(client_connection);
8792 let pending = forwarding
8793 .begin_route_bind_relay_for_test(
8794 client_connection,
8795 client_ctx.egress.clone(),
8796 1,
8797 module_id,
8798 )
8799 .unwrap();
8800 assert_eq!(pending.endpoint, endpoint);
8801 let route_channel = pending.client_channel;
8802 let route_epoch = pending.client_epoch;
8803 forwarding
8804 .complete_pending_relay(
8805 module_connection,
8806 pending.corr,
8807 RouteBindRelayOutcome::Accepted,
8808 )
8809 .unwrap();
8810 (client_ctx, route_channel, route_epoch)
8811 }
8812
8813 struct FakeProcessLiveness {
8814 live: Option<bool>,
8815 }
8816
8817 impl ModuleProcessLiveness for FakeProcessLiveness {
8818 fn process_live(&self, _module_id: &str) -> Option<bool> {
8819 self.live
8820 }
8821 }
8822
8823 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8824 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8825 {
8826 let registry = Arc::new(Registry::default());
8827 let supervisor_handle = SupervisorHandle::new();
8828 let supervisor = Supervisor::new_for_test(
8829 Arc::clone(®istry),
8830 RestartPolicy::new(1, Duration::from_millis(10)),
8831 )
8832 .with_handle(supervisor_handle.clone());
8833 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8834 let module = supervisor
8835 .spawn(ModuleSpec {
8836 module_id: "stderr-tail-wire".to_string(),
8837 program: fake_aft_stub_path(),
8838 args: Vec::new(),
8839 env: vec![
8840 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8841 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8842 ],
8843 reserved: false,
8844 reserved_prefixes: Vec::new(),
8845 protocol: ModuleProtocol::Subc,
8846 overlap: Default::default(),
8847 })
8848 .unwrap();
8849
8850 let deadline = Instant::now() + Duration::from_secs(5);
8851 loop {
8852 let tail = module.stderr_tail(None, None);
8853 if tail
8854 .entries
8855 .iter()
8856 .any(|entry| matches!(entry, TailEntry::ProcessStart))
8857 && tail.entries.iter().any(|entry| {
8858 matches!(
8859 entry,
8860 TailEntry::Line {
8861 truncated: true,
8862 ..
8863 }
8864 )
8865 })
8866 {
8867 break;
8868 }
8869 assert!(
8870 Instant::now() < deadline,
8871 "module did not produce a truncated line and restart boundary: {tail:?}"
8872 );
8873 sleep(Duration::from_millis(10)).await;
8874 }
8875
8876 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8877 let request = ClientControlRequest::SupervisorStderrTail {
8878 module_id: "stderr-tail-wire".to_string(),
8879 max_lines: None,
8880 max_bytes: None,
8881 };
8882 let frame = Frame::build(
8883 FrameType::Request,
8884 control_flags(),
8885 0,
8886 0,
8887 1,
8888 serde_json::to_vec(&request).unwrap(),
8889 )
8890 .unwrap();
8891 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8892 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8893 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8894 serde_json::from_slice(&responses[0].body).unwrap()
8895 else {
8896 panic!("expected supervisor.stderr_tail response");
8897 };
8898
8899 assert!(
8900 tail.entries
8901 .iter()
8902 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8903 "the control response lost the restart boundary"
8904 );
8905 let Some(StderrTailEntry::Line {
8906 text,
8907 truncated,
8908 at_ms,
8909 }) = tail.entries.iter().find(|entry| {
8910 matches!(
8911 entry,
8912 StderrTailEntry::Line {
8913 truncated: true,
8914 ..
8915 }
8916 )
8917 })
8918 else {
8919 panic!("the control response lost the truncated line");
8920 };
8921 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8922 assert!(*truncated);
8923 assert!(
8924 at_ms.is_some(),
8925 "the control response lost the line's capture time"
8926 );
8927 }
8928
8929 #[tokio::test(flavor = "current_thread")]
8934 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8935 let dir = ScratchDir::new("terminals-off-worker");
8936 let journal_path = dir.join("terminals.jsonl");
8937 let registry = Arc::new(Registry::default());
8938 let supervisor_handle = SupervisorHandle::new();
8939 let supervisor =
8940 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8941 .with_handle(supervisor_handle.clone())
8942 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8943 let module = supervisor
8944 .spawn(ModuleSpec {
8945 module_id: "terminal-off-worker".to_string(),
8946 program: fake_aft_stub_path(),
8947 args: Vec::new(),
8948 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8949 reserved: false,
8950 reserved_prefixes: Vec::new(),
8951 protocol: ModuleProtocol::Subc,
8952 overlap: Default::default(),
8953 })
8954 .unwrap();
8955 let deadline = Instant::now() + Duration::from_secs(5);
8956 while module.terminal_history().entries.len() != 2 {
8957 assert!(Instant::now() < deadline, "module did not record two exits");
8958 sleep(Duration::from_millis(10)).await;
8959 }
8960
8961 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8962 let handler =
8963 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
8964 let frame = Frame::build(
8965 FrameType::Request,
8966 control_flags(),
8967 0,
8968 0,
8969 1,
8970 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8971 module_id: "terminal-off-worker".to_string(),
8972 })
8973 .unwrap(),
8974 )
8975 .unwrap();
8976 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8977 let spawned_at = std::time::Instant::now();
8978 let read = tokio::spawn({
8979 let handler = Arc::clone(&handler);
8980 async move { handler.handle_control_frame(&ctx, frame).await }
8981 });
8982 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8985 .await
8986 .unwrap()
8987 .expect("the history read reached its pause");
8988 let elapsed = spawned_at.elapsed();
8989 assert!(
8990 elapsed < Duration::from_secs(2) && !read.is_finished(),
8991 "this task could not run while the history read was paused \
8992 (resumed after {elapsed:?}, read finished: {})",
8993 read.is_finished()
8994 );
8995
8996 drop(release);
8997 let responses = read.await.unwrap().unwrap();
8998 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8999 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
9000 panic!("expected supervisor.terminals response");
9001 };
9002 assert_eq!(terminals.entries.len(), 2);
9003 assert_eq!(terminals.journal_skipped_lines, 0);
9004 assert_eq!(terminals.journal_read_errors, 0);
9005 }
9006
9007 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9008 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
9009 let registry = Arc::new(Registry::default());
9010 let supervisor_handle = SupervisorHandle::new();
9011 let supervisor =
9012 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9013 .with_handle(supervisor_handle.clone());
9014 let module = supervisor
9015 .spawn(ModuleSpec {
9016 module_id: "terminal-golden".to_string(),
9017 program: fake_aft_stub_path(),
9018 args: Vec::new(),
9019 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9020 reserved: false,
9021 reserved_prefixes: Vec::new(),
9022 protocol: ModuleProtocol::Subc,
9023 overlap: Default::default(),
9024 })
9025 .unwrap();
9026
9027 let deadline = Instant::now() + Duration::from_secs(5);
9028 while module.terminal_history().entries.len() != 2 {
9029 assert!(
9030 Instant::now() < deadline,
9031 "module did not retain two terminal exits: {:?}",
9032 module.terminal_history()
9033 );
9034 sleep(Duration::from_millis(10)).await;
9035 }
9036
9037 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9038 let request = ClientControlRequest::SupervisorTerminals {
9039 module_id: "terminal-golden".to_string(),
9040 };
9041 let frame = Frame::build(
9042 FrameType::Request,
9043 control_flags(),
9044 0,
9045 0,
9046 1,
9047 serde_json::to_vec(&request).unwrap(),
9048 )
9049 .unwrap();
9050 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9051 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9052 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9053 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9054 panic!("expected supervisor.terminals response");
9055 };
9056 assert_eq!(terminals.entries.len(), 2);
9057 assert_eq!(terminals.dropped, 0);
9058
9059 let mut rendered = serde_json::to_value(response).unwrap();
9060 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9063 for (index, entry) in rendered["entries"]
9064 .as_array_mut()
9065 .expect("terminal response entries array")
9066 .iter_mut()
9067 .enumerate()
9068 {
9069 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9070 }
9071
9072 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9073 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9074 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9075 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9076 std::fs::write(&golden_path, &serialized).unwrap();
9077 }
9078 let expected: Value =
9079 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9080 assert_eq!(rendered, expected);
9081 }
9082
9083 #[test]
9084 fn hello_registers_manifest_and_returns_ack() {
9085 let registry = Arc::new(Registry::default());
9086 let handler = ControlHandler::new(Arc::clone(®istry));
9087 let conn = ConnectionId::new(1);
9088
9089 let responses = handler
9090 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9091 .unwrap();
9092
9093 assert_eq!(responses.len(), 1);
9094 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9095 assert_eq!(responses[0].header.channel, 0);
9096 assert_eq!(responses[0].header.corr, 7);
9097 let ack = parse_ack(&responses[0]);
9098 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9099 assert!(ack
9100 .subc_capabilities
9101 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9102 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9103 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9104 assert!(ack
9105 .subc_ops
9106 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9107 assert!(ack
9108 .subc_ops
9109 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9110
9111 let registration = registry.get_module("aft").unwrap().unwrap();
9112 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9113 assert_eq!(registration.state, ChannelState::Active);
9114 assert_eq!(registration.connection_id, conn);
9115 assert_eq!(registration.control_ops, module_baseline_control_ops());
9116 }
9117
9118 #[test]
9119 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9120 let invalid_identifiers = [
9121 ("case_change", "credentials-Provider/v1"),
9122 ("leading_zero", "credentials-provider/v01"),
9123 ("trailing_hyphen", "credentials-provider-/v1"),
9124 ("consecutive_hyphens", "credentials--provider/v1"),
9125 ("uppercase", "Credentials-provider/v1"),
9126 ("missing_v", "credentials-provider/1"),
9127 ("whitespace", "credentials provider/v1"),
9128 ("zero_version", "credentials-provider/v0"),
9129 ("out_of_range_version", "credentials-provider/v4294967296"),
9130 (
9131 "overlength_name",
9132 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9133 ),
9134 ];
9135 let mut cases = invalid_identifiers
9136 .into_iter()
9137 .map(|(name, identifier)| {
9138 (
9139 format!("identifier_{name}"),
9140 "capabilities.provides[0]".to_string(),
9141 identifier.to_string(),
9142 json!({ "provides": [identifier] }),
9143 None,
9144 )
9145 })
9146 .collect::<Vec<_>>();
9147 cases.extend([
9148 (
9149 "unknown_need".to_string(),
9150 "capabilities.requires[0].need".to_string(),
9151 "deferred".to_string(),
9152 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9153 None,
9154 ),
9155 (
9156 "duplicate_provides".to_string(),
9157 "capabilities.provides[1]".to_string(),
9158 "credentials-provider/v1".to_string(),
9159 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9160 None,
9161 ),
9162 (
9163 "duplicate_must_never_reach".to_string(),
9164 "capabilities.must_never_reach[1]".to_string(),
9165 "credentials-provider/v1".to_string(),
9166 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9167 None,
9168 ),
9169 (
9170 "duplicate_requires_same_need".to_string(),
9171 "capabilities.requires[1]".to_string(),
9172 "credentials-provider/v1".to_string(),
9173 json!({ "requires": [
9174 { "capability": "credentials-provider/v1", "need": "required" },
9175 { "capability": "credentials-provider/v1", "need": "required" }
9176 ] }),
9177 None,
9178 ),
9179 (
9180 "duplicate_requires_conflicting_need".to_string(),
9181 "capabilities.requires[1]".to_string(),
9182 "credentials-provider/v1".to_string(),
9183 json!({ "requires": [
9184 { "capability": "credentials-provider/v1", "need": "required" },
9185 { "capability": "credentials-provider/v1", "need": "optional" }
9186 ] }),
9187 None,
9188 ),
9189 (
9190 "capabilities_root_pointer".to_string(),
9191 "runtime_computed[0]".to_string(),
9192 "/capabilities".to_string(),
9193 json!({}),
9194 Some(json!(["/capabilities"])),
9195 ),
9196 (
9197 "capabilities_descendant_pointer".to_string(),
9198 "runtime_computed[0]".to_string(),
9199 "/capabilities/provides".to_string(),
9200 json!({}),
9201 Some(json!(["/capabilities/provides"])),
9202 ),
9203 (
9204 "malformed_pointer_without_leading_slash".to_string(),
9205 "runtime_computed[0]".to_string(),
9206 "capabilities".to_string(),
9207 json!({}),
9208 Some(json!(["capabilities"])),
9209 ),
9210 (
9211 "malformed_pointer_escape".to_string(),
9212 "runtime_computed[0]".to_string(),
9213 "/roles/~2/tools".to_string(),
9214 json!({}),
9215 Some(json!(["/roles/~2/tools"])),
9216 ),
9217 (
9218 "unknown_capabilities_field".to_string(),
9219 "capabilities.future".to_string(),
9220 "<array>".to_string(),
9221 json!({ "future": [] }),
9222 None,
9223 ),
9224 ]);
9225
9226 for (index, (name, field, value, capabilities, runtime_computed)) in
9227 cases.into_iter().enumerate()
9228 {
9229 let registry = Arc::new(Registry::default());
9230 let handler = ControlHandler::new(Arc::clone(®istry));
9231 let response = handler
9232 .handle_control(
9233 ConnectionId::new((index + 1) as u64),
9234 capability_grammar_hello_frame(
9235 capabilities,
9236 runtime_computed,
9237 index as u64 + 1,
9238 ),
9239 )
9240 .expect("invalid HELLO returns a refusal");
9241
9242 assert_eq!(response.len(), 1, "{name} must emit one refusal");
9243 let error = parse_error(&response[0]);
9244 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9245 let message = error["message"]
9246 .as_str()
9247 .expect("error message is a string");
9248 assert!(
9249 message.contains(&field),
9250 "{name}: field missing from {message}"
9251 );
9252 assert!(
9253 message.contains(&value),
9254 "{name}: value missing from {message}"
9255 );
9256 assert_eq!(
9257 registry
9258 .active_registration_count()
9259 .expect("registry reads"),
9260 0,
9261 "{name}: refused HELLO must not create a catalog entry"
9262 );
9263 }
9264 }
9265
9266 #[test]
9267 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9268 let registry = Arc::new(Registry::default());
9269 let handler = ControlHandler::new(Arc::clone(®istry));
9270 let capabilities = json!({
9271 "provides": ["credentials-provider/v1"],
9272 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9273 "must_never_reach": ["federation-transport/v1"]
9274 });
9275 let response = handler
9276 .handle_control(
9277 ConnectionId::new(99),
9278 capability_grammar_hello_frame(
9279 capabilities.clone(),
9280 Some(json!(["/roles/0/tools"])),
9281 99,
9282 ),
9283 )
9284 .expect("valid HELLO registers");
9285 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9286
9287 let request = Frame::build(
9288 FrameType::Request,
9289 control_flags(),
9290 0,
9291 0,
9292 100,
9293 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9294 .expect("catalog request serializes"),
9295 )
9296 .expect("catalog request frame builds");
9297 let response = handler
9298 .handle_catalog_list(request, None)
9299 .expect("catalog list succeeds");
9300 let ClientControlResponse::CatalogList { modules, .. } =
9301 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9302 else {
9303 panic!("catalog request must return catalog.list");
9304 };
9305 assert_eq!(modules.len(), 1);
9306 assert_eq!(
9307 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9308 capabilities
9309 );
9310 }
9311
9312 #[test]
9313 fn catalog_list_mirrors_management_operation_description() {
9314 let registry = Arc::new(Registry::default());
9315 let handler = ControlHandler::new(Arc::clone(®istry));
9316 let description = "List managed records and return their identifiers and metadata.";
9317 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9318 manifest.provides = vec![ProviderRole::ManagementSurface {
9319 operations: vec![ManagementOperation {
9320 name: "records.list".to_string(),
9321 kind: ManagementOperationKind::Query,
9322 description: Some(description.to_string()),
9323 }],
9324 config_schema: json!({"type": "object"}),
9325 observability: vec![ObservabilitySurface {
9326 name: "records.stats".to_string(),
9327 kind: ObservabilityKind::Snapshot,
9328 }],
9329 identity_scope: vec![IdentityScope::Project],
9330 concurrency: Concurrency::ModuleManaged,
9331 }];
9332 registry
9333 .register_with_control_ops(
9334 manifest,
9335 PROTOCOL_VERSION,
9336 ConnectionId::new(99),
9337 Vec::new(),
9338 )
9339 .expect("described management manifest registers");
9340
9341 let request = Frame::build(
9342 FrameType::Request,
9343 control_flags(),
9344 0,
9345 0,
9346 100,
9347 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9348 .expect("catalog request serializes"),
9349 )
9350 .expect("catalog request frame builds");
9351 let response = handler
9352 .handle_catalog_list(request, None)
9353 .expect("catalog list succeeds");
9354 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9355 assert_eq!(
9356 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9357 "catalog.list must preserve the declared operation description verbatim"
9358 );
9359 }
9360
9361 #[test]
9362 fn catalog_list_mirrors_registered_operation_delivery() {
9363 let registry = Arc::new(Registry::default());
9364 let handler = ControlHandler::new(Arc::clone(®istry));
9365 let delivery =
9366 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9367 let mut body = serde_json::to_value(ModuleHelloBody {
9368 manifest: manifest("watch-management", PROTOCOL_VERSION),
9369 protocol_ver: PROTOCOL_VERSION,
9370 control_ops: None,
9371 launch_nonce: None,
9372 })
9373 .unwrap();
9374 body["manifest"]["provides"] = json!([{
9375 "role": "management_surface",
9376 "operations": [
9377 {"name": "status", "kind": "query"},
9378 {"name": "future", "kind": "query"},
9379 {"name": "reset", "kind": "mutate"}
9380 ],
9381 "config_schema": {}, "observability": [], "identity_scope": []
9382 }]);
9383 body["manifest"]["operation_delivery"] = delivery.clone();
9384 let hello = Frame::build(
9385 FrameType::Hello,
9386 control_flags(),
9387 0,
9388 0,
9389 99,
9390 serde_json::to_vec(&body).unwrap(),
9391 )
9392 .unwrap();
9393 let response = handler
9394 .handle_control(ConnectionId::new(99), hello)
9395 .unwrap();
9396 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9397
9398 let (_, registrations) = registry.list_modules().unwrap();
9399 let registered = ®istrations[0].manifest;
9400 assert!(registered.is_watch_operation("status"));
9401 assert!(!registered.is_watch_operation("future"));
9402 assert!(!registered.is_watch_operation("reset"));
9403 assert!(!registered.is_watch_operation("missing"));
9404
9405 let request = Frame::build(
9406 FrameType::Request,
9407 control_flags(),
9408 0,
9409 0,
9410 100,
9411 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9412 )
9413 .unwrap();
9414 let response = handler.handle_catalog_list(request, None).unwrap();
9415 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9416 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9417 let ClientControlResponse::CatalogList { modules, .. } =
9418 serde_json::from_slice(&response[0].body).unwrap()
9419 else {
9420 panic!("expected catalog.list");
9421 };
9422 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9423 }
9424
9425 #[test]
9426 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
9427 let registry = Arc::new(Registry::default());
9428 let supervisor = SupervisorHandle::new();
9429 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9430 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9431 let descriptor = json!({
9432 "schema": "ckios.richui.manifest/1",
9433 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9434 "bytes": 4096,
9435 "chunks": 4
9436 });
9437 let connection = ConnectionId::new(201);
9438 let hello = handler
9439 .handle_control(
9440 connection,
9441 hello_frame_with_rich_ui(
9442 "vault",
9443 PROTOCOL_VERSION,
9444 201,
9445 Some("the-real-nonce"),
9446 descriptor.clone(),
9447 ),
9448 )
9449 .unwrap();
9450 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9451
9452 let request =
9453 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
9454 let provides = registry
9455 .get_module_by_connection(connection)
9456 .unwrap()
9457 .unwrap()
9458 .manifest
9459 .provides;
9460 handler
9461 .handle_catalog_update(connection, request, provides, None, Some(false))
9462 .unwrap();
9463
9464 let request = Frame::build(
9465 FrameType::Request,
9466 control_flags(),
9467 0,
9468 0,
9469 203,
9470 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9471 )
9472 .unwrap();
9473 let response = handler.handle_catalog_list(request, None).unwrap();
9474 let ClientControlResponse::CatalogList { modules, .. } =
9475 serde_json::from_slice(&response[0].body).unwrap()
9476 else {
9477 panic!("expected catalog.list");
9478 };
9479 assert_eq!(
9480 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
9481 descriptor
9482 );
9483 assert!(modules[0].nonce_verified);
9484 }
9485
9486 fn hello_with_role_declarations(declarations: Value) -> Frame {
9487 let mut body = serde_json::to_value(ModuleHelloBody {
9488 manifest: manifest("role-test", PROTOCOL_VERSION),
9489 protocol_ver: PROTOCOL_VERSION,
9490 control_ops: None,
9491 launch_nonce: None,
9492 })
9493 .unwrap();
9494 body["manifest"]["role_declarations"] = declarations;
9495 Frame::build(
9496 FrameType::Hello,
9497 control_flags(),
9498 0,
9499 0,
9500 301,
9501 serde_json::to_vec(&body).unwrap(),
9502 )
9503 .unwrap()
9504 }
9505
9506 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
9507 let request = Frame::build(
9508 FrameType::Request,
9509 control_flags(),
9510 0,
9511 0,
9512 302,
9513 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9514 )
9515 .unwrap();
9516 let response = handler.handle_catalog_list(request, None).unwrap();
9517 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
9518 let ClientControlResponse::CatalogList { mut modules, .. } =
9519 serde_json::from_slice(&response[0].body).unwrap()
9520 else {
9521 panic!("expected catalog.list");
9522 };
9523 (raw["modules"][0].clone(), modules.remove(0))
9524 }
9525
9526 #[test]
9527 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
9528 let registry = Arc::new(Registry::default());
9529 let handler = ControlHandler::new(Arc::clone(®istry));
9530 let declarations = json!({
9531 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
9532 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
9533 });
9534 let connection = ConnectionId::new(301);
9535 let response = handler
9536 .handle_control(
9537 connection,
9538 hello_with_role_declarations(declarations.clone()),
9539 )
9540 .unwrap();
9541 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9542 let registered = registry
9543 .get_module_by_connection(connection)
9544 .unwrap()
9545 .unwrap()
9546 .manifest;
9547 assert_eq!(
9551 serde_json::to_value(®istered.role_declarations).unwrap(),
9552 declarations
9553 );
9554 let (raw, entry) = role_declarations_catalog(&handler);
9555 assert_eq!(raw["role_declarations"], declarations);
9556 assert_eq!(entry.role_declarations, registered.role_declarations);
9557
9558 let request =
9559 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
9560 handler
9561 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
9562 .unwrap();
9563 let (raw, entry) = role_declarations_catalog(&handler);
9564 assert_eq!(raw["role_declarations"], declarations);
9565 assert_eq!(entry.role_declarations, registered.role_declarations);
9566 }
9567
9568 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
9569 let registry = Arc::new(Registry::default());
9570 let handler = ControlHandler::new(Arc::clone(®istry));
9571 let capture = EventCapture::default();
9572 let _subscriber =
9573 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9574 let response = handler
9575 .handle_control(
9576 ConnectionId::new(301),
9577 hello_with_role_declarations(declarations),
9578 )
9579 .unwrap();
9580 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9581 assert!(registry
9582 .get_module_by_connection(ConnectionId::new(301))
9583 .unwrap()
9584 .unwrap()
9585 .manifest
9586 .role_declarations
9587 .is_empty());
9588 let (raw, entry) = role_declarations_catalog(&handler);
9589 assert!(raw.get("role_declarations").is_none());
9590 assert!(entry.role_declarations.is_empty());
9591 let warnings: Vec<_> = capture
9592 .events()
9593 .into_iter()
9594 .filter(|event| {
9595 event.fields.get("message").map(String::as_str)
9596 == Some("dropping whole role_declarations map from HELLO")
9597 })
9598 .collect();
9599 assert_eq!(warnings.len(), 1);
9600 assert_eq!(warnings[0].level, tracing::Level::WARN);
9601 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
9602 assert_eq!(warnings[0].fields["reason"], reason);
9603 }
9604
9605 #[test]
9606 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
9607 for value in [
9608 json!(null),
9609 json!(false),
9610 json!(0),
9611 json!("opaque"),
9612 json!([]),
9613 ] {
9614 assert_hello_drops_role_declarations_with_warning(
9615 json!({"routing/v1": {}, "broken/v1": value}),
9616 "role_declarations values must be objects",
9617 );
9618 }
9619 assert_hello_drops_role_declarations_with_warning(
9620 json!([]),
9621 "role_declarations must be an object",
9622 );
9623 }
9624
9625 #[test]
9626 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
9627 assert_hello_drops_role_declarations_with_warning(
9628 json!({"routing/v1": {}, "bad key": {}}),
9629 "role_declarations key must use capability-name grammar",
9630 );
9631 }
9632
9633 #[test]
9634 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
9635 assert_hello_drops_role_declarations_with_warning(
9636 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
9637 "role_declarations exceeds 16384 compact JSON bytes",
9638 );
9639 }
9640
9641 #[test]
9642 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
9643 let registry = Arc::new(Registry::default());
9644 let handler = ControlHandler::new(registry.clone());
9645 let descriptor = json!({
9646 "schema": "ckios.richui.manifest/1",
9647 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9648 "bytes": 128,
9649 "chunks": 2
9650 });
9651 let malformed = [
9652 ("schema", json!("")),
9653 ("schema", json!("a".repeat(65))),
9654 ("schema", json!("Bad schema")),
9655 (
9656 "manifest_digest",
9657 json!(format!("sha256:{}", "A".repeat(64))),
9658 ),
9659 ("manifest_digest", json!("sha256:abc")),
9660 ("bytes", json!(0)),
9661 ("bytes", json!(16_777_217)),
9662 ("bytes", json!("128")),
9663 ("chunks", json!(0)),
9664 ("chunks", json!(1025)),
9665 ("chunks", json!("2")),
9666 ];
9667 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
9668 let mut invalid = descriptor.clone();
9669 invalid[field] = invalid_value;
9670 let response = handler
9671 .handle_control(
9672 ConnectionId::new(210 + index as u64),
9673 hello_frame_with_rich_ui(
9674 &format!("bad-rich-ui-{index}"),
9675 PROTOCOL_VERSION,
9676 210 + index as u64,
9677 None,
9678 invalid,
9679 ),
9680 )
9681 .unwrap();
9682 assert_eq!(response[0].header.ty, FrameType::Error);
9683 let error = parse_error(&response[0]);
9684 assert_eq!(error["code"], "invalid_manifest");
9685 assert!(error["message"]
9686 .as_str()
9687 .unwrap()
9688 .contains(&format!("rich_ui.{field}")));
9689 assert!(registry
9690 .get_module(&format!("bad-rich-ui-{index}"))
9691 .unwrap()
9692 .is_none());
9693 }
9694 }
9695
9696 #[test]
9697 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
9698 let registry = Arc::new(Registry::default());
9699 let handler = ControlHandler::new(Arc::clone(®istry));
9700 let connection = ConnectionId::new(219);
9701 registry
9702 .register_with_control_ops(
9703 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
9704 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
9705 "bad schema",
9706 format!("sha256:{}", "a".repeat(64)),
9707 128,
9708 2,
9709 )))
9710 .build(),
9711 PROTOCOL_VERSION,
9712 connection,
9713 Vec::new(),
9714 )
9715 .unwrap();
9716 let request =
9717 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
9718 let response = handler
9719 .handle_catalog_update(connection, request, Vec::new(), None, None)
9720 .unwrap();
9721 assert_eq!(response[0].header.ty, FrameType::Error);
9722 let error = parse_error(&response[0]);
9723 assert_eq!(error["code"], "invalid_manifest");
9724 assert!(error["message"]
9725 .as_str()
9726 .unwrap()
9727 .contains("rich_ui.schema"));
9728 }
9729
9730 #[test]
9731 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
9732 let registry = Arc::new(Registry::default());
9733 let supervisor = SupervisorHandle::new();
9734 supervisor.open_swap("provider", "candidate-nonce".to_string());
9735 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9736 let connection = ConnectionId::new(218);
9737 let hello = handler
9738 .handle_control(
9739 connection,
9740 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
9741 )
9742 .unwrap();
9743 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9744 assert!(registry.list_modules().unwrap().1.is_empty());
9745
9746 registry.promote_candidate("provider").unwrap().unwrap();
9747 let request = Frame::build(
9748 FrameType::Request,
9749 control_flags(),
9750 0,
9751 0,
9752 219,
9753 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9754 )
9755 .unwrap();
9756 let response = handler.handle_catalog_list(request, None).unwrap();
9757 let ClientControlResponse::CatalogList { modules, .. } =
9758 serde_json::from_slice(&response[0].body).unwrap()
9759 else {
9760 panic!("expected catalog.list");
9761 };
9762 assert_eq!(modules.len(), 1);
9763 assert!(modules[0].nonce_verified);
9764 }
9765
9766 #[test]
9767 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
9768 let registry = Arc::new(Registry::default());
9769 let handler = ControlHandler::new(Arc::clone(®istry));
9770 let hello = Frame::build(
9771 FrameType::Hello,
9772 control_flags(),
9773 0,
9774 0,
9775 220,
9776 serde_json::to_vec(&ModuleHelloBody {
9777 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
9778 protocol_ver: PROTOCOL_VERSION,
9779 control_ops: None,
9780 launch_nonce: Some("unverified".to_string()),
9781 })
9782 .unwrap(),
9783 )
9784 .unwrap();
9785 assert_eq!(
9786 handler
9787 .handle_control(ConnectionId::new(220), hello)
9788 .unwrap()[0]
9789 .header
9790 .ty,
9791 FrameType::HelloAck
9792 );
9793 let request = Frame::build(
9794 FrameType::Request,
9795 control_flags(),
9796 0,
9797 0,
9798 221,
9799 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9800 )
9801 .unwrap();
9802 let response = handler.handle_catalog_list(request, None).unwrap();
9803 let ClientControlResponse::CatalogList { modules, .. } =
9804 serde_json::from_slice(&response[0].body).unwrap()
9805 else {
9806 panic!("expected catalog.list");
9807 };
9808 assert_eq!(
9809 serde_json::to_vec(&modules[0]).unwrap(),
9810 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
9811 );
9812 assert!(!modules[0].nonce_verified);
9813 }
9814
9815 #[test]
9816 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
9817 let registry = Arc::new(Registry::default());
9818 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9819 [("vault".to_string(), true), ("squatter".to_string(), true)],
9820 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9821 );
9822 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
9823 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9824 provides: vec!["credentials-provider/v1".to_string()],
9825 requires: Vec::new(),
9826 must_never_reach: Vec::new(),
9827 });
9828 let frame = Frame::build(
9829 FrameType::Hello,
9830 control_flags(),
9831 0,
9832 0,
9833 77,
9834 serde_json::to_vec(&ModuleHelloBody {
9835 manifest: squatter,
9836 protocol_ver: PROTOCOL_VERSION,
9837 control_ops: None,
9838 launch_nonce: None,
9839 })
9840 .expect("HELLO serializes"),
9841 )
9842 .expect("HELLO frame builds");
9843 let response = handler
9844 .handle_control(ConnectionId::new(77), frame)
9845 .expect("reserved claim receives a typed refusal");
9846 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9847 assert_eq!(
9848 registry
9849 .active_registration_count()
9850 .expect("registry reads"),
9851 0,
9852 "a reserved capability refusal must not leave a catalog entry"
9853 );
9854 }
9855
9856 #[test]
9857 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
9858 for settlement in ["timeout", "inconclusive", "drop"] {
9859 let breakers = RouteBindBreakers::default();
9860 let RouteBindAdmission::Admitted {
9861 guard: mut old,
9862 probe: false,
9863 } = breakers.admit("prov")
9864 else {
9865 panic!("ordinary relay admitted")
9866 };
9867 let RouteBindAdmission::Admitted {
9868 guard: mut opener, ..
9869 } = breakers.admit("prov")
9870 else {
9871 panic!("second relay admitted")
9872 };
9873 assert!(
9874 !opener
9875 .record_timeout(1, Duration::ZERO)
9876 .unwrap()
9877 .reopened_after_probe
9878 );
9879 let RouteBindAdmission::Admitted {
9880 guard: mut probe,
9881 probe: true,
9882 } = breakers.admit("prov")
9883 else {
9884 panic!("one cooldown probe admitted")
9885 };
9886 match settlement {
9887 "timeout" => assert!(
9888 !old.record_timeout(1, Duration::ZERO)
9889 .unwrap()
9890 .reopened_after_probe
9891 ),
9892 "inconclusive" => old.record_inconclusive(),
9893 "drop" => drop(old),
9894 _ => unreachable!(),
9895 }
9896 assert!(
9897 matches!(
9898 breakers.admit("prov"),
9899 RouteBindAdmission::Refused {
9900 probe_in_flight: true,
9901 ..
9902 }
9903 ),
9904 "{settlement} of a pre-open relay cannot release the real probe"
9905 );
9906 assert!(
9907 probe
9908 .record_timeout(1, Duration::ZERO)
9909 .unwrap()
9910 .reopened_after_probe
9911 );
9912 assert!(matches!(
9913 breakers.admit("prov"),
9914 RouteBindAdmission::Admitted { probe: true, .. }
9915 ));
9916 }
9917 let breakers = RouteBindBreakers::default();
9918 let admit = || match breakers.admit("prov") {
9919 RouteBindAdmission::Admitted { guard, .. } => guard,
9920 _ => panic!("relay admitted"),
9921 };
9922 admit().record_timeout(1, Duration::ZERO);
9923 let mut old_probe = admit();
9924 breakers.reset_for_new_module_connection("prov");
9925 admit().record_timeout(1, Duration::ZERO);
9926 let _new_probe = admit();
9927 old_probe.record_inconclusive();
9928 assert!(matches!(
9929 breakers.admit("prov"),
9930 RouteBindAdmission::Refused {
9931 probe_in_flight: true,
9932 ..
9933 }
9934 ));
9935 }
9936
9937 #[tokio::test]
9938 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
9939 for candidate in [false, true] {
9940 let registry = Arc::new(Registry::default());
9941 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9942 [("vault".to_string(), true), ("squatter".to_string(), true)],
9943 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9944 );
9945 let conn = ConnectionId::new(77);
9946 let (ctx, mut rx) = route_ctx(conn);
9947 let initial = capability_manifest("squatter", &[], &[]);
9948 if candidate {
9949 registry
9950 .register_candidate_with_control_ops(
9951 initial.clone(),
9952 PROTOCOL_VERSION,
9953 conn,
9954 module_baseline_control_ops(),
9955 )
9956 .unwrap();
9957 handler
9958 .forwarding
9959 .register_candidate_module_connection(
9960 conn,
9961 "squatter".to_string(),
9962 PROTOCOL_VERSION,
9963 manifest_concurrency(&initial),
9964 ctx.egress.clone(),
9965 )
9966 .unwrap();
9967 } else {
9968 hello_via_sink(
9969 &handler,
9970 &ctx,
9971 &mut rx,
9972 hello_frame_with_manifest(initial.clone(), 1),
9973 )
9974 .await;
9975 }
9976 let response = handler
9977 .handle_control_frame(
9978 &ctx,
9979 catalog_update_with_capabilities_frame(
9980 2,
9981 capability_manifest("squatter", &["credentials-provider/v1"], &[])
9982 .capabilities
9983 .unwrap(),
9984 ),
9985 )
9986 .await
9987 .unwrap();
9988 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9989 assert_eq!(
9990 registry
9991 .get_module_by_connection(conn)
9992 .unwrap()
9993 .unwrap()
9994 .manifest,
9995 initial
9996 );
9997 }
9998 }
9999
10000 #[test]
10001 fn server_describe_surfaces_required_capability_verdict_fields() {
10002 let registry = Arc::new(Registry::default());
10003 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10004 [
10005 ("consumer".to_string(), true),
10006 ("provider".to_string(), false),
10007 ],
10008 BTreeMap::new(),
10009 );
10010 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10011 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10012 provides: Vec::new(),
10013 requires: vec![subc_protocol::manifest::CapabilityRequirement {
10014 capability: "credentials-provider/v1".to_string(),
10015 need: subc_protocol::manifest::CapabilityNeed::Required,
10016 }],
10017 must_never_reach: Vec::new(),
10018 });
10019 let hello = Frame::build(
10020 FrameType::Hello,
10021 control_flags(),
10022 0,
10023 0,
10024 78,
10025 serde_json::to_vec(&ModuleHelloBody {
10026 manifest: consumer,
10027 protocol_ver: PROTOCOL_VERSION,
10028 control_ops: None,
10029 launch_nonce: None,
10030 })
10031 .expect("HELLO serializes"),
10032 )
10033 .expect("HELLO frame builds");
10034 handler
10035 .handle_control(ConnectionId::new(78), hello)
10036 .expect("consumer registers");
10037 let describe = Frame::build(
10038 FrameType::Request,
10039 control_flags(),
10040 0,
10041 0,
10042 79,
10043 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
10044 .expect("request serializes"),
10045 )
10046 .expect("describe frame builds");
10047 let response = handler
10048 .handle_server_describe(describe)
10049 .expect("server.describe succeeds");
10050 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
10051 let requirement = &rendered["capability_requirements"][0];
10052 assert_eq!(requirement["consumer"], "consumer");
10053 assert_eq!(requirement["verdict"], "never_provided");
10054 assert_eq!(requirement["episode_seq"], 1);
10055 assert_eq!(requirement["config_satisfiable"], false);
10056 assert_eq!(requirement["runtime_available"], false);
10057 assert!(requirement["detail"]
10058 .as_str()
10059 .expect("detail string")
10060 .contains("credentials-provider/v1"));
10061 }
10062
10063 fn deregister_required_provider(
10069 daemon_shutdown_begun: bool,
10070 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
10071 const CAPABILITY: &str = "project-identity/v1";
10072 let registry = Arc::new(Registry::default());
10073 let daemon_shutdown = DaemonShutdownFlag::default();
10074 let handler = ControlHandler::new(Arc::clone(®istry))
10075 .with_capability_config(
10076 [
10077 ("consumer".to_string(), true),
10078 ("provider".to_string(), true),
10079 ],
10080 BTreeMap::new(),
10081 )
10082 .with_daemon_shutdown_flag(daemon_shutdown.clone());
10083 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10084 consumer.capabilities = Some(CapabilityDeclarations {
10085 provides: Vec::new(),
10086 requires: vec![subc_protocol::manifest::CapabilityRequirement {
10087 capability: CAPABILITY.to_string(),
10088 need: subc_protocol::manifest::CapabilityNeed::Required,
10089 }],
10090 must_never_reach: Vec::new(),
10091 });
10092 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
10093 let provider_conn = ConnectionId::new(182);
10094 for (conn, manifest, corr) in [
10095 (ConnectionId::new(181), consumer, 181),
10096 (provider_conn, provider, 182),
10097 ] {
10098 let hello = handler
10099 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
10100 .expect("module registers");
10101 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10102 }
10103 assert_eq!(
10104 handler.capability_evaluator.verdict("consumer", CAPABILITY),
10105 Some(CapabilityVerdict::Provided),
10106 "the registered provider must satisfy the requirement first"
10107 );
10108 if daemon_shutdown_begun {
10109 daemon_shutdown.set_for_test();
10110 }
10111
10112 let capture = EventCapture::default();
10113 let _subscriber =
10114 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10115 handler
10116 .cleanup_connection(provider_conn)
10117 .expect("provider deregisters");
10118 let events = capture
10119 .events()
10120 .into_iter()
10121 .filter(|event| {
10122 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
10123 })
10124 .collect();
10125 (
10126 events,
10127 handler.capability_evaluator.verdict("consumer", CAPABILITY),
10128 )
10129 }
10130
10131 #[test]
10132 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
10133 let (events, verdict) = deregister_required_provider(true);
10134 assert_eq!(
10135 verdict,
10136 Some(CapabilityVerdict::NeverProvided),
10137 "shutdown must not change the verdict route.open refusals read"
10138 );
10139 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10140 let event = &events[0];
10141 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
10142 assert_eq!(
10143 event.fields.get("cause").map(String::as_str),
10144 Some("\"daemon_shutdown\""),
10145 "{event:?}"
10146 );
10147 assert_eq!(
10148 event.fields.get("verdict").map(String::as_str),
10149 Some("\"never_provided\""),
10150 "{event:?}"
10151 );
10152 }
10153
10154 #[test]
10155 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
10156 let (events, verdict) = deregister_required_provider(false);
10157 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
10158 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10159 let event = &events[0];
10160 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
10161 assert_eq!(event.fields.get("cause"), None, "{event:?}");
10162 assert_eq!(
10163 event.fields.get("verdict").map(String::as_str),
10164 Some("\"never_provided\""),
10165 "{event:?}"
10166 );
10167 }
10168
10169 #[test]
10170 fn catalog_list_omits_capabilities_for_legacy_manifest() {
10171 let registry = Arc::new(Registry::default());
10172 let handler = ControlHandler::new(Arc::clone(®istry));
10173 let hello = handler
10174 .handle_control(
10175 ConnectionId::new(101),
10176 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
10177 )
10178 .expect("legacy HELLO registers");
10179 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10180
10181 let request = Frame::build(
10182 FrameType::Request,
10183 control_flags(),
10184 0,
10185 0,
10186 102,
10187 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10188 .expect("catalog request serializes"),
10189 )
10190 .expect("catalog request frame builds");
10191 let response = handler
10192 .handle_catalog_list(request, None)
10193 .expect("catalog list succeeds");
10194 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10195 assert!(
10196 body["modules"][0].get("capabilities").is_none(),
10197 "legacy manifest must retain an absent capabilities field on catalog.list"
10198 );
10199 }
10200
10201 #[test]
10202 fn hello_ack_omits_storage_when_no_storage_config() {
10203 let registry = Arc::new(Registry::default());
10204 let handler = ControlHandler::new(Arc::clone(®istry));
10205 let responses = handler
10206 .handle_control(
10207 ConnectionId::new(1),
10208 hello_frame("aft", PROTOCOL_VERSION, 7),
10209 )
10210 .unwrap();
10211 let ack = parse_ack(&responses[0]);
10212 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
10213 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
10214 }
10215
10216 #[tokio::test]
10217 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
10218 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
10219 let registry = Arc::new(Registry::default());
10220 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
10221 let responses = handler
10222 .handle_control(
10223 ConnectionId::new(1),
10224 hello_frame("aft", PROTOCOL_VERSION, 7),
10225 )
10226 .unwrap();
10227 let ack = parse_ack(&responses[0]);
10228 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
10229
10230 let described = handler
10231 .handle_control_frame(
10232 &route_ctx(ConnectionId::new(2)).0,
10233 Frame::build(
10234 FrameType::Request,
10235 control_flags(),
10236 0,
10237 0,
10238 9,
10239 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10240 )
10241 .unwrap(),
10242 )
10243 .await
10244 .unwrap();
10245 let ClientControlResponse::ServerDescribe { machine_id, .. } =
10246 serde_json::from_slice(&described[0].body).unwrap()
10247 else {
10248 panic!("server.describe answered with another shape");
10249 };
10250 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
10251 }
10252
10253 #[test]
10254 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
10255 let registry = Arc::new(Registry::default());
10258 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
10259 crate::daemon_config::StorageConfig::Sqlite {
10260 data_home: std::path::PathBuf::from("/data"),
10261 },
10262 ));
10263
10264 let responses = handler
10265 .handle_control(
10266 ConnectionId::new(1),
10267 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
10268 )
10269 .unwrap();
10270 let ack = parse_ack(&responses[0]);
10271 assert_eq!(
10272 ack.storage,
10273 Some(serde_json::json!({
10274 "module_id": "alfonso-routing",
10275 "storage_namespace": "default",
10276 "isolation": { "kind": "module" },
10277 "backend": {
10278 "backend": "sqlite",
10279 "path": "/data/cortexkit/alfonso-routing/store.db"
10280 }
10281 })),
10282 "the delivered descriptor is the module's own sqlite store path"
10283 );
10284 }
10285
10286 #[test]
10287 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
10288 let registry = Arc::new(Registry::default());
10289 let handler = ControlHandler::new(Arc::clone(®istry));
10290 let conn = ConnectionId::new(1);
10291 let responses = handler
10292 .handle_control(
10293 conn,
10294 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10295 )
10296 .unwrap();
10297 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
10298 let registration = registry.get_module("aft").unwrap().unwrap();
10299 assert_eq!(registration.control_ops, module_baseline_control_ops());
10300
10301 let frame =
10302 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
10303 assert!(handler
10304 .guard_module_control_op(&frame, "aft", "route.bind")
10305 .unwrap()
10306 .is_none());
10307 let error = handler
10308 .guard_module_control_op(&frame, "aft", "test.synthetic")
10309 .unwrap()
10310 .expect("synthetic ungranted op should be rejected");
10311 assert_eq!(error.header.ty, FrameType::Error);
10312 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
10313 }
10314
10315 #[test]
10316 fn hello_control_ops_some_adds_optional_grants() {
10317 let registry = Arc::new(Registry::default());
10318 let handler = ControlHandler::new(Arc::clone(®istry));
10319 handler
10320 .handle_control(
10321 ConnectionId::new(1),
10322 hello_frame_with_control_ops(
10323 "aft",
10324 PROTOCOL_VERSION,
10325 7,
10326 Some(vec![
10327 "future.synthetic".to_string(),
10328 "route.bind".to_string(),
10329 ]),
10330 ),
10331 )
10332 .unwrap();
10333 let registration = registry.get_module("aft").unwrap().unwrap();
10334 assert_eq!(
10335 registration.control_ops,
10336 vec![
10337 "route.bind".to_string(),
10338 "route.status".to_string(),
10339 "future.synthetic".to_string(),
10340 ]
10341 );
10342 let frame =
10343 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
10344 assert!(handler
10345 .guard_module_control_op(&frame, "aft", "future.synthetic")
10346 .unwrap()
10347 .is_none());
10348 }
10349
10350 #[tokio::test]
10351 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
10352 let registry = Arc::new(Registry::default());
10353 let forwarding = Arc::new(ForwardingTable::default());
10354 let handler =
10355 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10356 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
10357 hello_via_sink(
10358 &handler,
10359 &module_ctx,
10360 &mut module_rx,
10361 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10362 )
10363 .await;
10364
10365 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
10366 let responses = handler
10367 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
10368 .await
10369 .unwrap();
10370 assert_eq!(responses.len(), 1);
10371 assert_eq!(responses[0].header.ty, FrameType::Error);
10372 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
10373 assert!(module_rx.try_recv().is_err());
10374 }
10375
10376 #[tokio::test]
10377 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
10378 let registry = Arc::new(Registry::default());
10379 let forwarding = Arc::new(ForwardingTable::default());
10380 let handler =
10381 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10382 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
10383 hello_via_sink(
10384 &handler,
10385 &module_ctx,
10386 &mut module_rx,
10387 hello_frame_with_control_ops(
10388 "aft",
10389 PROTOCOL_VERSION,
10390 7,
10391 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10392 ),
10393 )
10394 .await;
10395
10396 let project_root = unique_project_root("demux");
10397 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
10398 let route_handler = handler.clone();
10399 let route_task = tokio::spawn(async move {
10400 route_handler
10401 .handle_control_frame(
10402 &route_client_ctx,
10403 route_open_frame(100, "aft", project_root),
10404 )
10405 .await
10406 .unwrap()
10407 });
10408 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10409 .await
10410 .unwrap()
10411 .unwrap();
10412 assert!(matches!(
10413 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
10414 ModuleControlRequest::RouteBind { .. }
10415 ));
10416
10417 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
10418 let health_handler = handler.clone();
10419 let health_task = tokio::spawn(async move {
10420 health_handler
10421 .handle_control_frame(
10422 &health_client_ctx,
10423 supervisor_health_probe_frame(101, "aft"),
10424 )
10425 .await
10426 .unwrap()
10427 });
10428 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10429 .await
10430 .unwrap()
10431 .unwrap();
10432 assert_eq!(
10433 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10434 ModuleControlRequest::HealthCheck {}
10435 );
10436
10437 handler
10438 .handle_control_frame(
10439 &module_ctx,
10440 health_response(health_frame.header.corr, HealthStatus::Degraded),
10441 )
10442 .await
10443 .unwrap();
10444 let health_response = health_task.await.unwrap();
10445 assert_eq!(health_response.len(), 1);
10446 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10447 ClientControlResponse::SupervisorHealthProbe {
10448 module_id,
10449 status,
10450 detail,
10451 metrics,
10452 } => {
10453 assert_eq!(module_id, "aft");
10454 assert_eq!(status, HealthStatus::Degraded);
10455 assert_eq!(detail.as_deref(), Some("warming"));
10456 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
10457 }
10458 other => panic!("unexpected health response: {other:?}"),
10459 }
10460
10461 handler
10462 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10463 .await
10464 .unwrap();
10465 let route_response = route_task.await.unwrap();
10466 assert!(route_response.is_empty());
10467 let published = route_client_rx.recv().await.unwrap();
10468 assert!(matches!(
10469 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10470 ClientControlResponse::RouteOpen { .. }
10471 ));
10472 }
10473
10474 async fn relay_route_open(
10479 handler: &ControlHandler,
10480 client_connection: ConnectionId,
10481 client_egress: &FrameSink,
10482 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10483 corr: u64,
10484 module_id: &str,
10485 project_root_label: &str,
10486 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
10487 let ctx = RouteCtx {
10488 connection_id: client_connection,
10489 egress: client_egress.clone(),
10490 };
10491 let handler = handler.clone();
10492 let project_root = unique_project_root(project_root_label);
10493 let module_id = module_id.to_string();
10494 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
10495 let task = tokio::spawn(async move {
10496 let _guard = tracing::dispatcher::set_default(&dispatch);
10497 handler
10498 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
10499 .await
10500 .unwrap()
10501 });
10502 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
10503 .await
10504 .expect("module receives the relayed route.bind")
10505 .expect("module egress is open");
10506 (task, bind.frame)
10507 }
10508
10509 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
10510 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
10511 ModuleControlRequest::RouteBind {
10512 route_channel,
10513 epoch,
10514 ..
10515 } => (route_channel, epoch),
10516 other => panic!("expected a route.bind request, got {other:?}"),
10517 }
10518 }
10519
10520 fn published_route(frame: &Frame) -> (u16, u32) {
10521 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
10522 ClientControlResponse::RouteOpen {
10523 route_channel,
10524 route_epoch,
10525 } => (route_channel, route_epoch),
10526 other => panic!("expected a route.open response, got {other:?}"),
10527 }
10528 }
10529
10530 #[tokio::test]
10550 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
10551 let registry = Arc::new(Registry::default());
10552 let forwarding = Arc::new(ForwardingTable::default());
10553 let handler =
10554 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10555
10556 let module_connection = ConnectionId::new(30);
10557 let (module_ctx, mut module_rx) = route_ctx(module_connection);
10558 hello_via_sink(
10559 &handler,
10560 &module_ctx,
10561 &mut module_rx,
10562 hello_frame("aft", PROTOCOL_VERSION, 7),
10563 )
10564 .await;
10565
10566 let dying_client = ConnectionId::new(31);
10567 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
10568
10569 let (first_task, first_bind) = relay_route_open(
10572 &handler,
10573 dying_client,
10574 &dying_ctx.egress,
10575 &mut module_rx,
10576 100,
10577 "aft",
10578 "closing-first",
10579 )
10580 .await;
10581 handler
10582 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
10583 .await
10584 .unwrap();
10585 assert!(first_task.await.unwrap().is_empty());
10586 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
10587
10588 let (second_task, second_bind) = relay_route_open(
10590 &handler,
10591 dying_client,
10592 &dying_ctx.egress,
10593 &mut module_rx,
10594 101,
10595 "aft",
10596 "closing-second",
10597 )
10598 .await;
10599 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
10600
10601 assert!(forwarding
10604 .escalate_client_delivery_failure(
10605 dying_client,
10606 first_channel,
10607 first_epoch,
10608 CloseReason::new(
10609 "module_to_client_delivery_failed",
10610 "client egress refused a module frame",
10611 ),
10612 crate::forwarding::UndeliveredFrame {
10613 module_id: None,
10614 sink: &dying_ctx.egress,
10615 },
10616 )
10617 .unwrap());
10618 assert!(!dying_ctx.egress.is_closed());
10619
10620 let ack = handler
10622 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
10623 .await;
10624 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10625 if module_loop_error.is_some() {
10626 handler.cleanup_connection(module_connection).unwrap();
10630 }
10631 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10635 .await
10636 .ok()
10637 .flatten();
10638
10639 assert!(
10641 registry
10642 .get_module_by_connection(module_connection)
10643 .unwrap()
10644 .is_some(),
10645 "one client's closing connection ended the shared module connection: \
10646 {module_loop_error:?}"
10647 );
10648 let cotenant = ConnectionId::new(32);
10650 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10651 let (cotenant_task, cotenant_bind) = relay_route_open(
10652 &handler,
10653 cotenant,
10654 &cotenant_ctx.egress,
10655 &mut module_rx,
10656 102,
10657 "aft",
10658 "closing-cotenant",
10659 )
10660 .await;
10661 handler
10662 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
10663 .await
10664 .unwrap();
10665 assert!(cotenant_task.await.unwrap().is_empty());
10666 let (cotenant_channel, cotenant_epoch) =
10667 published_route(&cotenant_rx.recv().await.unwrap());
10668 assert!(matches!(
10669 forwarding
10670 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10671 .unwrap(),
10672 DataRoute::Client(DataRouteState::Bound(_))
10673 ));
10674
10675 let goodbye = post_ack_module_frame
10678 .expect("module receives a GOODBYE for the abandoned route channel");
10679 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10680 assert_eq!(goodbye.header.channel, abandoned_channel);
10681 assert_eq!(goodbye.header.epoch, abandoned_epoch);
10682
10683 assert!(dying_rx.try_recv().is_err());
10687 let second_response = second_task.await.unwrap();
10688 assert_eq!(second_response.len(), 1);
10689 assert_eq!(
10690 parse_error(&second_response[0])["code"],
10691 "target_unavailable"
10692 );
10693 }
10694
10695 async fn catalog_snapshot(
10698 handler: &ControlHandler,
10699 client_ctx: &RouteCtx,
10700 corr: u64,
10701 ) -> (u64, Vec<(String, bool)>) {
10702 let body =
10703 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
10704 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
10705 let reply = handler
10706 .handle_control_frame(client_ctx, request)
10707 .await
10708 .unwrap()
10709 .pop()
10710 .unwrap();
10711 let ClientControlResponse::CatalogList {
10712 generation,
10713 modules,
10714 ..
10715 } = serde_json::from_slice(&reply.body).unwrap()
10716 else {
10717 panic!("not a catalog.list reply");
10718 };
10719 (
10720 generation,
10721 modules
10722 .into_iter()
10723 .map(|entry| (entry.module_id, entry.ready))
10724 .collect(),
10725 )
10726 }
10727
10728 fn hello_held_before_forwarding() -> (
10734 ControlHandler,
10735 Arc<std::sync::Barrier>,
10736 std::thread::JoinHandle<Vec<Frame>>,
10737 RouteCtx,
10738 mpsc::Receiver<crate::router::OutboundFrame>,
10739 ) {
10740 let hold = Arc::new(std::sync::Barrier::new(2));
10741 let handler = ControlHandler::with_forwarding(
10742 Arc::new(Registry::default()),
10743 Arc::new(ForwardingTable::default()),
10744 )
10745 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
10746 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10747 let hello = {
10748 let handler = handler.clone();
10749 let module_ctx = module_ctx.clone();
10750 std::thread::spawn(move || {
10751 handler
10752 .handle_hello(
10753 module_ctx.connection_id,
10754 Some(module_ctx.egress.clone()),
10755 hello_frame("aft", PROTOCOL_VERSION, 7),
10756 )
10757 .unwrap()
10758 })
10759 };
10760 (handler, hold, hello, module_ctx, module_rx)
10761 }
10762
10763 #[tokio::test]
10772 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
10773 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
10774 let client = ConnectionId::new(41);
10775 let (client_ctx, mut client_rx) = route_ctx(client);
10776
10777 hold.wait();
10778 assert!(handler.registry.get_module("aft").unwrap().is_some());
10780 assert!(!handler
10781 .forwarding
10782 .has_live_module_connection("aft")
10783 .unwrap());
10784 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
10785 assert!(
10786 listed.is_empty(),
10787 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
10788 );
10789 let refused = handler
10790 .handle_control_frame(
10791 &client_ctx,
10792 route_open_frame(301, "aft", unique_project_root("listed-window")),
10793 )
10794 .await
10795 .unwrap();
10796 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
10797 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
10798 let duplicate = handler
10799 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
10800 .await
10801 .unwrap();
10802 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10803
10804 hold.wait();
10805 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10806 let ack = module_rx.try_recv().unwrap().frame;
10807 assert_eq!(ack.header.ty, FrameType::HelloAck);
10808 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10809 assert_eq!(listed, vec![("aft".to_string(), true)]);
10810
10811 let (open_task, bind) = relay_route_open(
10812 &handler,
10813 client,
10814 &client_ctx.egress,
10815 &mut module_rx,
10816 303,
10817 "aft",
10818 "listed-after",
10819 )
10820 .await;
10821 assert!(handler
10822 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10823 .await
10824 .unwrap()
10825 .is_empty());
10826 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
10827 published_route(&client_rx.try_recv().unwrap().frame);
10828 }
10829
10830 #[tokio::test]
10837 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
10838 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
10839 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10840 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10841
10842 hold.wait();
10843 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
10844 assert!(listed.is_empty());
10845 assert_eq!(
10846 during, before,
10847 "the generation moved while the module was not listed"
10848 );
10849
10850 hold.wait();
10851 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10852 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10853 assert_eq!(listed, vec![("aft".to_string(), true)]);
10854 assert_eq!(
10855 after,
10856 before + 1,
10857 "the module appeared without exactly one generation bump"
10858 );
10859 }
10860
10861 #[tokio::test]
10866 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
10867 let registry = Arc::new(Registry::default());
10868 let handler = ControlHandler::with_forwarding(
10869 Arc::clone(®istry),
10870 Arc::new(ForwardingTable::default()),
10871 );
10872 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10873 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10874 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10875 drop(module_rx);
10876
10877 let replies = handler
10878 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
10879 .await
10880 .unwrap();
10881 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
10882 assert!(registry.get_module("aft").unwrap().is_none());
10883 assert!(registry
10884 .get_module_by_connection(module_ctx.connection_id)
10885 .unwrap()
10886 .is_none());
10887 assert_eq!(
10888 catalog_snapshot(&handler, &client_ctx, 301).await,
10889 (before, Vec::new())
10890 );
10891 }
10892
10893 async fn route_open_held_after_bind_send() -> (
10897 ControlHandler,
10898 Arc<tokio::sync::Barrier>,
10899 RouteCtx,
10900 mpsc::Receiver<crate::router::OutboundFrame>,
10901 ) {
10902 let hold = Arc::new(tokio::sync::Barrier::new(2));
10903 let handler = ControlHandler::with_forwarding(
10904 Arc::new(Registry::default()),
10905 Arc::new(ForwardingTable::default()),
10906 )
10907 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
10908 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10909 hello_via_sink(
10910 &handler,
10911 &module_ctx,
10912 &mut module_rx,
10913 hello_frame("aft", PROTOCOL_VERSION, 7),
10914 )
10915 .await;
10916 (handler, hold, module_ctx, module_rx)
10917 }
10918
10919 #[tokio::test]
10926 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
10927 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10928 let client = ConnectionId::new(41);
10929 let (client_ctx, mut client_rx) = route_ctx(client);
10930
10931 let (open_task, bind) = relay_route_open(
10932 &handler,
10933 client,
10934 &client_ctx.egress,
10935 &mut module_rx,
10936 200,
10937 "aft",
10938 "acked-before-mark",
10939 )
10940 .await;
10941 let (module_channel, module_epoch) = route_bind_channel(&bind);
10942 hold.wait().await;
10944
10945 assert!(handler
10947 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10948 .await
10949 .unwrap()
10950 .is_empty());
10951 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
10952
10953 hold.wait().await;
10954 assert!(
10955 open_task.await.unwrap().is_empty(),
10956 "route.open answers accepted"
10957 );
10958
10959 match module_rx.try_recv() {
10962 Err(mpsc::error::TryRecvError::Empty) => {}
10963 Ok(outbound) => panic!(
10964 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
10965 outbound.frame.header.ty,
10966 outbound.frame.header.channel,
10967 outbound.frame.header.epoch
10968 ),
10969 Err(err) => panic!("module egress closed: {err}"),
10970 }
10971
10972 let router = Router::with_control_handler(Arc::new(handler.clone()));
10975 let request = Frame::build(
10976 FrameType::Request,
10977 Flags::new(true, Priority::Interactive, false),
10978 client_channel,
10979 client_epoch,
10980 201,
10981 b"first request".to_vec(),
10982 )
10983 .unwrap();
10984 router
10985 .route_for_connection(&client_ctx, request)
10986 .await
10987 .unwrap();
10988 let delivered = module_rx.try_recv().unwrap().frame;
10989 assert_eq!(delivered.header.ty, FrameType::Request);
10990 assert_eq!(
10991 (delivered.header.channel, delivered.header.epoch),
10992 (module_channel, module_epoch)
10993 );
10994 assert_eq!(delivered.body, b"first request");
10995 }
10996
10997 #[tokio::test]
11003 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
11004 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11005 let client = ConnectionId::new(42);
11006 let (client_ctx, _client_rx) = route_ctx(client);
11007
11008 let (open_task, bind) = relay_route_open(
11009 &handler,
11010 client,
11011 &client_ctx.egress,
11012 &mut module_rx,
11013 300,
11014 "aft",
11015 "released-before-mark",
11016 )
11017 .await;
11018 let (module_channel, module_epoch) = route_bind_channel(&bind);
11019 hold.wait().await;
11020
11021 handler.cleanup_connection(client).unwrap();
11022 assert!(
11023 module_rx.try_recv().is_err(),
11024 "client teardown must not GOODBYE a bind route.open is still queueing"
11025 );
11026
11027 hold.wait().await;
11028 let response = open_task.await.unwrap();
11029 assert_eq!(response.len(), 1);
11030 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11031
11032 let goodbye = module_rx
11033 .try_recv()
11034 .expect("module is told to drop the binding the daemon abandoned")
11035 .frame;
11036 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11037 assert_eq!(
11038 (goodbye.header.channel, goodbye.header.epoch),
11039 (module_channel, module_epoch)
11040 );
11041 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11042 }
11043
11044 #[tokio::test]
11049 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
11050 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11051 let client = ConnectionId::new(43);
11052 let (client_ctx, client_rx) = route_ctx(client);
11053
11054 let (open_task, bind) = relay_route_open(
11055 &handler,
11056 client,
11057 &client_ctx.egress,
11058 &mut module_rx,
11059 400,
11060 "aft",
11061 "gone-client-before-mark",
11062 )
11063 .await;
11064 let (module_channel, module_epoch) = route_bind_channel(&bind);
11065 hold.wait().await;
11066
11067 drop(client_rx);
11068 assert!(client_ctx.egress.is_closed());
11069 handler
11070 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11071 .await
11072 .unwrap();
11073
11074 hold.wait().await;
11075 let response = open_task.await.unwrap();
11076 assert_eq!(response.len(), 1);
11077 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11078
11079 let goodbye = module_rx
11080 .try_recv()
11081 .expect("module is told to drop the binding it acked")
11082 .frame;
11083 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11084 assert_eq!(
11085 (goodbye.header.channel, goodbye.header.epoch),
11086 (module_channel, module_epoch)
11087 );
11088 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11089 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11090 }
11091
11092 #[test]
11099 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
11100 let handler = ControlHandler::default();
11101 let module_connection = ConnectionId::new(30);
11102 let client_connection = ConnectionId::new(31);
11103
11104 handler
11105 .refuse_to_end_module_connection_for_a_client(
11106 module_connection,
11107 77,
11108 ForwardingError::ConnectionClosing {
11109 connection_id: client_connection,
11110 },
11111 )
11112 .expect("a closing client must never end the module connection");
11113
11114 assert!(matches!(
11115 handler.refuse_to_end_module_connection_for_a_client(
11116 module_connection,
11117 78,
11118 ForwardingError::ConnectionClosing {
11119 connection_id: module_connection,
11120 },
11121 ),
11122 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
11123 connection_id
11124 })) if connection_id == module_connection
11125 ));
11126 assert!(matches!(
11127 handler.refuse_to_end_module_connection_for_a_client(
11128 module_connection,
11129 79,
11130 ForwardingError::Poisoned,
11131 ),
11132 Err(RouterError::Forwarding(ForwardingError::Poisoned))
11133 ));
11134 assert!(matches!(
11135 handler.refuse_to_end_module_connection_for_a_client(
11136 module_connection,
11137 80,
11138 ForwardingError::StaleModuleEndpoint,
11139 ),
11140 Err(RouterError::Forwarding(
11141 ForwardingError::StaleModuleEndpoint
11142 ))
11143 ));
11144 }
11145
11146 #[tokio::test]
11159 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
11160 let registry = Arc::new(Registry::default());
11161 let forwarding = Arc::new(ForwardingTable::default());
11162 let supervisor = SupervisorHandle::new();
11163 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11164 let handler =
11165 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11166 .with_supervisor(supervisor);
11167
11168 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11169 hello_via_sink(
11170 &handler,
11171 &target_ctx,
11172 &mut target_rx,
11173 hello_frame("target", PROTOCOL_VERSION, 1),
11174 )
11175 .await;
11176
11177 let wrong_nonce = handler
11181 .handle_control_frame(
11182 &route_ctx(ConnectionId::new(91)).0,
11183 route_open_frame_with_admission_facts(
11184 20,
11185 "target",
11186 unique_project_root("admission-facts"),
11187 Some(subc_control::ConsumerIdentity {
11188 module_id: "fed".to_string(),
11189 launch_nonce: "not-the-real-nonce".to_string(),
11190 }),
11191 None,
11192 ),
11193 )
11194 .await
11195 .unwrap();
11196 assert_eq!(
11197 parse_error(&wrong_nonce[0])["code"],
11198 "bad_consumer_identity",
11199 "a mismatched launch nonce must be refused, not stamped Reserved"
11200 );
11201
11202 let never_spawned = handler
11206 .handle_control_frame(
11207 &route_ctx(ConnectionId::new(92)).0,
11208 route_open_frame_with_admission_facts(
11209 21,
11210 "target",
11211 unique_project_root("admission-facts"),
11212 Some(subc_control::ConsumerIdentity {
11213 module_id: "never-spawned".to_string(),
11214 launch_nonce: "any-nonce".to_string(),
11215 }),
11216 None,
11217 ),
11218 )
11219 .await
11220 .unwrap();
11221 assert_eq!(
11222 parse_error(&never_spawned[0])["code"],
11223 "bad_consumer_identity",
11224 "an unspawned module_id must be refused rather than accepted for lack of a record"
11225 );
11226 }
11227
11228 #[tokio::test]
11251 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
11252 let registry = Arc::new(Registry::default());
11253 let forwarding = Arc::new(ForwardingTable::default());
11254 let supervisor = SupervisorHandle::new();
11255 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11256 let handler =
11257 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11258 .with_supervisor(supervisor);
11259
11260 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
11261 hello_via_sink(
11262 &handler,
11263 &target_ctx,
11264 &mut target_rx,
11265 hello_frame("target", PROTOCOL_VERSION, 1),
11266 )
11267 .await;
11268
11269 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
11270 let route_handler = handler.clone();
11271 let route_task = tokio::spawn(async move {
11272 route_handler
11273 .handle_control_frame(
11274 &client_ctx,
11275 route_open_frame_with_admission_facts(
11276 30,
11277 "target",
11278 unique_project_root("admission-facts"),
11279 Some(subc_control::ConsumerIdentity {
11280 module_id: "fed".to_string(),
11281 launch_nonce: "fed-nonce".to_string(),
11282 }),
11283 None,
11284 ),
11285 )
11286 .await
11287 .unwrap()
11288 });
11289
11290 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11297 .await
11298 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
11299 .expect("module control channel closed before route.bind");
11300 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11301 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11302 panic!("expected route.bind")
11303 };
11304 assert_eq!(
11305 principal,
11306 Some(Principal::Reserved {
11307 module_id: "fed".to_string()
11308 }),
11309 "a correctly attested consumer must be stamped Reserved for its own id"
11310 );
11311
11312 handler
11313 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11314 .await
11315 .unwrap();
11316 assert!(route_task.await.unwrap().is_empty());
11317 assert!(
11318 matches!(
11319 serde_json::from_slice::<ClientControlResponse>(
11320 &client_rx.recv().await.unwrap().body
11321 )
11322 .unwrap(),
11323 ClientControlResponse::RouteOpen { .. }
11324 ),
11325 "the route must actually open, not merely avoid an error"
11326 );
11327 }
11328
11329 #[tokio::test(start_paused = true)]
11330 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
11331 let registry = Arc::new(Registry::default());
11332 let forwarding = Arc::new(ForwardingTable::default());
11333 let supervisor = SupervisorHandle::new();
11334 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11335 let handler =
11336 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11337 .with_supervisor(supervisor);
11338
11339 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
11340 hello_via_sink(
11341 &handler,
11342 &target_ctx,
11343 &mut target_rx,
11344 hello_frame("target", PROTOCOL_VERSION, 1),
11345 )
11346 .await;
11347
11348 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
11349 let direct_handler = handler.clone();
11350 let direct_open = tokio::spawn(async move {
11351 direct_handler
11352 .handle_control_frame(
11353 &direct_ctx,
11354 route_open_frame(2, "target", unique_project_root("route-census-direct")),
11355 )
11356 .await
11357 .unwrap()
11358 });
11359 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11360 .await
11361 .expect("no direct route.bind within 5s")
11362 .expect("target control channel closed before direct route.bind");
11363 handler
11364 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
11365 .await
11366 .unwrap();
11367 assert!(direct_open.await.unwrap().is_empty());
11368 let _ = direct_rx.recv().await.unwrap();
11369
11370 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
11371 let reserved_handler = handler.clone();
11372 let reserved_open = tokio::spawn(async move {
11373 reserved_handler
11374 .handle_control_frame(
11375 &reserved_ctx,
11376 route_open_frame_with_admission_facts(
11377 3,
11378 "target",
11379 unique_project_root("admission-facts"),
11380 Some(ConsumerIdentity {
11381 module_id: "fed".to_string(),
11382 launch_nonce: "fed-nonce".to_string(),
11383 }),
11384 None,
11385 ),
11386 )
11387 .await
11388 .unwrap()
11389 });
11390 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11391 .await
11392 .expect("no reserved route.bind within 5s")
11393 .expect("target control channel closed before reserved route.bind");
11394 handler
11395 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
11396 .await
11397 .unwrap();
11398 assert!(reserved_open.await.unwrap().is_empty());
11399 let _ = reserved_rx.recv().await.unwrap();
11400
11401 forwarding
11402 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
11403 .unwrap();
11404 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
11405 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
11406 module_id: Some("target".to_string()),
11407 })
11408 .unwrap();
11409 let census_frame =
11410 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
11411 let response = handler
11412 .handle_control_frame(&census_ctx, census_frame)
11413 .await
11414 .unwrap()
11415 .pop()
11416 .unwrap();
11417 let actual: Value = serde_json::from_slice(&response.body).unwrap();
11418 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
11419 assert!(matches!(
11420 decoded,
11421 ClientControlResponse::SupervisorRoutes { .. }
11422 ));
11423 let routes = actual["modules"][0]["routes"].as_array().unwrap();
11424 assert_eq!(routes.len(), 2);
11425 assert!(routes.iter().all(|route| route["draining"] == true));
11426 assert!(
11429 routes.iter().all(|route| route["drain_reason"] == "reload"),
11430 "draining routes must name the drain's reason: {routes:?}"
11431 );
11432 assert!(routes.iter().any(|route| {
11433 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
11434 }));
11435 assert!(routes.iter().any(|route| {
11436 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
11437 }));
11438
11439 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11440 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
11441 if std::env::var_os("UPDATE_GOLDEN").is_some() {
11442 std::fs::write(
11443 &golden_path,
11444 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
11445 )
11446 .unwrap();
11447 }
11448 let expected: Value =
11449 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
11450 assert_eq!(actual, expected);
11451 }
11452
11453 async fn query_live_roots(
11454 handler: &ControlHandler,
11455 module_ctx: &RouteCtx,
11456 ) -> ModuleControlResponseToModule {
11457 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
11458 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
11459 let response = handler
11460 .handle_control_frame(module_ctx, frame)
11461 .await
11462 .unwrap()
11463 .pop()
11464 .unwrap();
11465 serde_json::from_slice(&response.body).unwrap()
11466 }
11467
11468 #[tokio::test(start_paused = true)]
11469 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
11470 let registry = Arc::new(Registry::default());
11471 let forwarding = Arc::new(ForwardingTable::default());
11472 let handler = ControlHandler::with_forwarding(registry, forwarding);
11473 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
11474 hello_via_sink(
11475 &handler,
11476 &target_ctx,
11477 &mut target_rx,
11478 hello_frame("target", PROTOCOL_VERSION, 1),
11479 )
11480 .await;
11481 let root = unique_project_root("live-roots-known");
11482 let path = ProjectRootId::from_path_allowing_missing(root.path())
11483 .unwrap()
11484 .as_path()
11485 .to_path_buf();
11486 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
11487 let open_handler = handler.clone();
11488 let opened = tokio::spawn(async move {
11489 open_handler
11490 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
11491 .await
11492 .unwrap()
11493 });
11494 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11495 .await
11496 .unwrap()
11497 .unwrap();
11498 handler
11499 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
11500 .await
11501 .unwrap();
11502 assert!(opened.await.unwrap().is_empty());
11503 let _ = client_rx.recv().await.unwrap();
11504
11505 let root = unique_project_root("live-roots-pending");
11506 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
11507 .unwrap()
11508 .as_path()
11509 .to_path_buf();
11510 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
11511 let open_handler = handler.clone();
11512 let pending = tokio::spawn(async move {
11513 open_handler
11514 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
11515 .await
11516 .unwrap()
11517 });
11518 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11519 .await
11520 .unwrap()
11521 .unwrap();
11522 let actual = query_live_roots(&handler, &target_ctx).await;
11523 let ModuleControlResponseToModule::LiveRoots {
11524 roots,
11525 unknown_root_bindings,
11526 total_bindings,
11527 } = actual
11528 else {
11529 panic!("expected live roots")
11530 };
11531 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
11532 assert_eq!(unknown_root_bindings, 0);
11533 assert_eq!(
11534 roots.len(),
11535 2,
11536 "root-known arm must retain each canonical root"
11537 );
11538 assert_eq!(
11539 total_bindings,
11540 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11541 );
11542 let counts = roots
11543 .iter()
11544 .map(|root| (root.project_root.clone(), root.bound, root.pending))
11545 .collect::<Vec<_>>();
11546 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
11547 expected.sort_by(|a, b| a.0.cmp(&b.0));
11548 assert_eq!(
11549 counts, expected,
11550 "roots must sort by path and count pending separately"
11551 );
11552 handler
11553 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
11554 .await
11555 .unwrap();
11556 assert!(pending.await.unwrap().is_empty());
11557 }
11558
11559 #[tokio::test(start_paused = true)]
11560 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
11561 let forwarding = Arc::new(ForwardingTable::default());
11562 let handler =
11563 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11564 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
11565 hello_via_sink(
11566 &handler,
11567 &target_ctx,
11568 &mut target_rx,
11569 hello_frame("target", PROTOCOL_VERSION, 1),
11570 )
11571 .await;
11572 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
11573 let pending = forwarding
11574 .begin_route_bind_relay_for_test(
11575 client_ctx.connection_id,
11576 client_ctx.egress.clone(),
11577 2,
11578 "target",
11579 )
11580 .unwrap();
11581 forwarding
11582 .complete_pending_relay(
11583 target_ctx.connection_id,
11584 pending.corr,
11585 RouteBindRelayOutcome::Accepted,
11586 )
11587 .unwrap();
11588 let actual = query_live_roots(&handler, &target_ctx).await;
11589 let ModuleControlResponseToModule::LiveRoots {
11590 roots,
11591 unknown_root_bindings,
11592 total_bindings,
11593 } = actual
11594 else {
11595 panic!("expected live roots")
11596 };
11597 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
11598 assert_eq!(
11599 unknown_root_bindings, 1,
11600 "unknown-root arm must not read as no bindings"
11601 );
11602 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
11603 assert_eq!(
11604 total_bindings,
11605 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11606 );
11607 }
11608
11609 #[tokio::test(start_paused = true)]
11617 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
11618 let forwarding = Arc::new(ForwardingTable::default());
11619 let handler =
11620 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11621 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
11622 let replies = handler
11623 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
11624 .await
11625 .unwrap();
11626 let queued_by_hello = module_rx.len();
11627
11628 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
11629 let open_handler = handler.clone();
11630 let open = tokio::spawn(async move {
11631 open_handler
11632 .handle_control_frame(
11633 &client_ctx,
11634 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
11635 )
11636 .await
11637 .unwrap()
11638 });
11639 let mut spins = 0;
11641 while module_rx.len() == queued_by_hello {
11642 spins += 1;
11643 assert!(spins < 10_000, "route.open never queued a route.bind");
11644 tokio::task::yield_now().await;
11645 }
11646
11647 for reply in replies {
11649 module_ctx.egress.send(reply).await.unwrap();
11650 }
11651
11652 let first = module_rx.recv().await.unwrap().frame;
11653 assert_eq!(
11654 first.header.ty,
11655 FrameType::HelloAck,
11656 "the first frame a registering module reads must be its HELLO_ACK"
11657 );
11658 assert_eq!(first.header.corr, 7);
11659 let second = module_rx.recv().await.unwrap().frame;
11660 assert_eq!(second.header.ty, FrameType::Request);
11661 assert!(
11662 matches!(
11663 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
11664 ModuleControlRequest::RouteBind { .. }
11665 ),
11666 "the route.bind follows the ack"
11667 );
11668 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
11669
11670 handler
11671 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
11672 .await
11673 .unwrap();
11674 assert!(open.await.unwrap().is_empty());
11675 let _ = client_rx.recv().await.unwrap();
11676 }
11677
11678 #[tokio::test(start_paused = true)]
11679 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
11680 let handler = ControlHandler::with_forwarding(
11681 Arc::new(Registry::default()),
11682 Arc::new(ForwardingTable::default()),
11683 );
11684 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
11685 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
11686 hello_via_sink(
11687 &handler,
11688 &first_ctx,
11689 &mut first_rx,
11690 hello_frame("first", PROTOCOL_VERSION, 1),
11691 )
11692 .await;
11693 hello_via_sink(
11694 &handler,
11695 &second_ctx,
11696 &mut second_rx,
11697 hello_frame("second", PROTOCOL_VERSION, 2),
11698 )
11699 .await;
11700 let root = unique_project_root("second-only");
11701 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
11702 let cloned = handler.clone();
11703 let open = tokio::spawn(async move {
11704 cloned
11705 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
11706 .await
11707 .unwrap()
11708 });
11709 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
11710 .await
11711 .unwrap()
11712 .unwrap();
11713 let first = query_live_roots(&handler, &first_ctx).await;
11714 let second = query_live_roots(&handler, &second_ctx).await;
11715 assert!(
11716 matches!(
11717 first,
11718 ModuleControlResponseToModule::LiveRoots {
11719 total_bindings: 0,
11720 ..
11721 }
11722 ),
11723 "cross-module scope must not expose another module's roots"
11724 );
11725 assert!(
11726 matches!(
11727 second,
11728 ModuleControlResponseToModule::LiveRoots {
11729 total_bindings: 1,
11730 ..
11731 }
11732 ),
11733 "second module must see its pending route"
11734 );
11735 handler
11736 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
11737 .await
11738 .unwrap();
11739 assert!(open.await.unwrap().is_empty());
11740 }
11741
11742 #[tokio::test(start_paused = true)]
11743 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
11744 let handler = ControlHandler::with_forwarding(
11745 Arc::new(Registry::default()),
11746 Arc::new(ForwardingTable::default()),
11747 );
11748 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
11749 hello_via_sink(
11750 &handler,
11751 &target_ctx,
11752 &mut target_rx,
11753 hello_frame("target", PROTOCOL_VERSION, 1),
11754 )
11755 .await;
11756 let actual = query_live_roots(&handler, &target_ctx).await;
11757 let ModuleControlResponseToModule::LiveRoots {
11758 roots,
11759 unknown_root_bindings,
11760 total_bindings,
11761 } = actual
11762 else {
11763 panic!("expected live roots")
11764 };
11765 assert!(roots.is_empty());
11766 assert_eq!(unknown_root_bindings, 0);
11767 assert_eq!(total_bindings, 0);
11768 assert_eq!(
11769 total_bindings,
11770 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11771 );
11772 }
11773
11774 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
11781 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11782 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
11783 let text = std::fs::read_to_string(&path)
11784 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
11785 let vectors: Vec<(String, Value)> = text
11786 .lines()
11787 .filter(|line| !line.trim().is_empty())
11788 .map(|line| {
11789 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
11790 let id = entry["corpus_id"]
11791 .as_str()
11792 .expect("every vector carries a corpus_id")
11793 .to_string();
11794 (id, entry["package"].clone())
11795 })
11796 .collect();
11797 assert_eq!(
11801 vectors.len(),
11802 3,
11803 "vendored fed corpus changed size; re-sync from subc-federation"
11804 );
11805
11806 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
11821 let richest = vectors
11822 .iter()
11823 .filter_map(|(_, package)| package.as_object())
11824 .map(|object| {
11825 object
11826 .keys()
11827 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
11828 .count()
11829 })
11830 .max()
11831 .unwrap_or(0);
11832 assert!(
11833 richest >= 2,
11834 "vendored corpus no longer carries a package with unmodelled fields, \
11835 so the relay test can no longer distinguish a verbatim relay from a lossy one"
11836 );
11837
11838 vectors
11839 }
11840
11841 #[tokio::test]
11849 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
11850 for (corpus_id, package) in fed_admission_facts_vectors() {
11851 let registry = Arc::new(Registry::default());
11852 let forwarding = Arc::new(ForwardingTable::default());
11853 let supervisor = SupervisorHandle::new();
11854 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11855 let handler =
11856 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11857 .with_supervisor(supervisor)
11858 .with_admission_facts_config(
11859 Some("fed".to_string()),
11860 Some(vec!["target".to_string()]),
11861 );
11862
11863 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11864 hello_via_sink(
11865 &handler,
11866 &target_ctx,
11867 &mut target_rx,
11868 hello_frame("target", PROTOCOL_VERSION, 1),
11869 )
11870 .await;
11871
11872 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
11873 let route_handler = handler.clone();
11874 let expected = package.clone();
11875 let route_task = tokio::spawn(async move {
11876 route_handler
11877 .handle_control_frame(
11878 &client_ctx,
11879 route_open_frame_with_admission_facts(
11880 20,
11881 "target",
11882 unique_project_root("admission-facts"),
11883 Some(subc_control::ConsumerIdentity {
11884 module_id: "fed".to_string(),
11885 launch_nonce: "fed-nonce".to_string(),
11886 }),
11887 Some(package),
11888 ),
11889 )
11890 .await
11891 .unwrap()
11892 });
11893
11894 let bind_frame = target_rx.recv().await.unwrap();
11895 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11896 let ModuleControlRequest::RouteBind {
11897 admission_facts, ..
11898 } = bind
11899 else {
11900 panic!("{corpus_id}: expected route.bind")
11901 };
11902 assert_eq!(
11903 admission_facts,
11904 Some(expected),
11905 "{corpus_id}: relay must not add, drop or reshape any field"
11906 );
11907
11908 handler
11909 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11910 .await
11911 .unwrap();
11912 route_task.await.unwrap();
11913 }
11914 }
11915
11916 #[tokio::test]
11917 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
11918 let registry = Arc::new(Registry::default());
11919 let forwarding = Arc::new(ForwardingTable::default());
11920 let supervisor = SupervisorHandle::new();
11921 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11922 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
11923 let handler =
11924 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11925 .with_supervisor(supervisor)
11926 .with_admission_facts_config(
11927 Some("fed".to_string()),
11928 Some(vec!["target".to_string()]),
11929 );
11930
11931 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
11932 hello_via_sink(
11933 &handler,
11934 &target_ctx,
11935 &mut target_rx,
11936 hello_frame("target", PROTOCOL_VERSION, 1),
11937 )
11938 .await;
11939 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
11940 hello_via_sink(
11941 &handler,
11942 &other_ctx,
11943 &mut other_rx,
11944 hello_frame("other", PROTOCOL_VERSION, 2),
11945 )
11946 .await;
11947
11948 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
11949 let expected_facts = facts.clone();
11950 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
11951 let route_handler = handler.clone();
11952 let route_task = tokio::spawn(async move {
11953 route_handler
11954 .handle_control_frame(
11955 &client_ctx,
11956 route_open_frame_with_admission_facts(
11957 10,
11958 "target",
11959 unique_project_root("admission-facts"),
11960 Some(subc_control::ConsumerIdentity {
11961 module_id: "fed".to_string(),
11962 launch_nonce: "fed-nonce".to_string(),
11963 }),
11964 Some(facts.clone()),
11965 ),
11966 )
11967 .await
11968 .unwrap()
11969 });
11970 let bind_frame = target_rx.recv().await.unwrap();
11971 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11972 let ModuleControlRequest::RouteBind {
11973 admission_facts, ..
11974 } = bind
11975 else {
11976 panic!("expected route.bind")
11977 };
11978 assert_eq!(admission_facts, Some(expected_facts));
11979 handler
11980 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11981 .await
11982 .unwrap();
11983 assert!(route_task.await.unwrap().is_empty());
11984 assert!(matches!(
11985 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
11986 .unwrap(),
11987 ClientControlResponse::RouteOpen { .. }
11988 ));
11989
11990 let direct = handler
11991 .handle_control_frame(
11992 &route_ctx(ConnectionId::new(73)).0,
11993 route_open_frame_with_admission_facts(
11994 11,
11995 "target",
11996 unique_project_root("admission-facts"),
11997 None,
11998 Some(json!({"x": 1})),
11999 ),
12000 )
12001 .await
12002 .unwrap();
12003 assert_eq!(
12004 parse_error(&direct[0])["code"],
12005 "admission_facts_not_permitted"
12006 );
12007
12008 let different_reserved = handler
12009 .handle_control_frame(
12010 &route_ctx(ConnectionId::new(77)).0,
12011 route_open_frame_with_admission_facts(
12012 15,
12013 "target",
12014 unique_project_root("admission-facts"),
12015 Some(subc_control::ConsumerIdentity {
12016 module_id: "other".to_string(),
12017 launch_nonce: "other-nonce".to_string(),
12018 }),
12019 Some(json!({"x": 1})),
12020 ),
12021 )
12022 .await
12023 .unwrap();
12024 assert_eq!(
12025 parse_error(&different_reserved[0])["code"],
12026 "admission_facts_not_permitted"
12027 );
12028
12029 let other_target = handler
12030 .handle_control_frame(
12031 &route_ctx(ConnectionId::new(74)).0,
12032 route_open_frame_with_admission_facts(
12033 12,
12034 "other",
12035 unique_project_root("admission-facts"),
12036 Some(subc_control::ConsumerIdentity {
12037 module_id: "fed".to_string(),
12038 launch_nonce: "fed-nonce".to_string(),
12039 }),
12040 Some(json!({"x": 1})),
12041 ),
12042 )
12043 .await
12044 .unwrap();
12045 assert_eq!(
12046 parse_error(&other_target[0])["code"],
12047 "admission_facts_target_not_allowed"
12048 );
12049
12050 let nonexistent = handler
12051 .handle_control_frame(
12052 &route_ctx(ConnectionId::new(75)).0,
12053 route_open_frame_with_admission_facts(
12054 13,
12055 "missing",
12056 unique_project_root("admission-facts"),
12057 None,
12058 Some(json!({"x": 1})),
12059 ),
12060 )
12061 .await
12062 .unwrap();
12063 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
12064
12065 let described = handler
12066 .handle_control_frame(
12067 &route_ctx(ConnectionId::new(76)).0,
12068 Frame::build(
12069 FrameType::Request,
12070 control_flags(),
12071 0,
12072 0,
12073 14,
12074 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12075 )
12076 .unwrap(),
12077 )
12078 .await
12079 .unwrap();
12080 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12081 serde_json::from_slice(&described[0].body).unwrap()
12082 else {
12083 panic!("expected server.describe response")
12084 };
12085 assert!(capabilities
12086 .iter()
12087 .any(|cap| cap == "admission_facts_relay_v1"));
12088 }
12089
12090 #[tokio::test]
12091 async fn admission_facts_without_configured_carrier_are_rejected() {
12092 let registry = Arc::new(Registry::default());
12093 let forwarding = Arc::new(ForwardingTable::default());
12094 let handler = ControlHandler::with_forwarding(registry, forwarding);
12095 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
12096 hello_via_sink(
12097 &handler,
12098 &target_ctx,
12099 &mut target_rx,
12100 hello_frame("target", PROTOCOL_VERSION, 1),
12101 )
12102 .await;
12103
12104 let responses = handler
12105 .handle_control_frame(
12106 &route_ctx(ConnectionId::new(79)).0,
12107 route_open_frame_with_admission_facts(
12108 16,
12109 "target",
12110 unique_project_root("admission-facts"),
12111 None,
12112 Some(json!({"x": 1})),
12113 ),
12114 )
12115 .await
12116 .unwrap();
12117 assert_eq!(
12118 parse_error(&responses[0])["code"],
12119 "admission_facts_not_permitted"
12120 );
12121 }
12122
12123 #[tokio::test]
12124 async fn route_open_relays_consumer_capabilities_verbatim() {
12125 let registry = Arc::new(Registry::default());
12126 let forwarding = Arc::new(ForwardingTable::default());
12127 let handler =
12128 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12129 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
12130 hello_via_sink(
12131 &handler,
12132 &module_ctx,
12133 &mut module_rx,
12134 hello_frame("aft", PROTOCOL_VERSION, 7),
12135 )
12136 .await;
12137
12138 let expected = vec!["elicitation".to_string(), "roots".to_string()];
12139 let expected_for_request = expected.clone();
12140 let project_root = unique_project_root("consumer-capabilities-present");
12141 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
12142 let route_handler = handler.clone();
12143 let route_task = tokio::spawn(async move {
12144 route_handler
12145 .handle_control_frame(
12146 &client_ctx,
12147 route_open_frame_with_consumer_capabilities(
12148 401,
12149 "aft",
12150 project_root,
12151 Some(expected_for_request),
12152 ),
12153 )
12154 .await
12155 .unwrap()
12156 });
12157 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12158 .await
12159 .unwrap()
12160 .unwrap();
12161 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12162 let ModuleControlRequest::RouteBind {
12163 consumer_capabilities,
12164 ..
12165 } = bind
12166 else {
12167 panic!("expected route.bind request, got {bind:?}");
12168 };
12169 assert_eq!(consumer_capabilities, Some(expected.clone()));
12170
12171 handler
12172 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12173 .await
12174 .unwrap();
12175 let route_response = route_task.await.unwrap();
12176 assert!(route_response.is_empty());
12177 let published = client_rx.recv().await.unwrap();
12178 assert!(matches!(
12179 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12180 ClientControlResponse::RouteOpen { .. }
12181 ));
12182 }
12183
12184 #[tokio::test]
12185 async fn route_open_without_consumer_capabilities_relays_none() {
12186 let registry = Arc::new(Registry::default());
12187 let forwarding = Arc::new(ForwardingTable::default());
12188 let handler =
12189 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12190 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
12191 hello_via_sink(
12192 &handler,
12193 &module_ctx,
12194 &mut module_rx,
12195 hello_frame("aft", PROTOCOL_VERSION, 7),
12196 )
12197 .await;
12198
12199 let project_root = unique_project_root("consumer-capabilities-absent");
12200 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
12201 let route_handler = handler.clone();
12202 let route_task = tokio::spawn(async move {
12203 route_handler
12204 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
12205 .await
12206 .unwrap()
12207 });
12208 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12209 .await
12210 .unwrap()
12211 .unwrap();
12212 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12213 let ModuleControlRequest::RouteBind {
12214 consumer_capabilities,
12215 ..
12216 } = bind
12217 else {
12218 panic!("expected route.bind request, got {bind:?}");
12219 };
12220 assert_eq!(consumer_capabilities, None);
12221
12222 handler
12223 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12224 .await
12225 .unwrap();
12226 let route_response = route_task.await.unwrap();
12227 assert!(route_response.is_empty());
12228 let published = client_rx.recv().await.unwrap();
12229 assert!(matches!(
12230 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12231 ClientControlResponse::RouteOpen { .. }
12232 ));
12233 }
12234
12235 async fn bind_role_versions_for(
12239 sent: Option<BTreeMap<String, String>>,
12240 connection: u64,
12241 ) -> Option<BTreeMap<String, String>> {
12242 let registry = Arc::new(Registry::default());
12243 let forwarding = Arc::new(ForwardingTable::default());
12244 let handler =
12245 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12246 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
12247 hello_via_sink(
12248 &handler,
12249 &module_ctx,
12250 &mut module_rx,
12251 hello_frame("aft", PROTOCOL_VERSION, 7),
12252 )
12253 .await;
12254 let project_root = unique_project_root("role-versions");
12255 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
12256 let route_handler = handler.clone();
12257 let route_task = tokio::spawn(async move {
12258 route_handler
12259 .handle_control_frame(
12260 &client_ctx,
12261 route_open_frame_with_role_versions(403, "aft", project_root, sent),
12262 )
12263 .await
12264 .unwrap()
12265 });
12266 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12267 .await
12268 .expect("a well-formed route.open reaches the module as a bind")
12269 .unwrap();
12270 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12271 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
12272 panic!("expected route.bind request, got {bind:?}");
12273 };
12274 handler
12275 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12276 .await
12277 .unwrap();
12278 assert!(route_task.await.unwrap().is_empty());
12279 let published = client_rx.recv().await.unwrap();
12280 assert!(matches!(
12281 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12282 ClientControlResponse::RouteOpen { .. }
12283 ));
12284 role_versions
12285 }
12286
12287 #[tokio::test]
12288 async fn route_open_relays_role_versions_verbatim() {
12289 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
12290 assert_eq!(
12291 bind_role_versions_for(Some(sent.clone()), 141).await,
12292 Some(sent)
12293 );
12294 }
12295
12296 #[tokio::test]
12299 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
12300 assert_eq!(bind_role_versions_for(None, 143).await, None);
12301 assert_eq!(
12302 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
12303 None
12304 );
12305 }
12306
12307 #[tokio::test]
12308 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
12309 let registry = Arc::new(Registry::default());
12310 let forwarding = Arc::new(ForwardingTable::default());
12311 let handler =
12312 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12313 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
12314 hello_via_sink(
12315 &handler,
12316 &module_ctx,
12317 &mut module_rx,
12318 hello_frame("aft", PROTOCOL_VERSION, 7),
12319 )
12320 .await;
12321
12322 let nine: BTreeMap<String, String> = (0..9)
12323 .map(|index| (format!("role-{index}"), "v1".to_string()))
12324 .collect();
12325 for (label, malformed) in [
12326 (
12327 "invalid role name",
12328 role_versions(&[("Tool_Provider", "v1")]),
12329 ),
12330 ("invalid version", role_versions(&[("tool-provider", "v0")])),
12331 ("nine entries", nine),
12332 ] {
12333 let responses = tokio::time::timeout(
12336 Duration::from_secs(1),
12337 handler.handle_control_frame(
12338 &route_ctx(ConnectionId::new(148)).0,
12339 route_open_frame_with_role_versions(
12340 404,
12341 "aft",
12342 unique_project_root("role-versions-malformed"),
12343 Some(malformed),
12344 ),
12345 ),
12346 )
12347 .await
12348 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
12349 .unwrap();
12350 assert_eq!(responses.len(), 1, "{label}");
12351 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
12352 let error = parse_error(&responses[0]);
12353 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
12354 assert_eq!(
12355 error["detail"]["field"], "role_versions",
12356 "{label}: {error}"
12357 );
12358 assert!(
12359 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
12360 "{label}: a malformed declaration is terminal"
12361 );
12362 assert!(
12363 module_rx.try_recv().is_err(),
12364 "{label}: the module must never see a bind"
12365 );
12366 }
12367 }
12368
12369 #[tokio::test]
12373 async fn route_role_versions_capability_is_advertised() {
12374 let handler = ControlHandler::new(Arc::new(Registry::default()));
12375 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
12376 let ack = hello_via_sink(
12377 &handler,
12378 &ctx,
12379 &mut rx,
12380 hello_frame("m", PROTOCOL_VERSION, 1),
12381 )
12382 .await;
12383 let ack = parse_ack(&ack);
12384 assert!(
12385 ack.subc_capabilities
12386 .iter()
12387 .any(|c| c == "route-role-versions/v1"),
12388 "{:?}",
12389 ack.subc_capabilities
12390 );
12391
12392 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12393 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12394 let reply = handler
12395 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
12396 .await
12397 .unwrap()
12398 .pop()
12399 .unwrap();
12400 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12401 serde_json::from_slice(&reply.body).unwrap()
12402 else {
12403 panic!("not a server.describe reply");
12404 };
12405 assert!(
12406 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
12407 "{capabilities:?}"
12408 );
12409 }
12410
12411 #[tokio::test]
12415 async fn catalog_events_capability_is_advertised() {
12416 let handler = ControlHandler::new(Arc::new(Registry::default()));
12417 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12418 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12419 let reply = handler
12420 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
12421 .await
12422 .unwrap()
12423 .pop()
12424 .unwrap();
12425 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12426 serde_json::from_slice(&reply.body).unwrap()
12427 else {
12428 panic!("not a server.describe reply");
12429 };
12430 assert!(
12431 capabilities
12432 .iter()
12433 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
12434 "{capabilities:?}"
12435 );
12436 }
12437
12438 #[tokio::test]
12439 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
12440 let handler = ControlHandler::new(Arc::new(Registry::default()));
12441 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
12442 let ack = hello_via_sink(
12443 &handler,
12444 &module,
12445 &mut module_rx,
12446 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
12447 )
12448 .await;
12449 let ack = parse_ack(&ack);
12450 assert!(
12451 ack.subc_capabilities
12452 .iter()
12453 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12454 "{:?}",
12455 ack.subc_capabilities
12456 );
12457 assert!(ack
12458 .subc_capabilities
12459 .iter()
12460 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12461
12462 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
12463 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12464 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12465 let reply = handler
12466 .handle_control_frame(&client, frame)
12467 .await
12468 .unwrap()
12469 .pop()
12470 .unwrap();
12471 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12472 serde_json::from_slice(&reply.body).unwrap()
12473 else {
12474 panic!("not a server.describe reply");
12475 };
12476 assert!(
12477 capabilities
12478 .iter()
12479 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12480 "{capabilities:?}"
12481 );
12482 assert!(capabilities
12483 .iter()
12484 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12485 }
12486
12487 #[tokio::test]
12488 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
12489 let handler = ControlHandler::new(Arc::new(Registry::default()));
12490 let (module, mut rx) = route_ctx(ConnectionId::new(156));
12491 let ack = hello_via_sink(
12492 &handler,
12493 &module,
12494 &mut rx,
12495 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
12496 )
12497 .await;
12498 assert!(parse_ack(&ack)
12499 .subc_capabilities
12500 .iter()
12501 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12502 assert!(parse_ack(&ack)
12503 .subc_capabilities
12504 .iter()
12505 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12506 let (client, _rx) = route_ctx(ConnectionId::new(157));
12507 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12508 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12509 let reply = handler
12510 .handle_control_frame(&client, frame)
12511 .await
12512 .unwrap()
12513 .pop()
12514 .unwrap();
12515 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12516 serde_json::from_slice(&reply.body).unwrap()
12517 else {
12518 panic!("server.describe reply")
12519 };
12520 assert!(capabilities
12521 .iter()
12522 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12523 assert!(capabilities
12524 .iter()
12525 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12526 }
12527
12528 #[tokio::test]
12529 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
12530 let handler = ControlHandler::new(Arc::new(Registry::default()));
12531 let (module, mut rx) = route_ctx(ConnectionId::new(158));
12532 let ack = hello_via_sink(
12533 &handler,
12534 &module,
12535 &mut rx,
12536 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
12537 )
12538 .await;
12539 assert!(parse_ack(&ack)
12540 .subc_capabilities
12541 .iter()
12542 .any(|cap| cap == "scope-workdir/v1"));
12543 let (client, _rx) = route_ctx(ConnectionId::new(159));
12544 let frame = Frame::build(
12545 FrameType::Request,
12546 control_flags(),
12547 0,
12548 0,
12549 5,
12550 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12551 )
12552 .unwrap();
12553 let reply = handler
12554 .handle_control_frame(&client, frame)
12555 .await
12556 .unwrap()
12557 .remove(0);
12558 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12559 serde_json::from_slice(&reply.body).unwrap()
12560 else {
12561 panic!("server.describe reply")
12562 };
12563 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
12564 }
12565
12566 #[tokio::test]
12567 async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
12568 let handler = ControlHandler::new(Arc::new(Registry::default()));
12569 let (module, mut rx) = route_ctx(ConnectionId::new(160));
12570 let ack = hello_via_sink(
12571 &handler,
12572 &module,
12573 &mut rx,
12574 hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
12575 )
12576 .await;
12577 assert!(parse_ack(&ack)
12578 .subc_capabilities
12579 .iter()
12580 .any(|cap| cap == "worker-runs/v1"));
12581 let (client, _rx) = route_ctx(ConnectionId::new(161));
12582 let frame = Frame::build(
12583 FrameType::Request,
12584 control_flags(),
12585 0,
12586 0,
12587 5,
12588 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12589 )
12590 .unwrap();
12591 let reply = handler
12592 .handle_control_frame(&client, frame)
12593 .await
12594 .unwrap()
12595 .remove(0);
12596 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12597 serde_json::from_slice(&reply.body).unwrap()
12598 else {
12599 panic!("server.describe reply")
12600 };
12601 assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
12602 }
12603
12604 #[tokio::test]
12605 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
12606 let handler = ControlHandler::new(Arc::new(Registry::default()));
12607 let (module, mut rx) = route_ctx(ConnectionId::new(154));
12608 let ack = hello_via_sink(
12609 &handler,
12610 &module,
12611 &mut rx,
12612 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
12613 )
12614 .await;
12615 assert!(parse_ack(&ack)
12616 .subc_capabilities
12617 .iter()
12618 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12619 let (client, _rx) = route_ctx(ConnectionId::new(155));
12620 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12621 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12622 let reply = handler
12623 .handle_control_frame(&client, frame)
12624 .await
12625 .unwrap()
12626 .pop()
12627 .unwrap();
12628 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12629 serde_json::from_slice(&reply.body).unwrap()
12630 else {
12631 panic!("not a server.describe reply")
12632 };
12633 assert!(capabilities
12634 .iter()
12635 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12636 }
12637
12638 #[tokio::test]
12639 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
12640 let registry = Arc::new(Registry::default());
12641 let forwarding = Arc::new(ForwardingTable::default());
12642 let handler =
12643 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12644 .with_health_probe_timeout(Duration::from_secs(5));
12645 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
12646 hello_via_sink(
12647 &handler,
12648 &module_ctx,
12649 &mut module_rx,
12650 non_routable_hello_frame_with_control_ops(
12651 "mcp",
12652 300,
12653 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12654 ),
12655 )
12656 .await;
12657 assert!(registry
12658 .get_module("mcp")
12659 .unwrap()
12660 .unwrap()
12661 .manifest
12662 .provides
12663 .is_empty());
12664
12665 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
12666 let route_response = handler
12667 .handle_control_frame(
12668 &route_client_ctx,
12669 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
12670 )
12671 .await
12672 .unwrap();
12673 assert_eq!(route_response[0].header.ty, FrameType::Error);
12674 assert_eq!(
12675 parse_error(&route_response[0])["code"],
12676 "target_unavailable"
12677 );
12678 assert!(parse_error(&route_response[0])["message"]
12679 .as_str()
12680 .unwrap()
12681 .contains("does not provide the requested target"));
12682 assert!(module_rx.try_recv().is_err());
12683
12684 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
12685 let health_handler = handler.clone();
12686 let health_task = tokio::spawn(async move {
12687 health_handler
12688 .handle_control_frame(
12689 &health_client_ctx,
12690 supervisor_health_probe_frame(302, "mcp"),
12691 )
12692 .await
12693 .unwrap()
12694 });
12695 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12696 .await
12697 .unwrap()
12698 .unwrap();
12699 assert_eq!(
12700 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
12701 ModuleControlRequest::HealthCheck {}
12702 );
12703 handler
12704 .handle_control_frame(
12705 &module_ctx,
12706 health_response(health_frame.header.corr, HealthStatus::Ok),
12707 )
12708 .await
12709 .unwrap();
12710 let health_response = health_task.await.unwrap();
12711 assert_eq!(health_response[0].header.ty, FrameType::Response);
12712 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
12713 ClientControlResponse::SupervisorHealthProbe {
12714 module_id, status, ..
12715 } => {
12716 assert_eq!(module_id, "mcp");
12717 assert_eq!(status, HealthStatus::Ok);
12718 }
12719 other => panic!("unexpected health response: {other:?}"),
12720 }
12721
12722 forwarding
12727 .cleanup_connection(module_ctx.connection_id)
12728 .unwrap();
12729 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
12730 let cleanup_response = tokio::time::timeout(
12731 Duration::from_millis(200),
12732 handler.handle_control_frame(
12733 &cleanup_probe_ctx,
12734 supervisor_health_probe_frame(303, "mcp"),
12735 ),
12736 )
12737 .await
12738 .expect("probe should fail immediately when the control lane is gone")
12739 .unwrap();
12740 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
12741 assert_eq!(
12742 parse_error(&cleanup_response[0])["code"],
12743 "target_unavailable"
12744 );
12745 assert!(parse_error(&cleanup_response[0])["message"]
12746 .as_str()
12747 .unwrap()
12748 .contains("no module connection"));
12749
12750 handler
12751 .cleanup_connection(module_ctx.connection_id)
12752 .unwrap();
12753 }
12754
12755 #[tokio::test]
12756 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
12757 let registry = Arc::new(Registry::default());
12758 let supervisor_handle = SupervisorHandle::new();
12759 let supervisor =
12760 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12761 .with_handle(supervisor_handle.clone())
12762 .with_connection_file_path(
12763 std::env::temp_dir()
12764 .join(format!("subc-route-open-warming-{}", std::process::id())),
12765 );
12766 let module = supervisor
12767 .supervise_configured(
12768 ModuleSpec {
12769 module_id: "warming".to_string(),
12770 program: fake_aft_stub_path(),
12771 args: Vec::new(),
12772 env: Vec::new(),
12773 reserved: false,
12774 reserved_prefixes: Vec::new(),
12775 protocol: ModuleProtocol::Subc,
12776 overlap: Default::default(),
12777 },
12778 true,
12779 )
12780 .unwrap();
12781 assert_eq!(module.state().unwrap(), ModuleState::Running);
12782
12783 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12784 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
12785 let response = handler
12786 .handle_control_frame(
12787 &ctx,
12788 route_open_frame(304, "warming", unique_project_root("warming")),
12789 )
12790 .await
12791 .unwrap();
12792 module.stop().await.unwrap();
12793
12794 assert_eq!(response[0].header.ty, FrameType::Error);
12795 let error = parse_error(&response[0]);
12796 assert_eq!(error["code"], "module_warming");
12797 assert!(error["message"]
12798 .as_str()
12799 .unwrap()
12800 .contains("state=running, enabled=true, live=false"));
12801 }
12802
12803 #[test]
12804 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
12805 let handler = ControlHandler::new(Arc::new(Registry::default()));
12806 let capture = EventCapture::default();
12807 let _subscriber =
12808 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12809 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
12810 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
12811 let pending = (0..limit).collect::<Vec<_>>();
12812 let response = handler
12813 .route_open_capacity_refusal(
12814 &ctx,
12815 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
12816 "busy",
12817 pending.len(),
12818 limit,
12819 )
12820 .unwrap();
12821 assert_eq!(parse_error(&response)["code"], "target_unavailable");
12822 let event = capture
12823 .events()
12824 .into_iter()
12825 .find(|event| {
12826 event.target == "control"
12827 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
12828 })
12829 .expect("connection admission refusal event");
12830 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12831 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12832 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
12835 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
12836 }
12837
12838 #[test]
12839 fn route_open_target_cap_logs_admission_reason_and_capacity() {
12840 let handler = ControlHandler::new(Arc::new(Registry::default()));
12841 let capture = EventCapture::default();
12842 let _subscriber =
12843 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12844 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
12845 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
12846 let guards = (0..limit)
12847 .map(|_| {
12848 handler
12849 .route_bind_concurrency
12850 .try_admit("busy", limit)
12851 .unwrap()
12852 })
12853 .collect::<Vec<_>>();
12854 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
12855 Err(in_flight) => in_flight,
12856 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
12857 };
12858 let response = handler
12859 .route_open_target_capacity_refusal(
12860 &ctx,
12861 &RouteOpenCaller::unknown(),
12862 &route_open_frame(397, "busy", unique_project_root("target-cap")),
12863 "busy",
12864 in_flight,
12865 )
12866 .unwrap();
12867 assert_eq!(parse_error(&response)["code"], "target_unavailable");
12868 let event = capture
12869 .events()
12870 .into_iter()
12871 .find(|event| {
12872 event.target == "control"
12873 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
12874 })
12875 .expect("target admission refusal event");
12876 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12877 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12878 drop(guards);
12879 }
12880
12881 #[cfg(unix)]
12882 #[tokio::test(start_paused = true)]
12883 async fn supervisor_list_shows_a_failed_modules_next_retry() {
12884 let registry = Arc::new(Registry::default());
12885 let handle = SupervisorHandle::new();
12886 let supervisor =
12887 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
12888 .with_handle(handle.clone());
12889 let module = supervisor
12890 .spawn(crate::ModuleSpec {
12891 module_id: "retry-list".into(),
12892 program: PathBuf::from("/bin/sh"),
12893 args: vec!["-c".into(), "exit 17".into()],
12894 env: vec![],
12895 reserved: false,
12896 reserved_prefixes: vec![],
12897 protocol: ModuleProtocol::None,
12898 overlap: Default::default(),
12899 })
12900 .unwrap();
12901 let deadline = std::time::Instant::now() + Duration::from_secs(10);
12902 while module.state().unwrap() != crate::ModuleState::Failed {
12903 assert!(
12904 std::time::Instant::now() < deadline,
12905 "waiting for budget failure"
12906 );
12907 tokio::task::yield_now().await;
12908 }
12909 let handler = ControlHandler::new(registry).with_supervisor(handle);
12910 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12911 let replies = handler.handle_supervisor_list(frame).await.unwrap();
12912 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12913 assert_eq!(body["modules"][0]["state"], "failed");
12914 let retry = body["modules"][0]["next_retry_at_ms"]
12915 .as_u64()
12916 .expect("scheduled retry on the wire");
12917 let now = std::time::SystemTime::now()
12918 .duration_since(std::time::UNIX_EPOCH)
12919 .unwrap()
12920 .as_millis() as u64;
12921 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
12922 module.set_enabled(false).await.unwrap();
12923 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
12924 let replies = handler.handle_supervisor_list(frame).await.unwrap();
12925 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12926 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
12927 }
12928
12929 #[tokio::test]
12930 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
12931 let registry = Arc::new(Registry::default());
12932 let handle = SupervisorHandle::new();
12933 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
12934 .with_handle(handle.clone());
12935 supervisor
12936 .supervise_configured(
12937 crate::ModuleSpec {
12938 module_id: "drift".into(),
12939 program: PathBuf::from("disabled-module"),
12940 args: Vec::new(),
12941 env: Vec::new(),
12942 reserved: false,
12943 reserved_prefixes: Vec::new(),
12944 protocol: ModuleProtocol::Subc,
12945 overlap: Default::default(),
12946 },
12947 false,
12948 )
12949 .unwrap();
12950 let handler = ControlHandler::new(registry).with_supervisor(handle);
12951 for configured in [None, Some(false), Some(true)] {
12952 handler.capability_evaluator.configure(
12953 configured.map(|enabled| ("drift".to_string(), enabled)),
12954 BTreeMap::new(),
12955 );
12956 let frame =
12957 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12958 let replies = handler
12959 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
12960 .await
12961 .unwrap();
12962 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12963 assert!(body.get("configured_enabled").is_none());
12964 assert_eq!(body["op"], "supervisor.ack");
12965 let frame =
12966 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12967 let replies = handler.handle_supervisor_list(frame).await.unwrap();
12968 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12969 assert_eq!(
12970 body["modules"][0].get("configured_enabled"),
12971 configured.map(Value::Bool).as_ref()
12972 );
12973 }
12974 }
12975
12976 #[tokio::test(flavor = "current_thread")]
12977 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
12978 let handler = ControlHandler::new(Arc::new(Registry::default()));
12979 let capture = EventCapture::default();
12980 let _subscriber =
12981 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12982 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
12983 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12984
12985 let _ = handler
12986 .handle_client_control_request(
12987 &ctx,
12988 frame,
12989 ClientControlRequest::SupervisorSetEnabled {
12990 module_id: "broca".to_string(),
12991 enabled: false,
12992 },
12993 &mut None,
12994 )
12995 .await
12996 .unwrap();
12997
12998 let events = capture
12999 .events()
13000 .into_iter()
13001 .filter(|event| {
13002 event.target == "control"
13003 && event.fields.get("message").map(String::as_str)
13004 == Some("supervisor request received")
13005 })
13006 .collect::<Vec<_>>();
13007 assert_eq!(events.len(), 1, "one supervisor request log line");
13008 let event = &events[0];
13009 assert_eq!(event.level, tracing::Level::INFO);
13010 assert_eq!(
13011 event.fields.get("op"),
13012 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
13013 );
13014 assert_eq!(
13015 event.fields.get("module_id"),
13016 Some(&"\"broca\"".to_string())
13017 );
13018 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
13019 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
13020 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
13021 }
13022
13023 #[tokio::test(flavor = "current_thread")]
13024 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
13025 let registry = Arc::new(Registry::default());
13026 let module_connection = ConnectionId::new(303);
13027 registry
13028 .register_with_control_ops(
13029 manifest("aft", PROTOCOL_VERSION),
13030 PROTOCOL_VERSION,
13031 module_connection,
13032 module_baseline_control_ops(),
13033 )
13034 .unwrap();
13035 let handler = ControlHandler::new(registry);
13036 let capture = EventCapture::default();
13037 let _subscriber =
13038 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13039 let (ctx, _rx) = route_ctx(module_connection);
13040 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13041
13042 let _ = handler
13043 .handle_client_control_request(
13044 &ctx,
13045 frame,
13046 ClientControlRequest::SupervisorSetEnabled {
13047 module_id: "broca".to_string(),
13048 enabled: false,
13049 },
13050 &mut None,
13051 )
13052 .await
13053 .unwrap();
13054
13055 let callers = capture
13056 .events()
13057 .into_iter()
13058 .filter(|event| {
13059 event.fields.get("message").map(String::as_str)
13060 == Some("supervisor request received")
13061 })
13062 .map(|event| event.fields.get("caller").cloned())
13063 .collect::<Vec<_>>();
13064 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
13065 }
13066
13067 #[tokio::test(flavor = "current_thread")]
13068 async fn supervisor_list_does_not_log_request_received() {
13069 let handler = ControlHandler::new(Arc::new(Registry::default()));
13070 let capture = EventCapture::default();
13071 let _subscriber =
13072 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13073 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
13074 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13075
13076 handler
13077 .handle_client_control_request(
13078 &ctx,
13079 frame,
13080 ClientControlRequest::SupervisorList {},
13081 &mut None,
13082 )
13083 .await
13084 .unwrap();
13085
13086 assert!(capture.events().into_iter().all(|event| {
13087 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13088 }));
13089 }
13090
13091 #[tokio::test(flavor = "current_thread")]
13092 async fn supervisor_rescan_preview_does_not_log_request_received() {
13093 let handler = ControlHandler::new(Arc::new(Registry::default()));
13094 let capture = EventCapture::default();
13095 let _subscriber =
13096 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13097 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
13098 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13099
13100 handler
13101 .handle_client_control_request(
13102 &ctx,
13103 frame,
13104 ClientControlRequest::SupervisorRescan { preview: true },
13105 &mut None,
13106 )
13107 .await
13108 .unwrap();
13109
13110 assert!(capture.events().into_iter().all(|event| {
13111 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13112 }));
13113 }
13114
13115 struct ApproveOperator;
13116 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
13117 fn prompt(
13118 &self,
13119 _: &str,
13120 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
13121 ) -> crate::operator_confirm::ProviderResult {
13122 crate::operator_confirm::ProviderResult::Approved
13123 }
13124 }
13125
13126 #[tokio::test]
13127 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
13128 assert_eq!(
13129 ControlHandler::default().privileged_operator_prompt,
13130 PrivilegedOperatorPrompt::Log
13131 );
13132 for (initial, changed, running) in [
13133 ("log", "enforce", PrivilegedOperatorPrompt::Log),
13134 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
13135 ] {
13136 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
13137 let path = dir.join("subc.jsonc");
13138 std::fs::write(
13139 &path,
13140 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
13141 )
13142 .unwrap();
13143 let config = crate::daemon_config::load(&path).unwrap().unwrap();
13144 let registry = Arc::new(Registry::default());
13145 let handle = SupervisorHandle::new();
13146 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
13147 .with_handle(handle.clone());
13148 let handler = ControlHandler::new(registry)
13149 .with_supervisor(handle)
13150 .with_privileged_operator_prompt(config.privileged_operator_prompt)
13151 .with_supervisor_rescan(supervisor, &path, config.port);
13152 handler.forwarding.operator_confirms().configure(
13153 Arc::new(ApproveOperator),
13154 crate::operator_confirm::OperatorLimits::default(),
13155 );
13156 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
13157
13158 for (disk_mode, needs_restart) in [
13159 (Some(initial), false),
13160 (Some(changed), true),
13161 (Some(changed), true),
13162 (None, running == PrivilegedOperatorPrompt::Enforce),
13163 (Some(initial), false),
13164 ] {
13165 let mut doc = json!({"version":1});
13166 if let Some(mode) = disk_mode {
13167 doc["privileged_operator_prompt"] = json!(mode);
13168 }
13169 std::fs::write(&path, doc.to_string()).unwrap();
13170 for preview in [true, false] {
13171 let request = ClientControlRequest::SupervisorRescan { preview };
13172 let frame = Frame::build(
13173 FrameType::Request,
13174 control_flags(),
13175 0,
13176 0,
13177 1,
13178 serde_json::to_vec(&request).unwrap(),
13179 )
13180 .unwrap();
13181 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13182 assert_eq!(frames.len(), 1);
13183 let ClientControlResponse::SupervisorRescan { result } =
13184 serde_json::from_slice(&frames[0].body).unwrap()
13185 else {
13186 panic!("expected a rescan response");
13187 };
13188 let expected = if needs_restart {
13189 vec!["privileged_operator_prompt"]
13190 } else {
13191 Vec::new()
13192 };
13193 assert_eq!(result.restart_required, expected);
13194 assert_eq!(handler.privileged_operator_prompt, running);
13195 assert_eq!(
13196 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
13197 running
13198 );
13199 }
13200 }
13201 }
13202 }
13203
13204 #[tokio::test]
13205 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
13206 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
13207 let path = dir.join("subc.jsonc");
13208 let registry = Arc::new(Registry::default());
13209 let handle = SupervisorHandle::new();
13210 let supervisor =
13211 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
13212 let handler = ControlHandler::new(registry)
13213 .with_supervisor(handle)
13214 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
13215 .with_supervisor_rescan(supervisor, &path, None);
13216 handler.forwarding.operator_confirms().configure(
13217 Arc::new(ApproveOperator),
13218 crate::operator_confirm::OperatorLimits::default(),
13219 );
13220 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
13221 for preview in [true, false] {
13222 std::fs::write(
13223 &path,
13224 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
13225 )
13226 .unwrap();
13227 let request = ClientControlRequest::SupervisorRescan { preview };
13228 let frame = Frame::build(
13229 FrameType::Request,
13230 control_flags(),
13231 0,
13232 0,
13233 1,
13234 serde_json::to_vec(&request).unwrap(),
13235 )
13236 .unwrap();
13237 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13238 assert_eq!(frames.len(), 1);
13239 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
13240 assert_eq!(
13241 handler.privileged_operator_prompt,
13242 PrivilegedOperatorPrompt::Enforce
13243 );
13244 }
13245 }
13246
13247 #[tokio::test]
13252 async fn route_open_refusal_names_the_check_that_refused() {
13253 let handler = ControlHandler::new(Arc::new(Registry::default()));
13254 let capture = EventCapture::default();
13255 let _subscriber =
13256 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13257 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13258 let response = handler
13259 .handle_control_frame(
13260 &ctx,
13261 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
13262 )
13263 .await
13264 .unwrap();
13265
13266 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13267 let event = capture
13268 .events()
13269 .into_iter()
13270 .find(|event| {
13271 event.target == "control"
13272 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13273 })
13274 .expect("route.open refusal event");
13275 assert_eq!(
13276 event.fields.get("reason"),
13277 Some(&"\"not_registered\"".to_string())
13278 );
13279 }
13280
13281 #[tokio::test]
13282 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
13283 let registry = Arc::new(Registry::default());
13284 let supervisor_handle = SupervisorHandle::new();
13285 let supervisor =
13286 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13287 .with_handle(supervisor_handle.clone())
13288 .with_connection_file_path(std::env::temp_dir().join(format!(
13289 "subc-route-open-refusal-info-{}",
13290 std::process::id()
13291 )));
13292 let module = supervisor
13293 .supervise_configured(
13294 ModuleSpec {
13295 module_id: "warming".to_string(),
13296 program: fake_aft_stub_path(),
13297 args: Vec::new(),
13298 env: Vec::new(),
13299 reserved: false,
13300 reserved_prefixes: Vec::new(),
13301 protocol: ModuleProtocol::Subc,
13302 overlap: Default::default(),
13303 },
13304 true,
13305 )
13306 .unwrap();
13307 assert_eq!(module.state().unwrap(), ModuleState::Running);
13308
13309 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13310 assert!(handler
13311 .counters()
13312 .snapshot()
13313 .get("route_open_refused_by_code")
13314 .is_none());
13315 let capture = EventCapture::default();
13316 let _subscriber =
13317 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13318 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
13319 let response = handler
13320 .handle_control_frame(
13321 &ctx,
13322 route_open_frame(394, "warming", unique_project_root("refusal-info")),
13323 )
13324 .await
13325 .unwrap();
13326 module.stop().await.unwrap();
13327
13328 assert_eq!(parse_error(&response[0])["code"], "module_warming");
13329 let event = capture
13330 .events()
13331 .into_iter()
13332 .find(|event| {
13333 event.target == "control"
13334 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
13335 })
13336 .expect("route.open refusal event");
13337 assert_eq!(
13338 event.fields.get("module_id"),
13339 Some(&"\"warming\"".to_string())
13340 );
13341 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
13342 assert_eq!(
13343 event.fields.get("reason"),
13344 Some(&"\"supervised_not_registered\"".to_string())
13345 );
13346 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
13347 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
13348 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
13349 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13350 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13351 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
13352 assert_eq!(
13353 handler.counters().snapshot()["route_open_refused_by_code"],
13354 json!({ "module_warming": 1 })
13355 );
13356 }
13357
13358 const OUTAGE_START: &str = "route.open refusing module: not serving";
13359 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
13360
13361 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
13362 capture
13363 .events()
13364 .into_iter()
13365 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
13366 .collect()
13367 }
13368
13369 fn supervise_stub(
13370 registry: &Arc<Registry>,
13371 module_id: &str,
13372 enabled: bool,
13373 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
13374 let supervisor_handle = SupervisorHandle::new();
13375 let supervisor =
13376 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
13377 .with_handle(supervisor_handle.clone())
13378 .with_connection_file_path(std::env::temp_dir().join(format!(
13379 "subc-route-outage-{module_id}-{}",
13380 std::process::id()
13381 )));
13382 let module = supervisor
13383 .supervise_configured(
13384 ModuleSpec {
13385 module_id: module_id.to_string(),
13386 program: fake_aft_stub_path(),
13387 args: Vec::new(),
13388 env: Vec::new(),
13389 reserved: false,
13390 reserved_prefixes: Vec::new(),
13391 protocol: ModuleProtocol::Subc,
13392 overlap: Default::default(),
13393 },
13394 enabled,
13395 )
13396 .unwrap();
13397 (supervisor_handle, module)
13398 }
13399
13400 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
13401 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
13402 module_id: module_id.to_string(),
13403 drain_timeout_ms: Some(50),
13404 })
13405 .unwrap();
13406 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
13407 }
13408
13409 #[test]
13413 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
13414 let registry = Arc::new(Registry::default());
13415 let forwarding = Arc::new(ForwardingTable::default());
13416 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13417 let second = ControlHandler::with_forwarding(registry, forwarding);
13418 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
13419 }
13420
13421 #[tokio::test(flavor = "current_thread")]
13425 async fn route_open_unknown_module_refusals_add_no_outage_state() {
13426 let handler = ControlHandler::new(Arc::new(Registry::default()));
13427 let capture = EventCapture::default();
13428 let _subscriber =
13429 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13430 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
13431 for corr in 0..8 {
13432 let response = handler
13433 .handle_control_frame(
13434 &ctx,
13435 route_open_frame(
13436 380 + corr,
13437 &format!("nobody-{corr}"),
13438 unique_project_root("outage-unknown"),
13439 ),
13440 )
13441 .await
13442 .unwrap();
13443 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13444 }
13445
13446 assert_eq!(handler.route_outages.tracked_module_count(), 0);
13447 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
13448 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
13449 }
13450
13451 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13456 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
13457 let registry = Arc::new(Registry::default());
13458 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
13459 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13460 let capture = EventCapture::default();
13461 let _subscriber =
13462 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13463 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
13464 handler.route_outages.record_accepted("outage-restart");
13467
13468 let response = handler
13469 .handle_control_frame(
13470 &ctx,
13471 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
13472 )
13473 .await
13474 .unwrap();
13475 assert_eq!(response[0].header.ty, FrameType::Error);
13476 let starts = outage_lines(&capture, OUTAGE_START);
13477 assert_eq!(starts.len(), 1, "{starts:?}");
13478 assert_eq!(starts[0].level, tracing::Level::WARN);
13479 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
13480 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
13481 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
13482 handler.route_outages.record_accepted("outage-restart");
13483 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
13484
13485 let restart = handler
13486 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
13487 .await
13488 .unwrap();
13489 assert_eq!(
13490 restart[0].header.ty,
13491 FrameType::Response,
13492 "{:?}",
13493 parse_error(&restart[0])
13494 );
13495 handler
13496 .handle_control_frame(
13497 &ctx,
13498 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
13499 )
13500 .await
13501 .unwrap();
13502 module.stop().await.unwrap();
13503
13504 let starts = outage_lines(&capture, OUTAGE_START);
13505 assert_eq!(starts.len(), 2, "{starts:?}");
13506 assert_eq!(starts[1].level, tracing::Level::INFO);
13507 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
13508 }
13509
13510 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13514 async fn failed_operator_restart_leaves_no_operator_mark() {
13515 let registry = Arc::new(Registry::default());
13516 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
13517 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13518 let capture = EventCapture::default();
13519 let _subscriber =
13520 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13521 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
13522 handler.route_outages.record_accepted("outage-disabled");
13523
13524 let restart = handler
13525 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
13526 .await
13527 .unwrap();
13528 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
13529 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
13530
13531 handler
13532 .handle_control_frame(
13533 &ctx,
13534 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
13535 )
13536 .await
13537 .unwrap();
13538 let starts = outage_lines(&capture, OUTAGE_START);
13539 assert_eq!(starts.len(), 1, "{starts:?}");
13540 assert_eq!(starts[0].level, tracing::Level::WARN);
13541 }
13542
13543 #[tokio::test(flavor = "current_thread")]
13544 async fn route_open_unknown_module_escapes_target_module_id() {
13545 let handler = ControlHandler::new(Arc::new(Registry::default()));
13546 let capture = EventCapture::default();
13547 let _subscriber =
13548 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13549 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
13550 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13551 let response = handler
13552 .handle_control_frame(
13553 &ctx,
13554 route_open_frame(
13555 395,
13556 hostile_module_id,
13557 unique_project_root("hostile-target-module-id"),
13558 ),
13559 )
13560 .await
13561 .unwrap();
13562
13563 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13564 let event = capture
13565 .events()
13566 .into_iter()
13567 .find(|event| {
13568 event.target == "control"
13569 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13570 })
13571 .expect("route.open unknown-module refusal event");
13572 let logged = event.fields.get("module_id").expect("module_id field");
13573 assert!(!logged.bytes().any(|byte| byte < 0x20));
13574 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13575 }
13576
13577 #[tokio::test(flavor = "current_thread")]
13578 async fn route_open_module_rejection_uses_daemon_counter_key() {
13579 let registry = Arc::new(Registry::default());
13580 let forwarding = Arc::new(ForwardingTable::default());
13581 let handler =
13582 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13583 let module_connection = ConnectionId::new(95);
13584 let (module_ctx, mut module_rx) = route_ctx(module_connection);
13585 hello_via_sink(
13586 &handler,
13587 &module_ctx,
13588 &mut module_rx,
13589 hello_frame("aft", PROTOCOL_VERSION, 395),
13590 )
13591 .await;
13592
13593 let client_connection = ConnectionId::new(96);
13594 let (client_ctx, _client_rx) = route_ctx(client_connection);
13595 let capture = EventCapture::default();
13596 let _subscriber =
13597 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13598 let (route_task, bind) = relay_route_open(
13599 &handler,
13600 client_connection,
13601 &client_ctx.egress,
13602 &mut module_rx,
13603 396,
13604 "aft",
13605 "hostile-module-code",
13606 )
13607 .await;
13608 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
13609 let rejection = Frame::build(
13610 FrameType::Error,
13611 control_flags(),
13612 0,
13613 0,
13614 bind.header.corr,
13615 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
13616 )
13617 .unwrap();
13618 handler
13619 .handle_control_frame(&module_ctx, rejection)
13620 .await
13621 .unwrap();
13622
13623 let response = route_task.await.unwrap();
13624 assert_eq!(parse_error(&response[0])["code"], hostile_code);
13625 let counters = handler.counters().snapshot();
13626 assert_eq!(
13627 counters["route_open_refused_by_code"],
13628 json!({ "module_rejected": 1 })
13629 );
13630 assert!(counters["route_open_refused_by_code"]
13631 .get(hostile_code)
13632 .is_none());
13633
13634 let event = capture
13635 .events()
13636 .into_iter()
13637 .find(|event| {
13638 event.target == "control"
13639 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
13640 })
13641 .expect("route.open module-rejection refusal event");
13642 let logged = event.fields.get("module_code").expect("module_code field");
13643 assert!(!logged.bytes().any(|byte| byte < 0x20));
13644 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13645 }
13646
13647 #[tokio::test]
13648 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
13649 let registry = Arc::new(Registry::default());
13650 let supervisor_handle = SupervisorHandle::new();
13651 let missing_program = std::env::temp_dir().join(format!(
13652 "subc-route-open-missing-program-{}",
13653 std::process::id()
13654 ));
13655 let supervisor =
13656 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13657 .with_handle(supervisor_handle.clone());
13658 let module = supervisor
13659 .supervise_configured(
13660 ModuleSpec {
13661 module_id: "failed".to_string(),
13662 program: missing_program,
13663 args: Vec::new(),
13664 env: Vec::new(),
13665 reserved: false,
13666 reserved_prefixes: Vec::new(),
13667 protocol: ModuleProtocol::Subc,
13668 overlap: Default::default(),
13669 },
13670 true,
13671 )
13672 .unwrap();
13673 assert_eq!(module.state().unwrap(), ModuleState::Failed);
13674
13675 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13676 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
13677 let response = handler
13678 .handle_control_frame(
13679 &ctx,
13680 route_open_frame(305, "failed", unique_project_root("failed")),
13681 )
13682 .await
13683 .unwrap();
13684
13685 assert_eq!(response[0].header.ty, FrameType::Error);
13686 let error = parse_error(&response[0]);
13687 assert_eq!(error["code"], "target_unavailable");
13688 assert!(error["message"]
13689 .as_str()
13690 .unwrap()
13691 .contains("state=failed, enabled=true, live=false"));
13692 }
13693
13694 #[tokio::test]
13695 async fn route_open_role_mismatch_remains_target_unavailable() {
13696 let registry = Arc::new(Registry::default());
13697 let handler = ControlHandler::new(Arc::clone(®istry));
13698 handler
13699 .handle_control(
13700 ConnectionId::new(41),
13701 non_routable_hello_frame_with_control_ops("health-only", 306, None),
13702 )
13703 .unwrap();
13704
13705 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
13706 let response = handler
13707 .handle_control_frame(
13708 &ctx,
13709 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
13710 )
13711 .await
13712 .unwrap();
13713
13714 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13715 assert!(parse_error(&response[0])["message"]
13716 .as_str()
13717 .unwrap()
13718 .contains("does not provide the requested target"));
13719 }
13720
13721 #[tokio::test]
13722 async fn route_open_inactive_registration_remains_target_unavailable() {
13723 let registry = Arc::new(Registry::default());
13724 let handler = ControlHandler::new(Arc::clone(®istry));
13725 handler
13726 .handle_control(
13727 ConnectionId::new(43),
13728 hello_frame("inactive", PROTOCOL_VERSION, 308),
13729 )
13730 .unwrap();
13731 assert!(registry
13732 .set_module_state_for_test("inactive", ChannelState::Closed)
13733 .unwrap());
13734
13735 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
13736 let response = handler
13737 .handle_control_frame(
13738 &ctx,
13739 route_open_frame(309, "inactive", unique_project_root("inactive")),
13740 )
13741 .await
13742 .unwrap();
13743
13744 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13745 assert!(parse_error(&response[0])["message"]
13746 .as_str()
13747 .unwrap()
13748 .contains("is not active"));
13749 }
13750
13751 #[tokio::test]
13752 async fn late_health_reply_is_recorded_through_the_module_response_path() {
13753 let registry = Arc::new(Registry::default());
13754 let forwarding = Arc::new(ForwardingTable::default());
13755 let supervisor_handle = SupervisorHandle::new();
13756 let supervisor =
13757 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
13758 .with_forwarding(Arc::clone(&forwarding))
13759 .with_handle(supervisor_handle.clone());
13760 let module = supervisor
13761 .supervise_configured(
13762 crate::ModuleSpec {
13763 module_id: "late-health-response".to_string(),
13764 program: PathBuf::from("disabled-module"),
13765 args: Vec::new(),
13766 env: Vec::new(),
13767 reserved: false,
13768 reserved_prefixes: Vec::new(),
13769 protocol: ModuleProtocol::Subc,
13770 overlap: Default::default(),
13771 },
13772 false,
13773 )
13774 .unwrap();
13775 let handler =
13776 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13777 .with_supervisor(supervisor_handle);
13778 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
13779 handler
13780 .handle_control_frame(
13781 &module_ctx,
13782 hello_frame_with_control_ops(
13783 "late-health-response",
13784 PROTOCOL_VERSION,
13785 7,
13786 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13787 ),
13788 )
13789 .await
13790 .unwrap();
13791 let probe_started_at = Instant::now() - Duration::from_millis(80);
13792 let pending = forwarding
13793 .begin_health_probe_rpc_for(
13794 "late-health-response",
13795 MODULE_CONTROL_OP_HEALTH_CHECK,
13796 probe_started_at,
13797 Instant::now() - Duration::from_millis(1),
13798 )
13799 .unwrap();
13800 assert!(forwarding
13801 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
13802 .unwrap());
13803
13804 let responses = handler
13805 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
13806 .await
13807 .unwrap();
13808
13809 assert!(responses.is_empty());
13810 let health = module.status().unwrap().health;
13811 assert_eq!(health.late_answer_count, 1);
13812 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
13813 }
13814
13815 #[tokio::test]
13816 async fn health_probe_timeout_and_module_death_are_typed() {
13817 let registry = Arc::new(Registry::default());
13818 let forwarding = Arc::new(ForwardingTable::default());
13819 let handler =
13820 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13821 .with_health_probe_timeout(Duration::from_millis(50));
13822 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
13823 hello_via_sink(
13824 &handler,
13825 &module_ctx,
13826 &mut module_rx,
13827 hello_frame_with_control_ops(
13828 "aft",
13829 PROTOCOL_VERSION,
13830 7,
13831 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13832 ),
13833 )
13834 .await;
13835
13836 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
13837 let responses = handler
13838 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
13839 .await
13840 .unwrap();
13841 assert_eq!(responses[0].header.ty, FrameType::Error);
13842 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
13843 let _ = module_rx.try_recv();
13844
13845 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
13846 let health_handler = handler.clone();
13847 let death_task = tokio::spawn(async move {
13848 health_handler
13849 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
13850 .await
13851 .unwrap()
13852 });
13853 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13854 .await
13855 .unwrap()
13856 .unwrap();
13857 handler
13858 .cleanup_connection(module_ctx.connection_id)
13859 .unwrap();
13860 let responses = death_task.await.unwrap();
13861 assert_eq!(responses[0].header.ty, FrameType::Error);
13862 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
13863 }
13864
13865 #[test]
13866 fn hello_requires_exact_protocol_version() {
13867 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
13868 let registry = Arc::new(Registry::default());
13869 let handler = ControlHandler::new(Arc::clone(®istry));
13870 let responses = handler
13871 .handle_control(
13872 ConnectionId::new(connection),
13873 hello_frame("aft", offered, 9),
13874 )
13875 .unwrap();
13876
13877 assert_eq!(responses.len(), 1);
13878 assert_eq!(responses[0].header.ty, FrameType::Error);
13879 let error = parse_error(&responses[0]);
13880 assert_eq!(error["code"], "version_unsupported");
13881 assert!(registry.get_module("aft").unwrap().is_none());
13882 assert_eq!(registry.active_registration_count().unwrap(), 0);
13883 }
13884 }
13885
13886 #[test]
13887 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
13888 let registry = Arc::new(Registry::default());
13889 let forwarding = Arc::new(ForwardingTable::default());
13890 let handler =
13891 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13892 let module_connection = ConnectionId::new(301);
13893 let registration = registry
13894 .register_with_control_ops(
13895 manifest("aft-push", PROTOCOL_VERSION),
13896 PROTOCOL_VERSION,
13897 module_connection,
13898 module_baseline_control_ops(),
13899 )
13900 .unwrap();
13901 let (module_tx, _module_rx) = mpsc::channel(8);
13902 let endpoint = forwarding
13903 .register_module_connection(
13904 module_connection,
13905 "aft-push".to_string(),
13906 PROTOCOL_VERSION,
13907 manifest_concurrency(®istration.manifest),
13908 FrameSink::new(module_tx),
13909 )
13910 .unwrap();
13911
13912 let unknown = Frame::build(
13914 FrameType::Push,
13915 control_flags(),
13916 0,
13917 0,
13918 5,
13919 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
13920 )
13921 .unwrap();
13922 let out = handler.handle_status_update(endpoint, unknown).unwrap();
13923 assert!(
13924 out.is_empty(),
13925 "unknown push op must be ignored, got {out:?}"
13926 );
13927
13928 let malformed = Frame::build(
13930 FrameType::Push,
13931 control_flags(),
13932 0,
13933 0,
13934 6,
13935 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
13936 )
13937 .unwrap();
13938 let out = handler.handle_status_update(endpoint, malformed).unwrap();
13939 assert_eq!(out.len(), 1);
13940 assert_eq!(out[0].header.ty, FrameType::Error);
13941 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
13942 }
13943
13944 #[test]
13945 fn hello_rejected_when_connection_already_owns_client_routes() {
13946 let registry = Arc::new(Registry::default());
13947 let forwarding = Arc::new(ForwardingTable::default());
13948 let handler =
13949 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13950 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
13952 let client_connection = ConnectionId::new(202);
13953
13954 let responses = handler
13957 .handle_control(
13958 client_connection,
13959 hello_frame("aft-second", PROTOCOL_VERSION, 9),
13960 )
13961 .unwrap();
13962 assert_eq!(responses[0].header.ty, FrameType::Error);
13963 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
13964 assert!(registry.get_module("aft-second").unwrap().is_none());
13965 }
13966
13967 #[tokio::test]
13968 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
13969 let registry = Arc::new(Registry::default());
13970 let forwarding = Arc::new(ForwardingTable::default());
13971 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
13972 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
13973 hello_via_sink(
13974 &handler,
13975 &module_ctx,
13976 &mut module_rx,
13977 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
13978 )
13979 .await;
13980 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
13981 let pending = forwarding
13982 .begin_route_bind_relay_for_test(
13983 client_ctx.connection_id,
13984 client_ctx.egress.clone(),
13985 2,
13986 "alpha",
13987 )
13988 .unwrap();
13989 forwarding
13990 .complete_pending_relay(
13991 module_ctx.connection_id,
13992 pending.corr,
13993 RouteBindRelayOutcome::Accepted,
13994 )
13995 .unwrap();
13996 client_rx.try_recv().unwrap();
13997 for module_id in ["beta", "alpha"] {
13998 let replies = handler
13999 .handle_control_frame(
14000 &module_ctx,
14001 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
14002 )
14003 .await
14004 .unwrap();
14005 assert_eq!(replies.len(), 1, "second HELLO must be refused");
14006 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
14007 }
14008 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
14009 assert!(registry.get_module("beta").unwrap().is_none());
14010 assert!(matches!(
14011 forwarding
14012 .lookup_data_route(
14013 client_ctx.connection_id,
14014 pending.client_channel,
14015 pending.client_epoch,
14016 )
14017 .unwrap(),
14018 DataRoute::Client(DataRouteState::Bound(_))
14019 ));
14020 assert!(handler
14021 .hello_launch_nonces
14022 .lock()
14023 .unwrap()
14024 .presented(module_ctx.connection_id, Some("alpha-nonce")));
14025 assert!(module_rx.try_recv().is_err());
14026 }
14027
14028 #[test]
14029 fn reserved_module_hello_requires_matching_launch_nonce() {
14030 let registry = Arc::new(Registry::default());
14031 let supervisor = SupervisorHandle::new();
14032 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
14035 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14036
14037 let no_nonce = handler
14039 .handle_control(
14040 ConnectionId::new(1),
14041 hello_frame("vault", PROTOCOL_VERSION, 1),
14042 )
14043 .unwrap();
14044 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
14045 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
14046 assert!(registry.get_module("vault").unwrap().is_none());
14047
14048 let wrong = handler
14050 .handle_control(
14051 ConnectionId::new(2),
14052 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
14053 )
14054 .unwrap();
14055 assert_eq!(wrong[0].header.ty, FrameType::Error);
14056 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
14057 assert!(registry.get_module("vault").unwrap().is_none());
14058
14059 let ok = handler
14061 .handle_control(
14062 ConnectionId::new(3),
14063 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
14064 )
14065 .unwrap();
14066 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
14067 assert!(registry.get_module("vault").unwrap().is_some());
14068 }
14069
14070 #[test]
14071 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
14072 let registry = Arc::new(Registry::default());
14073 let supervisor = SupervisorHandle::new();
14074 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14075 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14076 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14077
14078 let squat = handler
14079 .handle_control(
14080 ConnectionId::new(1),
14081 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
14082 )
14083 .unwrap();
14084 assert_eq!(squat[0].header.ty, FrameType::Error);
14085 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
14086 assert!(parse_error(&squat[0])["message"]
14087 .as_str()
14088 .unwrap()
14089 .contains("fed:"));
14090
14091 let accepted_peer = handler
14092 .handle_control(
14093 ConnectionId::new(2),
14094 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
14095 )
14096 .unwrap();
14097 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
14098
14099 let accepted_short = handler
14100 .handle_control(
14101 ConnectionId::new(3),
14102 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
14103 )
14104 .unwrap();
14105 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
14106
14107 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
14108 let response = handler
14109 .handle_control(
14110 ConnectionId::new(conn),
14111 hello_frame(module_id, PROTOCOL_VERSION, conn),
14112 )
14113 .unwrap();
14114 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
14115 }
14116 }
14117
14118 #[test]
14119 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
14120 let registry = Arc::new(Registry::default());
14121 let supervisor = SupervisorHandle::new();
14122 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14123 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14124 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
14125 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14126
14127 let owner_nonce = handler
14128 .handle_control(
14129 ConnectionId::new(1),
14130 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
14131 )
14132 .unwrap();
14133 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
14134 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
14135 assert!(registry.get_module("fed:special").unwrap().is_none());
14136
14137 let exact_nonce = handler
14138 .handle_control(
14139 ConnectionId::new(2),
14140 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
14141 )
14142 .unwrap();
14143 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
14144 assert!(registry.get_module("fed:special").unwrap().is_some());
14145 }
14146
14147 #[test]
14148 fn non_reserved_module_ignores_launch_nonce() {
14149 let registry = Arc::new(Registry::default());
14150 let handler = ControlHandler::new(Arc::clone(®istry));
14153 let no_nonce = handler
14154 .handle_control(
14155 ConnectionId::new(1),
14156 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
14157 )
14158 .unwrap();
14159 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
14160 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
14161
14162 let echoed_nonce = handler
14163 .handle_control(
14164 ConnectionId::new(2),
14165 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
14166 )
14167 .unwrap();
14168 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
14169 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
14170 }
14171
14172 #[test]
14173 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
14174 let handler = ControlHandler::default();
14175 let conn = ConnectionId::new(1);
14176 let malformed = Frame::build(
14177 FrameType::Hello,
14178 control_flags(),
14179 0,
14180 0,
14181 3,
14182 b"{not json".to_vec(),
14183 )
14184 .unwrap();
14185
14186 let error = handler.handle_control(conn, malformed).unwrap();
14187 assert_eq!(error[0].header.ty, FrameType::Error);
14188 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
14189
14190 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14191 let pong = handler.handle_control(conn, ping).unwrap();
14192 assert_eq!(pong[0].header.ty, FrameType::Pong);
14193 assert_eq!(pong[0].header.corr, 4);
14194 }
14195
14196 #[test]
14197 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
14198 let registry = Arc::new(Registry::default());
14199 let handler = ControlHandler::new(Arc::clone(®istry));
14200
14201 handler
14202 .handle_control(
14203 ConnectionId::new(1),
14204 hello_frame("aft", PROTOCOL_VERSION, 1),
14205 )
14206 .unwrap();
14207 let duplicate = handler
14208 .handle_control(
14209 ConnectionId::new(2),
14210 hello_frame("aft", PROTOCOL_VERSION, 2),
14211 )
14212 .unwrap();
14213
14214 assert_eq!(duplicate[0].header.ty, FrameType::Error);
14215 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14216 let registration = registry.get_module("aft").unwrap().unwrap();
14217 assert_eq!(registration.connection_id, ConnectionId::new(1));
14218 }
14219
14220 #[test]
14221 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
14222 let registry = Arc::new(Registry::default());
14223 let forwarding = Arc::new(ForwardingTable::default());
14224 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
14225 let handler =
14226 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14227 .with_process_liveness(process_liveness);
14228 let (ctx, route_channel, route_epoch) =
14229 bind_liveness_route(®istry, &forwarding, "aft-dead");
14230 let responses = handler
14231 .handle_route_poll(
14232 &ctx,
14233 route_poll_frame(41, PollKind::Liveness, route_channel),
14234 route_channel,
14235 route_epoch,
14236 PollKind::Liveness,
14237 )
14238 .unwrap();
14239
14240 assert_eq!(responses.len(), 1);
14241 assert_eq!(responses[0].header.ty, FrameType::Response);
14242 assert_route_poll_liveness(&responses[0], false);
14243 }
14244
14245 #[test]
14246 fn liveness_poll_without_process_source_uses_bound_route() {
14247 let registry = Arc::new(Registry::default());
14248 let forwarding = Arc::new(ForwardingTable::default());
14249 let handler =
14250 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14251 let (ctx, route_channel, route_epoch) =
14252 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
14253 let responses = handler
14254 .handle_route_poll(
14255 &ctx,
14256 route_poll_frame(42, PollKind::Liveness, route_channel),
14257 route_channel,
14258 route_epoch,
14259 PollKind::Liveness,
14260 )
14261 .unwrap();
14262
14263 assert_route_poll_liveness(&responses[0], true);
14264 }
14265
14266 #[test]
14267 fn liveness_poll_untracked_process_source_uses_bound_route() {
14268 let registry = Arc::new(Registry::default());
14269 let forwarding = Arc::new(ForwardingTable::default());
14270 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
14271 let handler =
14272 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14273 .with_process_liveness(process_liveness);
14274 let (ctx, route_channel, route_epoch) =
14275 bind_liveness_route(®istry, &forwarding, "aft-untracked");
14276 let responses = handler
14277 .handle_route_poll(
14278 &ctx,
14279 route_poll_frame(43, PollKind::Liveness, route_channel),
14280 route_channel,
14281 route_epoch,
14282 PollKind::Liveness,
14283 )
14284 .unwrap();
14285
14286 assert_route_poll_liveness(&responses[0], true);
14287 }
14288
14289 #[tokio::test]
14290 async fn unknown_op_returns_unknown_control_op() {
14291 let handler = ControlHandler::default();
14292 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
14293 let request = Frame::build(
14294 FrameType::Request,
14295 control_flags(),
14296 0,
14297 0,
14298 55,
14299 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
14300 )
14301 .unwrap();
14302
14303 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14304
14305 assert_eq!(response.len(), 1);
14306 assert_eq!(response[0].header.ty, FrameType::Error);
14307 assert_eq!(response[0].header.corr, 55);
14308 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
14309 }
14310
14311 #[tokio::test]
14312 async fn supervisor_provenance_rejects_unknown_exact_module() {
14313 let handler = ControlHandler::default();
14314 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
14315 let request = Frame::build(
14316 FrameType::Request,
14317 control_flags(),
14318 0,
14319 0,
14320 57,
14321 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
14322 )
14323 .unwrap();
14324
14325 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14326
14327 assert_eq!(response.len(), 1);
14328 assert_eq!(response[0].header.ty, FrameType::Error);
14329 assert_eq!(response[0].header.corr, 57);
14330 let error = parse_error(&response[0]);
14331 assert_eq!(error["code"], "unknown_module");
14332 assert_eq!(error["message"], "module_id 'missing' is not supervised");
14333 }
14334
14335 #[test]
14336 fn provenance_probe_override_keeps_handler_tests_deterministic() {
14337 let expected = subc_control::RunningImageAgreement::Unavailable {
14338 reason: subc_control::RunningImageUnavailableReason::HashFailed,
14339 };
14340 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
14341 assert_eq!(handler.provenance_probe_override, Some(expected));
14342 }
14343
14344 #[test]
14345 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
14346 let verdict = reload_verdict(
14347 std::path::Path::new("/bin/new"),
14348 Some(std::path::Path::new("/bin/old")),
14349 subc_control::RunningImageAgreement::Unavailable {
14350 reason: subc_control::RunningImageUnavailableReason::HashFailed,
14351 },
14352 );
14353 assert!(matches!(
14354 verdict.path,
14355 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
14356 if configured == std::path::Path::new("/bin/new")
14357 && spawned_from == std::path::Path::new("/bin/old")
14358 ));
14359 }
14360
14361 #[test]
14362 fn launch_spec_without_running_digest_is_unavailable_not_match() {
14363 use subc_control::{
14364 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
14365 };
14366 let configured = crate::supervise::ModuleSpec {
14367 module_id: "legacy".into(),
14368 program: "/module".into(),
14369 args: Vec::new(),
14370 env: Vec::new(),
14371 reserved: false,
14372 reserved_prefixes: Vec::new(),
14373 protocol: subc_control::ModuleProtocol::Subc,
14374 overlap: Default::default(),
14375 };
14376 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
14377 assert_eq!(
14378 launch_spec_agreement(&configured, false, Some(&digest)),
14379 Agreement::Unavailable {
14380 reason: Reason::NotRunning
14381 }
14382 );
14383 assert_eq!(
14384 launch_spec_agreement(&configured, true, None),
14385 Agreement::Unavailable {
14386 reason: Reason::SpawnedSpecUnavailable
14387 }
14388 );
14389 assert_eq!(
14390 launch_spec_agreement(&configured, true, Some(&digest)),
14391 Agreement::Match
14392 );
14393 }
14394
14395 #[cfg(windows)]
14396 #[test]
14397 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
14398 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
14399 let configured = dir.join("module.exe");
14400 std::fs::write(&configured, b"executable path fixture").unwrap();
14401 let spawned = std::fs::canonicalize(&configured).unwrap();
14402 assert_ne!(
14403 configured, spawned,
14404 "fixture must exercise different path spellings"
14405 );
14406 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
14407 let lower_case = configured.to_string_lossy().to_lowercase();
14408 for spelling in [
14409 &configured,
14410 Path::new(&forward_slashes),
14411 Path::new(&lower_case),
14412 ] {
14413 let verdict = reload_verdict(
14414 spelling,
14415 Some(&spawned),
14416 subc_control::RunningImageAgreement::Unavailable {
14417 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14418 },
14419 );
14420 assert_eq!(
14421 verdict.path,
14422 ReloadPathAgreement::Match,
14423 "configured {spelling:?}, spawned {spawned:?}"
14424 );
14425 }
14426 }
14427
14428 #[cfg(windows)]
14429 #[test]
14430 fn windows_reload_path_keeps_different_existing_program_pending() {
14431 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
14432 let old = dir.join("old.exe");
14433 let new = dir.join("new.exe");
14434 std::fs::write(&old, b"same executable content").unwrap();
14435 std::fs::copy(&old, &new).unwrap();
14436 let spawned = std::fs::canonicalize(&old).unwrap();
14437 let image = subc_control::RunningImageAgreement::Unavailable {
14438 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14439 };
14440 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
14441 assert!(
14442 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
14443 if configured == new && spawned_from == spawned)
14444 );
14445 let missing = dir.join("missing.exe");
14446 assert!(matches!(
14447 reload_verdict(&missing, Some(&spawned), image).path,
14448 ReloadPathAgreement::Mismatch { .. }
14449 ));
14450 }
14451
14452 #[test]
14453 fn reload_verdict_detects_replaced_image_at_same_path() {
14454 let image = subc_control::RunningImageAgreement::Mismatch {
14455 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
14456 digest: "old".into(),
14457 },
14458 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
14459 digest: "new".into(),
14460 },
14461 };
14462 let verdict = reload_verdict(
14463 std::path::Path::new("/bin/same"),
14464 Some(std::path::Path::new("/bin/same")),
14465 image.clone(),
14466 );
14467 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14468 assert_eq!(verdict.image, image);
14469 }
14470
14471 #[test]
14472 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
14473 let image = subc_control::RunningImageAgreement::Unavailable {
14474 reason: subc_control::RunningImageUnavailableReason::NotRunning,
14475 };
14476 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
14477 assert_eq!(
14478 verdict.path,
14479 subc_control::ReloadPathAgreement::Unavailable {
14480 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
14481 }
14482 );
14483 assert_eq!(verdict.image, image);
14484
14485 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
14486 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14487 };
14488 let verdict = reload_verdict(
14489 std::path::Path::new("/bin/same"),
14490 Some(std::path::Path::new("/bin/same")),
14491 unconfirmed.clone(),
14492 );
14493 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14494 assert_eq!(verdict.image, unconfirmed);
14495 }
14496
14497 #[test]
14498 fn reload_verdict_preserves_each_image_unavailability_reason() {
14499 use subc_control::RunningImageUnavailableReason as Reason;
14500
14501 for reason in [
14502 Reason::NotRunning,
14503 Reason::UnsupportedPlatform,
14504 Reason::RunningExecutableUnreadable,
14505 Reason::SpawnedPathUnreadable,
14506 Reason::HashFailed,
14507 Reason::ProcessIdentityUnconfirmed,
14508 Reason::Unknown("future_probe_reason".to_string()),
14509 ] {
14510 let image = subc_control::RunningImageAgreement::Unavailable {
14511 reason: reason.clone(),
14512 };
14513 let verdict = reload_verdict(
14514 std::path::Path::new("/bin/same"),
14515 Some(std::path::Path::new("/bin/same")),
14516 image.clone(),
14517 );
14518 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14519 assert_eq!(verdict.image, image, "{reason:?}");
14520 }
14521 }
14522
14523 #[tokio::test]
14524 async fn malformed_control_bodies_return_invalid_control_body() {
14525 let handler = ControlHandler::default();
14526 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
14527
14528 for (corr, body) in [
14529 (56, br#"{"route_channel":1}"#.as_slice()),
14530 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
14531 (
14532 58,
14533 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
14534 ),
14535 ] {
14536 let request = Frame::build(
14537 FrameType::Request,
14538 control_flags(),
14539 0,
14540 0,
14541 corr,
14542 body.to_vec(),
14543 )
14544 .unwrap();
14545 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14546
14547 assert_eq!(response.len(), 1);
14548 assert_eq!(response[0].header.ty, FrameType::Error);
14549 assert_eq!(response[0].header.corr, corr);
14550 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
14551 }
14552 }
14553
14554 #[tokio::test]
14555 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
14556 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14557 let registry = Arc::new(Registry::default());
14558 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
14559 let router = Router::with_control_handler(Arc::clone(&control));
14560 let connection = router.begin_connection();
14561 let (ctx, mut rx) = route_ctx(connection.id());
14562
14563 router
14564 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
14565 .await
14566 .unwrap();
14567 let response = rx.recv().await.unwrap();
14568 let ack = parse_ack(&response);
14569 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14570 let channel = 1;
14571
14572 let goodbye =
14573 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
14574 router.route_for_connection(&ctx, goodbye).await.unwrap();
14575 assert!(rx.try_recv().is_err());
14576 assert!(registry.get_module("aft").unwrap().is_none());
14577
14578 router
14579 .route_for_connection(&ctx, channel_request(channel, 13))
14580 .await
14581 .unwrap();
14582 let error_frame = rx.recv().await.unwrap();
14583 assert_eq!(error_frame.header.ty, FrameType::Error);
14584 assert_eq!(error_frame.header.channel, channel);
14585 let captured = crate::router::test_log::captured_logs(&logs);
14586 assert_eq!(
14587 captured
14588 .lines()
14589 .filter(|line| {
14590 line.contains("module_id=aft")
14591 && line.contains("reason=explicit_goodbye")
14592 && line.contains("module registration ended")
14593 })
14594 .count(),
14595 1,
14596 "unexpected GOODBYE registry log: {captured}"
14597 );
14598 }
14599
14600 #[tokio::test]
14601 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
14602 let registry = Arc::new(Registry::default());
14603 let handler = ControlHandler::new(registry).with_capability_config(
14604 [("prov".to_string(), true), ("cons".to_string(), true)],
14605 BTreeMap::new(),
14606 );
14607 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
14608 register_capability_manifest(
14609 &handler,
14610 &provider_ctx,
14611 &mut provider_rx,
14612 capability_manifest("prov", &["thing/v1"], &[]),
14613 1,
14614 )
14615 .await;
14616 let mut consumer = capability_manifest("cons", &[], &[]);
14617 consumer.capabilities.as_mut().unwrap().requires.push(
14618 subc_protocol::manifest::CapabilityRequirement {
14619 capability: "thing/v1".to_string(),
14620 need: subc_protocol::manifest::CapabilityNeed::Required,
14621 },
14622 );
14623 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
14624 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
14625 assert_eq!(
14626 handler.capability_evaluator.verdict("cons", "thing/v1"),
14627 Some(CapabilityVerdict::Provided)
14628 );
14629 let (mut client_rx, _) = open_route_for_capability_test(
14630 &handler,
14631 &provider_ctx,
14632 &mut provider_rx,
14633 703,
14634 3,
14635 "prov",
14636 None,
14637 )
14638 .await;
14639 let goodbye =
14640 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14641 handler
14642 .handle_control_frame(&provider_ctx, goodbye)
14643 .await
14644 .unwrap();
14645 assert_eq!(
14646 handler.capability_evaluator.verdict("cons", "thing/v1"),
14647 Some(CapabilityVerdict::NeverProvided)
14648 );
14649 let closed = client_rx
14650 .try_recv()
14651 .expect("GOODBYE pushes route.closed before route GOODBYE");
14652 assert!(
14653 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
14654 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
14655 );
14656 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
14657 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14658 }
14659
14660 struct RouteClosedFixture {
14661 handler: ControlHandler,
14662 #[cfg(unix)]
14663 supervisor: Supervisor,
14664 module_ctx: RouteCtx,
14665 #[cfg(unix)]
14666 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14667 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14668 route: Arc<crate::forwarding::RouteBinding>,
14669 }
14670
14671 async fn route_closed_fixture() -> RouteClosedFixture {
14672 let registry = Arc::new(Registry::default());
14673 let handler = ControlHandler::new(Arc::clone(®istry));
14674 #[cfg(unix)]
14675 let supervisor = Supervisor::new(registry, RestartPolicy::default())
14676 .with_forwarding(Arc::clone(&handler.forwarding));
14677 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
14678 register_capability_manifest(
14679 &handler,
14680 &module_ctx,
14681 &mut module_rx,
14682 capability_manifest("prov", &[], &[]),
14683 1,
14684 )
14685 .await;
14686 let (client_rx, bind) = open_route_for_capability_test(
14687 &handler,
14688 &module_ctx,
14689 &mut module_rx,
14690 702,
14691 2,
14692 "prov",
14693 None,
14694 )
14695 .await;
14696 let ModuleControlRequest::RouteBind {
14697 route_channel: channel,
14698 epoch: route_epoch,
14699 ..
14700 } = bind
14701 else {
14702 panic!("expected route.bind");
14703 };
14704 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
14705 handler
14706 .forwarding
14707 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
14708 .unwrap()
14709 else {
14710 panic!("the acknowledged route must be bound");
14711 };
14712 RouteClosedFixture {
14713 handler,
14714 #[cfg(unix)]
14715 supervisor,
14716 module_ctx,
14717 #[cfg(unix)]
14718 module_rx,
14719 client_rx,
14720 route,
14721 }
14722 }
14723
14724 impl RouteClosedFixture {
14725 #[cfg(unix)]
14726 async fn cut(&mut self) {
14727 let notices = async {
14728 let mut draining = self.module_rx.recv().await.unwrap();
14729 assert!(matches!(
14730 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
14731 &draining.body
14732 )
14733 .unwrap(),
14734 subc_protocol::session::ModuleControlCommand::Draining {
14735 reason: RouteCloseReason::Restart,
14736 ..
14737 }
14738 ));
14739 draining.flushed.take().unwrap().send(()).unwrap();
14740 let mut closing = self.client_rx.recv().await.unwrap();
14741 assert!(matches!(
14742 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
14743 ClientControlPush::RouteClosing {
14744 reason: RouteCloseReason::Restart,
14745 ..
14746 }
14747 ));
14748 closing.flushed.take().unwrap().send(()).unwrap();
14749 };
14750 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
14751 result.unwrap();
14752 }
14753
14754 fn disconnect(&mut self) -> ClientControlPush {
14755 self.handler
14756 .cleanup_connection(self.module_ctx.connection_id)
14757 .unwrap();
14758 let closed = self.client_rx.try_recv().unwrap();
14759 assert_eq!(closed.header.ty, FrameType::Push);
14760 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
14761 assert!(matches!(
14762 &push,
14763 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
14764 if channels == &[self.route.client_channel]
14765 ));
14766 assert_eq!(
14767 self.client_rx.try_recv().unwrap().header.ty,
14768 FrameType::Goodbye
14769 );
14770 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
14771 push
14772 }
14773 }
14774
14775 #[cfg(unix)]
14776 #[tokio::test(start_paused = true)]
14777 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
14778 let mut fixture = route_closed_fixture().await;
14779 fixture.cut().await;
14780 assert!(matches!(
14781 fixture.disconnect(),
14782 ClientControlPush::RouteClosed {
14783 reason: RouteCloseReason::Restart,
14784 drained: true,
14785 ..
14786 }
14787 ));
14788 }
14789
14790 #[cfg(unix)]
14791 #[tokio::test(start_paused = true)]
14792 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
14793 let mut fixture = route_closed_fixture().await;
14794 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
14795 let started = tokio::time::Instant::now();
14796 fixture.cut().await;
14797 assert!(started.elapsed() >= Duration::from_secs(2));
14798 assert_eq!(fixture.route.flow.in_flight(), 1);
14799 assert!(matches!(
14800 fixture.disconnect(),
14801 ClientControlPush::RouteClosed {
14802 reason: RouteCloseReason::Restart,
14803 drained: false,
14804 ..
14805 }
14806 ));
14807 }
14808
14809 #[tokio::test]
14810 async fn module_connection_crash_reports_crash_and_not_drained() {
14811 let mut fixture = route_closed_fixture().await;
14812 assert!(matches!(
14816 fixture.disconnect(),
14817 ClientControlPush::RouteClosed {
14818 reason: RouteCloseReason::Crash,
14819 drained: false,
14820 ..
14821 }
14822 ));
14823 }
14824
14825 #[tokio::test]
14826 async fn dropping_router_connection_releases_registration() {
14827 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14828 let registry = Arc::new(Registry::default());
14829 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
14830 let router = Router::with_control_handler(Arc::clone(&control));
14831 let connection = router.begin_connection();
14832 let connection_id = connection.id();
14833 let (ctx, mut rx) = route_ctx(connection_id);
14834
14835 router
14836 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
14837 .await
14838 .unwrap();
14839 let response = rx.recv().await.unwrap();
14840 let ack = parse_ack(&response);
14841 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14842 assert!(registry.get_module("aft").unwrap().is_some());
14843
14844 drop(connection);
14845
14846 assert!(registry.get_module("aft").unwrap().is_none());
14847 assert_eq!(registry.active_registration_count().unwrap(), 0);
14848
14849 control
14850 .cleanup_connection(ConnectionId::new(u64::MAX))
14851 .unwrap();
14852 let captured = crate::router::test_log::captured_logs(&logs);
14853 println!("captured registry lifecycle logs:\n{captured}");
14854 let events: Vec<_> = captured
14855 .lines()
14856 .filter(|line| {
14857 line.contains("module registered module_id=aft ")
14858 || line.contains("module registration ended")
14859 })
14860 .collect();
14861 assert_eq!(
14862 events.len(),
14863 2,
14864 "unexpected registry lifecycle logs: {captured}"
14865 );
14866 assert!(events[0].contains("module registered module_id=aft "));
14867 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
14868 assert!(events[1].contains(&format!(
14869 "module_id=aft connection_id={}",
14870 connection_id.get()
14871 )));
14872 assert!(events[1].contains("reason=connection_closed"));
14873 assert!(events[1].contains("module registration ended"));
14874 assert_eq!(
14875 captured
14876 .lines()
14877 .filter(|line| line.contains("module registered module_id=aft "))
14878 .count(),
14879 1,
14880 "legacy registration admission line must appear once: {captured}"
14881 );
14882 }
14883
14884 #[tokio::test]
14885 async fn hello_registration_keeps_legacy_module_registered_line_once() {
14886 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14887 let registry = Arc::new(Registry::default());
14888 let control = ControlHandler::new(Arc::clone(®istry));
14889 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
14890 hello_via_sink(
14891 &control,
14892 &ctx,
14893 &mut rx,
14894 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
14895 )
14896 .await;
14897
14898 let captured = crate::router::test_log::captured_logs(&logs);
14899 let admissions: Vec<_> = captured
14900 .lines()
14901 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
14902 .collect();
14903 assert_eq!(
14904 admissions.len(),
14905 1,
14906 "legacy admission line must remain exactly once: {captured}"
14907 );
14908 assert!(admissions[0].contains("routable_provider=true"));
14909 assert!(admissions[0].contains("connection_id=777"));
14910 }
14911
14912 fn capability_manifest(
14913 module_id: &str,
14914 provides: &[&str],
14915 must_never_reach: &[&str],
14916 ) -> ModuleManifest {
14917 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
14918 manifest.capabilities = Some(CapabilityDeclarations {
14919 provides: provides
14920 .iter()
14921 .map(|capability| (*capability).to_string())
14922 .collect(),
14923 requires: Vec::new(),
14924 must_never_reach: must_never_reach
14925 .iter()
14926 .map(|capability| (*capability).to_string())
14927 .collect(),
14928 });
14929 manifest
14930 }
14931
14932 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
14933 Frame::build(
14934 FrameType::Hello,
14935 control_flags(),
14936 0,
14937 0,
14938 corr,
14939 serde_json::to_vec(&ModuleHelloBody {
14940 protocol_ver: manifest.protocol_ver,
14941 manifest,
14942 control_ops: None,
14943 launch_nonce: None,
14944 })
14945 .expect("capability test HELLO serializes"),
14946 )
14947 .expect("capability test HELLO frame builds")
14948 }
14949
14950 fn catalog_update_with_capabilities_frame(
14951 corr: u64,
14952 capabilities: CapabilityDeclarations,
14953 ) -> Frame {
14954 Frame::build(
14955 FrameType::Request,
14956 control_flags(),
14957 0,
14958 0,
14959 corr,
14960 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14961 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
14962 capabilities: Some(capabilities),
14963 ready: None,
14964 })
14965 .expect("capability catalog.update serializes"),
14966 )
14967 .expect("capability catalog.update frame builds")
14968 }
14969
14970 async fn register_capability_manifest(
14971 handler: &ControlHandler,
14972 ctx: &RouteCtx,
14973 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14974 manifest: ModuleManifest,
14975 corr: u64,
14976 ) {
14977 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
14978 }
14979
14980 async fn open_route_for_capability_test(
14981 handler: &ControlHandler,
14982 target_ctx: &RouteCtx,
14983 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14984 client_connection_id: u64,
14985 corr: u64,
14986 target_module_id: &str,
14987 consumer_identity: Option<ConsumerIdentity>,
14988 ) -> (
14989 mpsc::Receiver<crate::router::OutboundFrame>,
14990 ModuleControlRequest,
14991 ) {
14992 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
14993 let route_handler = handler.clone();
14994 let target_module_id = target_module_id.to_string();
14995 let route_task = tokio::spawn(async move {
14996 route_handler
14997 .handle_control_frame(
14998 &client_ctx,
14999 route_open_frame_with_admission_facts(
15000 corr,
15001 &target_module_id,
15002 unique_project_root("admission-facts"),
15003 consumer_identity,
15004 None,
15005 ),
15006 )
15007 .await
15008 .expect("capability test route.open succeeds")
15009 });
15010 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
15011 .await
15012 .expect("capability test route.open must reach route.bind")
15013 .expect("target control receiver stays open");
15014 let bind_request: ModuleControlRequest =
15015 serde_json::from_slice(&bind.body).expect("route.bind decodes");
15016 handler
15017 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
15018 .await
15019 .expect("capability test route.bind ACK succeeds");
15020 assert!(route_task.await.expect("route.open task joins").is_empty());
15021 let opened = client_rx
15022 .recv()
15023 .await
15024 .expect("successful route.open publishes a response");
15025 assert!(matches!(
15026 serde_json::from_slice::<ClientControlResponse>(&opened.body),
15027 Ok(ClientControlResponse::RouteOpen { .. })
15028 ));
15029 (client_rx, bind_request)
15030 }
15031
15032 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
15033 assert_eq!(frame.header.ty, FrameType::Push);
15034 assert_eq!(frame.header.channel, 0);
15035 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
15036 .expect("route.closed control push decodes");
15037 let ClientControlPush::RouteClosed { channels, .. } = &push else {
15038 panic!("expected route.closed");
15039 };
15040 assert_eq!(channels.len(), 1, "exactly one violating route closed");
15041 let closed_channel = channels[0];
15042 let channels = channels.clone();
15043 assert_eq!(
15044 push,
15045 ClientControlPush::RouteClosed {
15046 module_id: target_module_id.to_string(),
15047 channels,
15048 reason: RouteCloseReason::CapabilityDenied,
15049 drained: false,
15050 abandoned: 0,
15051 excluded_subscriptions: 0,
15052 terminal: Some(false),
15053 }
15054 );
15055 closed_channel
15056 }
15057
15058 #[tokio::test]
15059 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
15060 let registry = Arc::new(Registry::default());
15061 let forwarding = Arc::new(ForwardingTable::default());
15062 let supervisor = SupervisorHandle::new();
15063 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15064 let handler =
15065 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15066 .with_supervisor(supervisor);
15067 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
15068 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
15069 register_capability_manifest(
15070 &handler,
15071 &target_ctx,
15072 &mut target_rx,
15073 capability_manifest("target", &["credentials-provider/v1"], &[]),
15074 1,
15075 )
15076 .await;
15077 register_capability_manifest(
15078 &handler,
15079 &opener_ctx,
15080 &mut opener_rx,
15081 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15082 2,
15083 )
15084 .await;
15085
15086 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
15087 let replies = handler
15088 .handle_control_frame(
15089 &client_ctx,
15090 route_open_frame_with_admission_facts(
15091 3,
15092 "target",
15093 unique_project_root("admission-facts"),
15094 Some(ConsumerIdentity {
15095 module_id: "opener".to_string(),
15096 launch_nonce: "opener-nonce".to_string(),
15097 }),
15098 None,
15099 ),
15100 )
15101 .await
15102 .expect("denied route.open returns a typed frame");
15103 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15104 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15105 assert!(
15106 target_rx.try_recv().is_err(),
15107 "forbidden route.open must not relay route.bind"
15108 );
15109 }
15110
15111 #[tokio::test]
15112 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
15113 let registry = Arc::new(Registry::default());
15114 let forwarding = Arc::new(ForwardingTable::default());
15115 let supervisor = SupervisorHandle::new();
15116 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15117 let handler =
15118 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15119 .with_supervisor(supervisor);
15120 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
15121 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
15122 register_capability_manifest(
15123 &handler,
15124 &target_ctx,
15125 &mut target_rx,
15126 capability_manifest("target", &["credentials-provider/v1"], &[]),
15127 1,
15128 )
15129 .await;
15130 register_capability_manifest(
15131 &handler,
15132 &old_opener_ctx,
15133 &mut old_opener_rx,
15134 capability_manifest("opener", &[], &[]),
15135 2,
15136 )
15137 .await;
15138 let (mut client_rx, _) = open_route_for_capability_test(
15139 &handler,
15140 &target_ctx,
15141 &mut target_rx,
15142 712,
15143 3,
15144 "target",
15145 Some(ConsumerIdentity {
15146 module_id: "opener".to_string(),
15147 launch_nonce: "opener-nonce".to_string(),
15148 }),
15149 )
15150 .await;
15151 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15152
15153 handler
15154 .cleanup_connection(old_opener_ctx.connection_id)
15155 .expect("old opener registration cleans up");
15156 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
15157 register_capability_manifest(
15158 &handler,
15159 &new_opener_ctx,
15160 &mut new_opener_rx,
15161 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15162 4,
15163 )
15164 .await;
15165
15166 let closed_channel = assert_capability_denied_push(
15167 client_rx
15168 .try_recv()
15169 .expect("HELLO deny addition must emit route.closed")
15170 .frame,
15171 "target",
15172 );
15173 let goodbye = client_rx
15174 .try_recv()
15175 .expect("HELLO deny addition must send the client a route GOODBYE")
15176 .frame;
15177 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15178 assert_eq!(goodbye.header.channel, closed_channel);
15179 assert_ne!(goodbye.header.channel, 0);
15180 assert_ne!(goodbye.header.epoch, 0);
15181 assert!(client_rx.try_recv().is_err());
15182 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15183 assert!(matches!(
15184 target_rx.try_recv(),
15185 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15186 ));
15187 }
15188
15189 #[tokio::test]
15190 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
15191 let registry = Arc::new(Registry::default());
15192 let forwarding = Arc::new(ForwardingTable::default());
15193 let supervisor = SupervisorHandle::new();
15194 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15195 let handler =
15196 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15197 .with_supervisor(supervisor);
15198 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
15199 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
15200 register_capability_manifest(
15201 &handler,
15202 &target_ctx,
15203 &mut target_rx,
15204 capability_manifest("target", &[], &[]),
15205 1,
15206 )
15207 .await;
15208 register_capability_manifest(
15209 &handler,
15210 &opener_ctx,
15211 &mut opener_rx,
15212 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15213 2,
15214 )
15215 .await;
15216 let (mut client_rx, _) = open_route_for_capability_test(
15217 &handler,
15218 &target_ctx,
15219 &mut target_rx,
15220 722,
15221 3,
15222 "target",
15223 Some(ConsumerIdentity {
15224 module_id: "opener".to_string(),
15225 launch_nonce: "opener-nonce".to_string(),
15226 }),
15227 )
15228 .await;
15229 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15230
15231 let replies = handler
15232 .handle_control_frame(
15233 &target_ctx,
15234 catalog_update_with_capabilities_frame(
15235 4,
15236 CapabilityDeclarations {
15237 provides: vec!["credentials-provider/v1".to_string()],
15238 requires: Vec::new(),
15239 must_never_reach: Vec::new(),
15240 },
15241 ),
15242 )
15243 .await
15244 .expect("claim catalog.update succeeds");
15245 assert!(matches!(
15246 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
15247 Ok(ModuleControlResponseToModule::CatalogUpdate {})
15248 ));
15249 let closed_channel = assert_capability_denied_push(
15250 client_rx
15251 .try_recv()
15252 .expect("claim addition must emit route.closed")
15253 .frame,
15254 "target",
15255 );
15256 let goodbye = client_rx
15257 .try_recv()
15258 .expect("claim addition must send the client a route GOODBYE")
15259 .frame;
15260 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15261 assert_eq!(goodbye.header.channel, closed_channel);
15262 assert_ne!(goodbye.header.channel, 0);
15263 assert_ne!(goodbye.header.epoch, 0);
15264 assert!(client_rx.try_recv().is_err());
15265 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15266 assert!(matches!(
15267 target_rx.try_recv(),
15268 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15269 ));
15270 }
15271
15272 #[tokio::test]
15273 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
15274 let registry = Arc::new(Registry::default());
15275 let forwarding = Arc::new(ForwardingTable::default());
15276 let supervisor = SupervisorHandle::new();
15277 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15278 let handler =
15279 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15280 .with_supervisor(supervisor);
15281 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
15282 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
15283 register_capability_manifest(
15284 &handler,
15285 &target_ctx,
15286 &mut target_rx,
15287 capability_manifest("target", &["credentials-provider/v1"], &[]),
15288 1,
15289 )
15290 .await;
15291 register_capability_manifest(
15292 &handler,
15293 &opener_ctx,
15294 &mut opener_rx,
15295 capability_manifest("opener", &[], &[]),
15296 2,
15297 )
15298 .await;
15299 let (mut client_rx, _) = open_route_for_capability_test(
15300 &handler,
15301 &target_ctx,
15302 &mut target_rx,
15303 732,
15304 3,
15305 "target",
15306 Some(ConsumerIdentity {
15307 module_id: "opener".to_string(),
15308 launch_nonce: "opener-nonce".to_string(),
15309 }),
15310 )
15311 .await;
15312 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15313
15314 handler
15315 .handle_control_frame(
15316 &target_ctx,
15317 catalog_update_with_capabilities_frame(
15318 4,
15319 CapabilityDeclarations {
15320 provides: Vec::new(),
15321 requires: Vec::new(),
15322 must_never_reach: Vec::new(),
15323 },
15324 ),
15325 )
15326 .await
15327 .expect("claim removal catalog.update succeeds");
15328 assert_eq!(
15329 forwarding.active_binding_count().unwrap(),
15330 1,
15331 "removing an attested target claim must leave the route census unchanged"
15332 );
15333 assert!(
15334 client_rx.try_recv().is_err(),
15335 "claim removal must not emit route.closed or a client route GOODBYE"
15336 );
15337 assert!(
15338 target_rx.try_recv().is_err(),
15339 "claim removal must not send the target a route GOODBYE"
15340 );
15341 }
15342
15343 #[tokio::test]
15346 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
15347 let registry = Arc::new(Registry::default());
15348 let forwarding = Arc::new(ForwardingTable::default());
15349 let supervisor = SupervisorHandle::new();
15350 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15351 let handler =
15352 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15353 .with_supervisor(supervisor);
15354 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
15355 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
15356 register_capability_manifest(
15357 &handler,
15358 &target_ctx,
15359 &mut target_rx,
15360 capability_manifest("target", &["credentials-provider/v1"], &[]),
15361 1,
15362 )
15363 .await;
15364 register_capability_manifest(
15365 &handler,
15366 &opener_ctx,
15367 &mut opener_rx,
15368 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15369 2,
15370 )
15371 .await;
15372
15373 let (_client_rx, bind) = open_route_for_capability_test(
15374 &handler,
15375 &target_ctx,
15376 &mut target_rx,
15377 742,
15378 3,
15379 "target",
15380 None,
15381 )
15382 .await;
15383 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
15384 panic!("direct scope-honesty route must bind");
15385 };
15386 assert_eq!(principal, Some(Principal::Direct));
15387 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15388 }
15389
15390 #[tokio::test]
15393 async fn must_never_reach_self_route_is_capability_forbidden() {
15394 let registry = Arc::new(Registry::default());
15395 let forwarding = Arc::new(ForwardingTable::default());
15396 let supervisor = SupervisorHandle::new();
15397 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
15398 let handler =
15399 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15400 .with_supervisor(supervisor);
15401 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
15402 register_capability_manifest(
15403 &handler,
15404 &self_ctx,
15405 &mut self_rx,
15406 capability_manifest(
15407 "self-provider",
15408 &["credentials-provider/v1"],
15409 &["credentials-provider/v1"],
15410 ),
15411 1,
15412 )
15413 .await;
15414
15415 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
15416 let replies = handler
15417 .handle_control_frame(
15418 &client_ctx,
15419 route_open_frame_with_admission_facts(
15420 2,
15421 "self-provider",
15422 unique_project_root("admission-facts"),
15423 Some(ConsumerIdentity {
15424 module_id: "self-provider".to_string(),
15425 launch_nonce: "self-nonce".to_string(),
15426 }),
15427 None,
15428 ),
15429 )
15430 .await
15431 .expect("self-route refusal returns a typed frame");
15432 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15433 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15434 assert!(
15435 self_rx.try_recv().is_err(),
15436 "self denial must not relay route.bind"
15437 );
15438 }
15439
15440 #[test]
15441 fn unsupported_channel_zero_frame_returns_error() {
15442 let handler = ControlHandler::default();
15443 let request = Frame::build(
15444 FrameType::Request,
15445 control_flags(),
15446 0,
15447 0,
15448 21,
15449 b"opaque".to_vec(),
15450 )
15451 .unwrap();
15452
15453 let response = handler
15454 .handle_control(ConnectionId::new(1), request)
15455 .unwrap();
15456
15457 assert_eq!(response[0].header.ty, FrameType::Error);
15458 assert_eq!(
15459 parse_error(&response[0])["code"],
15460 "unsupported_control_frame"
15461 );
15462 }
15463
15464 mod swap {
15469 use super::*;
15470
15471 const INCUMBENT: ConnectionId = ConnectionId::new(30);
15472 const CANDIDATE: ConnectionId = ConnectionId::new(40);
15473
15474 struct Swap {
15475 registry: Arc<Registry>,
15476 forwarding: Arc<ForwardingTable>,
15477 handler: ControlHandler,
15478 incumbent_ctx: RouteCtx,
15479 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15480 candidate_ctx: RouteCtx,
15481 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15482 }
15483
15484 async fn swap_with_incumbent() -> Swap {
15485 let registry = Arc::new(Registry::default());
15486 let forwarding = Arc::new(ForwardingTable::default());
15487 let handler =
15488 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15489 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
15490 hello_via_sink(
15491 &handler,
15492 &incumbent_ctx,
15493 &mut incumbent_rx,
15494 hello_frame("aft", PROTOCOL_VERSION, 7),
15495 )
15496 .await;
15497 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
15498 Swap {
15499 registry,
15500 forwarding,
15501 handler,
15502 incumbent_ctx,
15503 incumbent_rx,
15504 candidate_ctx,
15505 candidate_rx,
15506 }
15507 }
15508
15509 fn register_candidate(swap: &Swap, ready: Option<bool>) {
15510 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
15511 candidate_manifest.ready = ready;
15512 let registration = swap
15513 .registry
15514 .register_candidate_with_control_ops(
15515 candidate_manifest,
15516 PROTOCOL_VERSION,
15517 CANDIDATE,
15518 module_baseline_control_ops(),
15519 )
15520 .unwrap();
15521 swap.forwarding
15522 .register_candidate_module_connection(
15523 CANDIDATE,
15524 "aft".to_string(),
15525 PROTOCOL_VERSION,
15526 manifest_concurrency(®istration.manifest),
15527 swap.candidate_ctx.egress.clone(),
15528 )
15529 .unwrap();
15530 }
15531
15532 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
15533 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
15534 swap.registry.promote_candidate("aft").unwrap().unwrap();
15535 cutover.incumbent.unwrap()
15536 }
15537
15538 #[tokio::test]
15547 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
15548 let mut swap = swap_with_incumbent().await;
15549 let hold = Arc::new(std::sync::Barrier::new(2));
15550 let handler = swap
15551 .handler
15552 .clone()
15553 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
15554 handler
15555 .supervisor
15556 .open_swap("aft", "candidate-nonce".to_string());
15557 let hello = {
15558 let handler = handler.clone();
15559 let candidate_ctx = swap.candidate_ctx.clone();
15560 std::thread::spawn(move || {
15561 handler
15562 .handle_hello(
15563 candidate_ctx.connection_id,
15564 Some(candidate_ctx.egress.clone()),
15565 hello_frame_with_nonce(
15566 "aft",
15567 PROTOCOL_VERSION,
15568 8,
15569 Some("candidate-nonce"),
15570 ),
15571 )
15572 .unwrap()
15573 })
15574 };
15575
15576 hold.wait();
15577 assert!(swap
15580 .registry
15581 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
15582 .unwrap()
15583 .is_some());
15584 assert!(swap
15585 .registry
15586 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15587 .unwrap()
15588 .is_none());
15589 assert!(swap
15590 .forwarding
15591 .module_endpoint_for_connection(CANDIDATE)
15592 .unwrap()
15593 .is_none());
15594
15595 hold.wait();
15596 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
15597 assert_eq!(
15598 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
15599 FrameType::HelloAck
15600 );
15601 let published = swap
15602 .registry
15603 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15604 .unwrap()
15605 .expect("candidate is published once its HELLO completes");
15606 assert_eq!(published.connection_id, CANDIDATE);
15607 assert!(swap
15608 .forwarding
15609 .module_endpoint_for_connection(CANDIDATE)
15610 .unwrap()
15611 .is_some());
15612 }
15613
15614 fn keyed_total(counters: &Value, key: &str) -> u64 {
15615 counters[key]
15616 .as_object()
15617 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
15618 .unwrap_or(0)
15619 }
15620
15621 #[tokio::test]
15622 async fn replacement_logs_old_end_and_new_admission_once() {
15623 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15624 let mut swap = swap_with_incumbent().await;
15625 swap.handler
15626 .supervisor
15627 .open_swap("aft", "candidate-nonce".to_string());
15628 hello_via_sink(
15629 &swap.handler,
15630 &swap.candidate_ctx,
15631 &mut swap.candidate_rx,
15632 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
15633 )
15634 .await;
15635 cutover(&swap);
15636 swap.handler.cleanup_connection(INCUMBENT).unwrap();
15637
15638 let captured = crate::router::test_log::captured_logs(&logs);
15639 println!("captured replacement registry logs:\n{captured}");
15640 let new_admissions: Vec<_> = captured
15641 .lines()
15642 .filter(|line| {
15643 line.contains("connection_id=40")
15644 && line.contains("swap candidate registered; not routable until cutover")
15645 })
15646 .collect();
15647 assert_eq!(
15648 new_admissions.len(),
15649 1,
15650 "unexpected admission logs: {captured}"
15651 );
15652 assert!(new_admissions[0].contains("module_id=aft"));
15653 assert!(new_admissions[0].contains("connection_id=40"));
15654 assert!(
15655 new_admissions[0].contains("swap candidate registered; not routable until cutover")
15656 );
15657 assert!(new_admissions[0].contains("ready=true"));
15658
15659 let old_admissions: Vec<_> = captured
15660 .lines()
15661 .filter(|line| {
15662 line.contains("module registered module_id=aft ")
15663 && line.contains("connection_id=30")
15664 })
15665 .collect();
15666 assert_eq!(
15667 old_admissions.len(),
15668 1,
15669 "unexpected incumbent admission: {captured}"
15670 );
15671
15672 let old_ends: Vec<_> = captured
15673 .lines()
15674 .filter(|line| {
15675 line.contains("connection_id=30") && line.contains("module registration ended")
15676 })
15677 .collect();
15678 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
15679 assert!(old_ends[0].contains("module_id=aft"));
15680 assert!(old_ends[0].contains("connection_id=30"));
15681 assert!(old_ends[0].contains("reason=replaced"));
15682 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
15683 assert_eq!(
15684 captured
15685 .lines()
15686 .filter(|line| line.contains("module registration promoted"))
15687 .count(),
15688 1,
15689 "unexpected promotion logs: {captured}"
15690 );
15691 }
15692
15693 #[tokio::test]
15699 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
15700 let mut swap = swap_with_incumbent().await;
15701 let handler = swap.handler.clone();
15702
15703 let cotenant = ConnectionId::new(31);
15705 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
15706 let (cotenant_task, cotenant_bind) = relay_route_open(
15707 &handler,
15708 cotenant,
15709 &cotenant_ctx.egress,
15710 &mut swap.incumbent_rx,
15711 100,
15712 "aft",
15713 "swap-cotenant",
15714 )
15715 .await;
15716 handler
15717 .handle_control_frame(
15718 &swap.incumbent_ctx,
15719 route_bind_ack(cotenant_bind.header.corr),
15720 )
15721 .await
15722 .unwrap();
15723 assert!(cotenant_task.await.unwrap().is_empty());
15724 let (cotenant_channel, cotenant_epoch) =
15725 published_route(&cotenant_rx.recv().await.unwrap());
15726
15727 let caller = ConnectionId::new(32);
15729 let (caller_ctx, mut caller_rx) = route_ctx(caller);
15730 let (caller_task, caller_bind) = relay_route_open(
15731 &handler,
15732 caller,
15733 &caller_ctx.egress,
15734 &mut swap.incumbent_rx,
15735 101,
15736 "aft",
15737 "swap-caller",
15738 )
15739 .await;
15740 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
15741
15742 register_candidate(&swap, None);
15743 cutover(&swap);
15744
15745 let ack = handler
15747 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
15748 .await;
15749 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
15750 if module_loop_error.is_some() {
15751 handler.cleanup_connection(INCUMBENT).unwrap();
15754 }
15755
15756 assert!(
15758 cotenant_rx.try_recv().is_err(),
15759 "the co-tenant route on the incumbent was torn down by one late ack: \
15760 {module_loop_error:?}"
15761 );
15762 assert!(matches!(
15763 swap.forwarding
15764 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
15765 .unwrap(),
15766 DataRoute::Client(DataRouteState::Bound(_))
15767 ));
15768 assert_eq!(module_loop_error, None);
15769 assert!(swap
15770 .registry
15771 .get_module_by_connection(INCUMBENT)
15772 .unwrap()
15773 .is_some());
15774
15775 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
15777 .await
15778 .expect("the incumbent is told to drop the abandoned binding")
15779 .unwrap()
15780 .frame;
15781 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15782 assert_eq!(goodbye.header.channel, abandoned_channel);
15783 assert_eq!(goodbye.header.epoch, abandoned_epoch);
15784 assert!(swap.incumbent_rx.try_recv().is_err());
15785
15786 let response = caller_task.await.unwrap();
15788 assert_eq!(response.len(), 1);
15789 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15790 assert!(caller_rx.try_recv().is_err());
15791
15792 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
15796 let counters = handler.counters().snapshot();
15797 assert_eq!(
15798 keyed_total(&counters, "route_open_accepted_by_principal"),
15799 1
15800 );
15801 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
15802 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
15803 }
15804
15805 #[tokio::test]
15809 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
15810 let mut swap = swap_with_incumbent().await;
15811 register_candidate(&swap, None);
15812 let incumbent = cutover(&swap);
15813 swap.forwarding
15814 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
15815 .unwrap()
15816 .expect("the incumbent is still registered");
15817
15818 let client = ConnectionId::new(33);
15819 let (client_ctx, mut client_rx) = route_ctx(client);
15820 let route_handler = swap.handler.clone();
15821 let open_ctx = RouteCtx {
15822 connection_id: client,
15823 egress: client_ctx.egress.clone(),
15824 };
15825 let mut route_task = tokio::spawn(async move {
15826 route_handler
15827 .handle_control_frame(
15828 &open_ctx,
15829 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
15830 )
15831 .await
15832 .unwrap()
15833 });
15834 let bind = tokio::select! {
15835 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
15836 response = &mut route_task => {
15837 let response = response.unwrap();
15838 panic!(
15839 "post-cutover route.open was refused instead of relayed to the candidate: {}",
15840 parse_error(&response[0])["code"]
15841 );
15842 }
15843 };
15844 swap.handler
15845 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
15846 .await
15847 .unwrap();
15848 assert!(route_task.await.unwrap().is_empty());
15849 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
15850 match swap
15851 .forwarding
15852 .lookup_data_route(client, channel, epoch)
15853 .unwrap()
15854 {
15855 DataRoute::Client(DataRouteState::Bound(route)) => {
15856 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
15857 }
15858 other => panic!("expected a bound route on the candidate, got {other:?}"),
15859 }
15860 assert!(swap.incumbent_rx.try_recv().is_err());
15861 }
15862
15863 #[tokio::test]
15868 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
15869 let swap = swap_with_incumbent().await;
15870 register_candidate(&swap, Some(false));
15871 let update = Frame::build(
15872 FrameType::Request,
15873 control_flags(),
15874 0,
15875 0,
15876 55,
15877 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
15878 provides: manifest("aft", PROTOCOL_VERSION).provides,
15879 capabilities: None,
15880 ready: Some(true),
15881 })
15882 .unwrap(),
15883 )
15884 .unwrap();
15885
15886 let replies = swap
15887 .handler
15888 .handle_control_frame(&swap.candidate_ctx, update)
15889 .await
15890 .unwrap();
15891
15892 assert_eq!(replies.len(), 1);
15893 assert_eq!(
15894 replies[0].header.ty,
15895 FrameType::Response,
15896 "candidate catalog.update was refused: {:?}",
15897 serde_json::from_slice::<Value>(&replies[0].body).ok()
15898 );
15899 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
15900 assert_eq!(
15901 swap.registry
15902 .get_module("aft")
15903 .unwrap()
15904 .unwrap()
15905 .connection_id,
15906 INCUMBENT
15907 );
15908 }
15909 }
15910
15911 mod swap_admission {
15915 use super::*;
15916
15917 const INCUMBENT_NONCE: &str = "incumbent-nonce";
15918 const CANDIDATE_NONCE: &str = "candidate-nonce";
15919
15920 fn handler_with_incumbent(
15921 module_id: &str,
15922 reserved: bool,
15923 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
15924 let registry = Arc::new(Registry::default());
15925 let supervisor = SupervisorHandle::new();
15926 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
15927 if reserved {
15928 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
15929 }
15930 let handler =
15931 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
15932 let incumbent = handler
15933 .handle_control(
15934 ConnectionId::new(1),
15935 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
15936 )
15937 .unwrap();
15938 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
15939 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
15940 (registry, supervisor, handler)
15941 }
15942
15943 #[test]
15949 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
15950 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
15951
15952 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
15953 let replies = handler
15954 .handle_control(
15955 ConnectionId::new(connection),
15956 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
15957 )
15958 .unwrap();
15959 assert_eq!(replies[0].header.ty, FrameType::Error);
15960 assert_eq!(
15961 parse_error(&replies[0])["code"],
15962 "swap_token_invalid",
15963 "nonce {nonce:?}"
15964 );
15965 }
15966 assert!(registry.get_candidate("aft").unwrap().is_none());
15967 assert_eq!(
15968 registry.get_module("aft").unwrap().unwrap().connection_id,
15969 ConnectionId::new(1)
15970 );
15971
15972 let admitted = handler
15975 .handle_control(
15976 ConnectionId::new(4),
15977 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
15978 )
15979 .unwrap();
15980 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
15981 assert_eq!(
15982 registry
15983 .get_candidate("aft")
15984 .unwrap()
15985 .unwrap()
15986 .connection_id,
15987 ConnectionId::new(4)
15988 );
15989 assert_eq!(
15990 registry.get_module("aft").unwrap().unwrap().connection_id,
15991 ConnectionId::new(1),
15992 "the candidate must not take the active slot"
15993 );
15994 let replayed = handler
15995 .handle_control(
15996 ConnectionId::new(5),
15997 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
15998 )
15999 .unwrap();
16000 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
16001
16002 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
16006 let squatter = handler
16007 .handle_control(
16008 ConnectionId::new(6),
16009 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
16010 )
16011 .unwrap();
16012 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
16013 assert!(
16014 registry.get_module("aft").unwrap().is_none(),
16015 "a squatter took the active slot of an id being swapped"
16016 );
16017 }
16018
16019 #[test]
16024 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
16025 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
16026
16027 let replies = handler
16028 .handle_control(
16029 ConnectionId::new(2),
16030 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16031 )
16032 .unwrap();
16033
16034 assert_eq!(
16035 replies[0].header.ty,
16036 FrameType::HelloAck,
16037 "reserved candidate refused: {:?}",
16038 serde_json::from_slice::<Value>(&replies[0].body).ok()
16039 );
16040 assert_eq!(
16041 registry
16042 .get_candidate("vault")
16043 .unwrap()
16044 .unwrap()
16045 .connection_id,
16046 ConnectionId::new(2)
16047 );
16048 }
16049
16050 #[test]
16053 fn without_an_open_swap_the_ordinary_gates_decide() {
16054 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
16055 supervisor.close_swap("vault");
16056
16057 let candidate = handler
16058 .handle_control(
16059 ConnectionId::new(2),
16060 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16061 )
16062 .unwrap();
16063 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
16064 let duplicate = handler
16065 .handle_control(
16066 ConnectionId::new(3),
16067 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
16068 )
16069 .unwrap();
16070 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
16071 assert!(registry.get_candidate("vault").unwrap().is_none());
16072 }
16073 }
16074
16075 mod scopes {
16079 mod idle_tests {
16080 use super::*;
16081 include!("scope_idle_tests.rs");
16082 }
16083 mod route_open_timings {
16084 use super::*;
16085 include!("route_open_timing_tests.rs");
16086 }
16087 mod route_open_refusal_log {
16088 use super::*;
16089 include!("route_open_refusal_log_tests.rs");
16090 }
16091 mod activity {
16092 include!("activity_scope_tests.rs");
16093 }
16094 mod delta {
16095 include!("scope_handler_delta_tests.rs");
16096 }
16097 mod call_modes {
16098 include!("scope_call_mode_tests.rs");
16099 }
16100 #[tokio::test]
16101 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
16102 let rig = rig().await;
16103 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
16104 "s".into(),
16105 1,
16106 "tool".into(),
16107 ))
16108 .unwrap();
16109 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
16110 let (direct, _rx) = wide_ctx(999);
16111 for (ctx, expected) in [
16112 (&rig.owner, FrameType::Response),
16113 (&direct, FrameType::Error),
16114 ] {
16115 let frame =
16116 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
16117 .unwrap();
16118 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
16119 assert_eq!(replies[0].header.ty, expected);
16120 if expected == FrameType::Response {
16121 let reply: subc_control::ScopeServedByReply =
16122 serde_json::from_slice(&replies[0].body).unwrap();
16123 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
16124 } else {
16125 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
16126 }
16127 }
16128 let reply = call(
16129 &rig.handler,
16130 &rig.owner,
16131 &ModuleControlRequestFromModule::ScopeDescribe {
16132 owner: Principal::Reserved {
16133 module_id: OWNER.into(),
16134 },
16135 scope_ref: "s".into(),
16136 },
16137 )
16138 .await;
16139 assert!(matches!(
16140 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
16141 ModuleControlResponseToModule::ScopeDescribe { .. }
16142 ));
16143 }
16144
16145 use subc_protocol::scope::{
16146 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
16147 ScopeStatus,
16148 };
16149
16150 use super::*;
16151
16152 const OWNER: &str = "prefrontal-core";
16153
16154 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
16155 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
16156 }
16157
16158 async fn call(
16159 handler: &ControlHandler,
16160 ctx: &RouteCtx,
16161 request: &ModuleControlRequestFromModule,
16162 ) -> Frame {
16163 let body = serde_json::to_vec(request).unwrap();
16164 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
16165 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
16166 assert_eq!(replies.len(), 1, "{replies:?}");
16167 replies.pop().unwrap()
16168 }
16169
16170 async fn sync(
16171 handler: &ControlHandler,
16172 ctx: &RouteCtx,
16173 generation: u64,
16174 scopes: Vec<ScopeRecord>,
16175 ) -> Result<ModuleControlResponseToModule, String> {
16176 let reply = call(
16177 handler,
16178 ctx,
16179 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
16180 )
16181 .await;
16182 match reply.header.ty {
16183 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
16184 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
16185 }
16186 }
16187
16188 async fn describe(
16189 handler: &ControlHandler,
16190 ctx: &RouteCtx,
16191 owner: &str,
16192 scope_ref: &str,
16193 ) -> ModuleControlResponseToModule {
16194 let reply = call(
16195 handler,
16196 ctx,
16197 &ModuleControlRequestFromModule::ScopeDescribe {
16198 owner: Principal::Reserved {
16199 module_id: owner.to_string(),
16200 },
16201 scope_ref: scope_ref.to_string(),
16202 },
16203 )
16204 .await;
16205 assert_eq!(
16206 reply.header.ty,
16207 FrameType::Response,
16208 "{:?}",
16209 parse_error(&reply)
16210 );
16211 serde_json::from_slice(&reply.body).unwrap()
16212 }
16213
16214 #[tokio::test]
16215 async fn scope_describe_never_echoes_bind_proof_or_hash() {
16216 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16217 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16218 let mut rig = rig().await;
16219 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16220 .await;
16221 let _bound = rig
16222 .bound(
16223 Some(AFT),
16224 PLEXUS,
16225 Some(
16226 rig.selector("s", Some(1))
16227 .with_bind_proof(Some(PROOF.into())),
16228 ),
16229 )
16230 .await;
16231 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
16232 let output = serde_json::to_string(&reply).unwrap();
16233 assert!(output.contains("\"status\":\"live\""));
16234 assert!(!output.contains(PROOF));
16235 assert!(!output.contains(HASH));
16236 }
16237
16238 async fn module(
16240 handler: &ControlHandler,
16241 connection: u64,
16242 module_id: &str,
16243 nonce: Option<&str>,
16244 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16245 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
16246 hello_via_sink(
16247 handler,
16248 &ctx,
16249 &mut rx,
16250 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
16251 )
16252 .await;
16253 (ctx, rx)
16254 }
16255
16256 #[tokio::test]
16259 async fn a_client_connection_cannot_sync_or_describe() {
16260 let handler = ControlHandler::new(Arc::new(Registry::default()));
16261 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
16262 for request in [
16263 ModuleControlRequestFromModule::ScopeSync {
16264 generation: 1,
16265 scopes: vec![head("s", 1)],
16266 },
16267 ModuleControlRequestFromModule::ScopeDescribe {
16268 owner: Principal::Direct,
16269 scope_ref: "s".to_string(),
16270 },
16271 ] {
16272 let reply = call(&handler, &ctx, &request).await;
16273 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
16274 }
16275 assert!(
16276 !handler
16277 .scopes
16278 .read()
16279 .unwrap()
16280 .describe(
16281 &Principal::Reserved {
16282 module_id: OWNER.to_string()
16283 },
16284 "s"
16285 )
16286 .owner_synced
16287 );
16288 }
16289
16290 #[tokio::test]
16293 async fn a_module_without_a_supervised_launch_cannot_sync() {
16294 let handler = ControlHandler::new(Arc::new(Registry::default()));
16295 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
16296 assert_eq!(
16297 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
16298 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16299 );
16300 }
16301
16302 #[tokio::test]
16303 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
16304 let supervisor = SupervisorHandle::new();
16305 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16306 let handler = ControlHandler::new(Arc::new(Registry::default()))
16307 .with_supervisor(supervisor.clone());
16308 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16309 sync(&handler, &incumbent, 1, vec![head("s", 1)])
16310 .await
16311 .expect("the current launch syncs");
16312
16313 supervisor.open_swap(OWNER, "n2".to_string());
16316 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
16317 assert_eq!(
16318 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16319 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16320 );
16321 sync(&handler, &incumbent, 2, vec![head("s", 1)])
16322 .await
16323 .expect("the serving owner syncs during the swap");
16324
16325 supervisor.close_swap(OWNER);
16328 assert_eq!(
16329 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16330 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16331 );
16332 sync(&handler, &incumbent, 3, vec![head("s", 1)])
16333 .await
16334 .expect("the serving owner syncs after the rollback");
16335 handler.cleanup_connection(candidate.connection_id).unwrap();
16336
16337 supervisor.open_swap(OWNER, "n3".to_string());
16342 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
16343 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
16344 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
16345 .await
16346 .expect("the promoted launch takes authority");
16347 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16348 panic!("unexpected reply {reply:?}");
16349 };
16350 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
16351 assert_eq!(
16352 sync(&handler, &incumbent, 4, Vec::new()).await,
16353 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16354 );
16355 }
16356
16357 #[tokio::test]
16360 async fn closing_the_authority_connection_frees_sync_authority() {
16361 let supervisor = SupervisorHandle::new();
16362 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16363 let handler = ControlHandler::new(Arc::new(Registry::default()))
16364 .with_supervisor(supervisor.clone());
16365 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16366 sync(&handler, &first, 10, vec![head("s", 1)])
16367 .await
16368 .unwrap();
16369 handler.cleanup_connection(first.connection_id).unwrap();
16370
16371 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
16372 sync(&handler, &second, 1, vec![head("s", 1)])
16373 .await
16374 .expect("the next connection takes the released authority");
16375 }
16376
16377 #[tokio::test]
16378 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
16379 let supervisor = SupervisorHandle::new();
16380 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16381 let handler =
16382 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
16383 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
16384 sync(&handler, &first, 10, vec![head("s", 1)])
16385 .await
16386 .unwrap();
16387 handler
16388 .handle_control_frame(
16389 &first,
16390 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
16391 )
16392 .await
16393 .unwrap();
16394 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
16395 sync(&handler, &second, 1, vec![head("s", 1)])
16396 .await
16397 .expect("GOODBYE releases authority even if the old socket remains open");
16398 }
16399
16400 #[tokio::test]
16401 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
16402 let registry = Arc::new(Registry::default());
16403 let supervisor_handle = SupervisorHandle::new();
16404 let supervisor =
16405 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
16406 .with_handle(supervisor_handle.clone())
16407 .with_daemon_incarnation("incarnation-7".to_string());
16408 supervisor
16411 .supervise_configured(
16412 ModuleSpec {
16413 module_id: OWNER.to_string(),
16414 program: PathBuf::from("/nonexistent/prefrontal-core"),
16415 args: Vec::new(),
16416 env: Vec::new(),
16417 reserved: false,
16418 reserved_prefixes: Vec::new(),
16419 protocol: ModuleProtocol::Subc,
16420 overlap: Default::default(),
16421 },
16422 false,
16423 )
16424 .unwrap();
16425 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
16426 let handler =
16427 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
16428 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
16429
16430 let ModuleControlResponseToModule::ScopeDescribe {
16432 status,
16433 daemon_incarnation,
16434 owner_synced,
16435 owner_configured,
16436 scope,
16437 ..
16438 } = describe(&handler, &reader, OWNER, "s").await
16439 else {
16440 panic!("not a describe reply");
16441 };
16442 assert_eq!(status, ScopeStatus::NotLive);
16443 assert_eq!(daemon_incarnation, "incarnation-7");
16444 assert!(!owner_synced);
16445 assert!(owner_configured);
16446 assert!(scope.is_none());
16447
16448 let ModuleControlResponseToModule::ScopeDescribe {
16451 status,
16452 owner_configured,
16453 ..
16454 } = describe(&handler, &reader, "ghost", "s").await
16455 else {
16456 panic!("not a describe reply");
16457 };
16458 assert_eq!(status, ScopeStatus::NotLive);
16459 assert!(!owner_configured);
16460
16461 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
16463 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
16464 let ModuleControlResponseToModule::ScopeDescribe {
16465 status,
16466 scope_epoch,
16467 owner_synced,
16468 scope,
16469 ..
16470 } = describe(&handler, &reader, OWNER, "s").await
16471 else {
16472 panic!("not a describe reply");
16473 };
16474 assert_eq!(status, ScopeStatus::Live);
16475 assert_eq!(scope_epoch, Some(4));
16476 assert!(owner_synced);
16477 let stamp = scope.expect("a live scope carries its stamp");
16478 assert!(
16479 stamp.owner_authorized,
16480 "prefrontal-core is the default authority"
16481 );
16482 assert_eq!(stamp.kind, ScopeKind::Head);
16483 }
16484
16485 #[tokio::test]
16486 async fn scope_authority_owners_decides_owner_authorized() {
16487 let supervisor = SupervisorHandle::new();
16488 supervisor.set_spawn_nonce("broca", "b1".to_string());
16489 let handler = ControlHandler::new(Arc::new(Registry::default()))
16490 .with_supervisor(supervisor)
16491 .with_scope_authority_owners(vec!["broca".to_string()]);
16492 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
16493 let mut gated = head("s", 1);
16494 gated.attributes.agent_id = Some("agent".to_string());
16495 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
16496 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
16497 describe(&handler, &broca, "broca", "s").await
16498 else {
16499 panic!("not a describe reply");
16500 };
16501 assert!(scope.unwrap().owner_authorized);
16502 }
16503
16504 #[tokio::test]
16508 async fn scope_ops_and_the_scopes_capability_are_advertised() {
16509 let handler = ControlHandler::new(Arc::new(Registry::default()));
16510 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
16511 let ack = hello_via_sink(
16512 &handler,
16513 &ctx,
16514 &mut rx,
16515 hello_frame("m", PROTOCOL_VERSION, 1),
16516 )
16517 .await;
16518 let ack = parse_ack(&ack);
16519 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
16520 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
16521 }
16522 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
16523
16524 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
16525 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16526 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16527 let reply = handler
16528 .handle_control_frame(&client, frame)
16529 .await
16530 .unwrap()
16531 .pop()
16532 .unwrap();
16533 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16534 serde_json::from_slice(&reply.body).unwrap()
16535 else {
16536 panic!("not a server.describe reply");
16537 };
16538 assert!(
16539 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
16540 "{capabilities:?}"
16541 );
16542 }
16543
16544 const PLEXUS: &str = "plexus";
16547 const OTHER: &str = "other";
16548 const AFT: &str = "aft";
16549 const BROCA: &str = "broca";
16550 const MAGIC: &str = "magic-context";
16551
16552 fn nonce(module_id: &str) -> String {
16553 format!("nonce-{module_id}")
16554 }
16555
16556 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16557 let (tx, rx) = mpsc::channel(64);
16558 (
16559 RouteCtx {
16560 connection_id: ConnectionId::new(connection),
16561 egress: FrameSink::new(tx),
16562 },
16563 rx,
16564 )
16565 }
16566
16567 struct Rig {
16571 handler: ControlHandler,
16572 forwarding: Arc<ForwardingTable>,
16573 owner: RouteCtx,
16574 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16575 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
16576 generation: u64,
16577 next_connection: u64,
16578 _supervisor: Supervisor,
16579 }
16580
16581 async fn rig() -> Rig {
16582 rig_with_flow_support(true).await
16583 }
16584
16585 async fn rig_with_flow_support(flow_support: bool) -> Rig {
16586 rig_with_scope_capabilities(flow_support, false).await
16587 }
16588
16589 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
16590 let mut provides = Vec::new();
16591 if flow_support {
16592 provides.push("flow-scopes/v1");
16593 }
16594 if run_support {
16595 provides.push("agent-run-scopes/v1");
16596 }
16597 rig_with_target_capabilities(&provides).await
16598 }
16599
16600 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
16601 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
16602 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16603 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16606 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
16607 Frame::build(
16608 FrameType::Hello,
16609 control_flags(),
16610 0,
16611 0,
16612 connection,
16613 serde_json::to_vec(&body).unwrap(),
16614 )
16615 .unwrap()
16616 }
16617
16618 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
16619 let registry = Arc::new(Registry::default());
16620 let forwarding = Arc::new(ForwardingTable::default());
16621 let supervisor_handle = SupervisorHandle::new();
16622 let supervisor =
16623 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
16624 .with_handle(supervisor_handle.clone());
16625 supervisor
16626 .supervise_configured(
16627 ModuleSpec {
16628 module_id: OWNER.to_string(),
16629 program: PathBuf::from("/nonexistent/prefrontal-core"),
16630 args: Vec::new(),
16631 env: Vec::new(),
16632 reserved: false,
16633 reserved_prefixes: Vec::new(),
16634 protocol: ModuleProtocol::Subc,
16635 overlap: Default::default(),
16636 },
16637 false,
16638 )
16639 .unwrap();
16640 for module_id in [OWNER, AFT, BROCA, MAGIC] {
16641 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
16642 }
16643 let handler =
16644 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16645 .with_supervisor(supervisor_handle);
16646 let (owner, mut owner_rx) = wide_ctx(1);
16647 hello_via_sink(
16648 &handler,
16649 &owner,
16650 &mut owner_rx,
16651 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
16652 )
16653 .await;
16654 let mut modules = BTreeMap::new();
16655 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
16656 let (ctx, mut rx) = wide_ctx(connection);
16657 let hello = hello_with_capabilities(module_id, connection, provides);
16658 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
16659 modules.insert(module_id.to_string(), (ctx, rx));
16660 }
16661 Rig {
16662 handler,
16663 forwarding,
16664 owner,
16665 _owner_rx: owner_rx,
16666 modules,
16667 generation: 0,
16668 next_connection: 100,
16669 _supervisor: supervisor,
16670 }
16671 }
16672
16673 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
16674 ScopeCarrier::new(Principal::Reserved {
16675 module_id: module_id.to_string(),
16676 })
16677 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
16678 }
16679
16680 fn session(scope_epoch: u64) -> ScopeRecord {
16683 let mut record = head("s", scope_epoch);
16684 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
16685 record.attributes.agent_id = Some("agent-1".to_string());
16686 record.attributes.delegates = true;
16687 record
16688 }
16689
16690 impl Rig {
16691 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
16692 self.generation += 1;
16693 sync(&self.handler, &self.owner, self.generation, scopes)
16694 .await
16695 .expect("the owner's sync is accepted");
16696 }
16697
16698 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16699 ScopeSelector::new(
16700 Principal::Reserved {
16701 module_id: OWNER.to_string(),
16702 },
16703 scope_ref.to_string(),
16704 scope_epoch,
16705 )
16706 }
16707
16708 fn open_frame(
16709 &mut self,
16710 opener: Option<&str>,
16711 target: &str,
16712 scope: Option<ScopeSelector>,
16713 ) -> (
16714 RouteCtx,
16715 mpsc::Receiver<crate::router::OutboundFrame>,
16716 Frame,
16717 ) {
16718 self.next_connection += 1;
16719 let (ctx, rx) = wide_ctx(self.next_connection);
16720 let root = unique_project_root("scoped-open");
16721 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
16722 target: RouteTarget::ToolProvider {
16723 module_id: target.to_string(),
16724 },
16725 identity: BindIdentity::new(
16726 root.path().to_path_buf(),
16727 "unit".to_string(),
16728 "session".to_string(),
16729 ),
16730 consumer_identity: opener.map(|module_id| ConsumerIdentity {
16731 module_id: module_id.to_string(),
16732 launch_nonce: nonce(module_id),
16733 }),
16734 consumer_capabilities: None,
16735 role_versions: None,
16736 admission_facts: None,
16737 scope,
16738 })
16739 .unwrap();
16740 let frame = Frame::build(
16741 FrameType::Request,
16742 control_flags(),
16743 0,
16744 0,
16745 self.next_connection,
16746 body,
16747 )
16748 .unwrap();
16749 (ctx, rx, frame)
16750 }
16751
16752 async fn refused(
16754 &mut self,
16755 opener: Option<&str>,
16756 target: &str,
16757 scope: Option<ScopeSelector>,
16758 ) -> String {
16759 self.refusal_body(opener, target, scope).await["code"]
16760 .as_str()
16761 .unwrap()
16762 .to_string()
16763 }
16764
16765 async fn refusal_body(
16766 &mut self,
16767 opener: Option<&str>,
16768 target: &str,
16769 scope: Option<ScopeSelector>,
16770 ) -> Value {
16771 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
16772 let replies = tokio::time::timeout(
16773 Duration::from_secs(2),
16774 self.handler.handle_control_frame(&ctx, frame),
16775 )
16776 .await
16777 .expect("the open must be refused before waiting for a bind ack")
16778 .unwrap();
16779 assert_eq!(replies.len(), 1, "{replies:?}");
16780 assert_eq!(replies[0].header.ty, FrameType::Error);
16781 let (_, module_rx) = self.modules.get_mut(target).unwrap();
16782 assert!(
16783 module_rx.try_recv().is_err(),
16784 "a refused open relays nothing"
16785 );
16786 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
16787 parse_error(&replies[0])
16788 }
16789
16790 async fn relayed(
16792 &mut self,
16793 opener: Option<&str>,
16794 target: &str,
16795 scope: Option<ScopeSelector>,
16796 ) -> Relayed {
16797 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
16798 let handler = self.handler.clone();
16799 let task_ctx = ctx.clone();
16800 let task = tokio::spawn(async move {
16801 handler
16802 .handle_control_frame(&task_ctx, frame)
16803 .await
16804 .unwrap()
16805 });
16806 let (_, module_rx) = self.modules.get_mut(target).unwrap();
16807 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
16808 .await
16809 .expect("the target receives the relayed route.bind")
16810 .unwrap()
16811 .frame;
16812 Relayed {
16813 target: target.to_string(),
16814 client: ctx,
16815 client_rx: rx,
16816 task,
16817 bind,
16818 }
16819 }
16820
16821 async fn ack(&self, relayed: &Relayed) {
16822 let (module, _) = &self.modules[&relayed.target];
16823 self.handler
16824 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
16825 .await
16826 .unwrap();
16827 }
16828
16829 async fn bound(
16831 &mut self,
16832 opener: Option<&str>,
16833 target: &str,
16834 scope: Option<ScopeSelector>,
16835 ) -> Bound {
16836 let relayed = self.relayed(opener, target, scope).await;
16837 self.ack(&relayed).await;
16838 let Relayed {
16839 target,
16840 client,
16841 mut client_rx,
16842 task,
16843 bind,
16844 } = relayed;
16845 assert!(
16846 task.await.unwrap().is_empty(),
16847 "the open is answered by commit"
16848 );
16849 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
16850 Bound {
16851 target,
16852 client,
16853 client_rx,
16854 channel,
16855 epoch,
16856 bind,
16857 }
16858 }
16859
16860 fn live(&self, route: &Bound) -> bool {
16861 matches!(
16862 self.forwarding
16863 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16864 .unwrap(),
16865 DataRoute::Client(DataRouteState::Bound(_))
16866 )
16867 }
16868 }
16869
16870 struct Relayed {
16871 target: String,
16872 client: RouteCtx,
16873 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16874 task: tokio::task::JoinHandle<Vec<Frame>>,
16875 bind: Frame,
16876 }
16877
16878 struct Bound {
16879 target: String,
16880 client: RouteCtx,
16881 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16882 channel: u16,
16883 epoch: u32,
16884 bind: Frame,
16885 }
16886
16887 impl Bound {
16888 fn closed_reason(&mut self) -> RouteCloseReason {
16891 let mut reason = None;
16892 let mut goodbye = false;
16893 while let Ok(outbound) = self.client_rx.try_recv() {
16894 let frame = outbound.frame;
16895 match frame.header.ty {
16896 FrameType::Goodbye => {
16897 assert_eq!(
16898 (frame.header.channel, frame.header.epoch),
16899 (self.channel, self.epoch)
16900 );
16901 goodbye = true;
16902 }
16903 FrameType::Push => {
16904 let ClientControlPush::RouteClosed {
16905 reason: r,
16906 module_id,
16907 ..
16908 } = serde_json::from_slice(&frame.body).unwrap()
16909 else {
16910 panic!("unexpected push");
16911 };
16912 assert_eq!(module_id, self.target);
16913 reason = Some(r);
16914 }
16915 other => panic!("unexpected frame {other:?}"),
16916 }
16917 }
16918 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
16919 reason.expect("the client is told why the route closed")
16920 }
16921
16922 fn untouched(&mut self) -> bool {
16923 self.client_rx.try_recv().is_err()
16924 }
16925
16926 fn stamp(&self) -> Option<ScopeStamp> {
16927 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
16928 ModuleControlRequest::RouteBind { scope, .. } => scope,
16929 other => panic!("expected a route.bind, got {other:?}"),
16930 }
16931 }
16932 }
16933
16934 #[tokio::test]
16935 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
16936 ) {
16937 let mut rig = rig().await;
16938 let mut record = session(1);
16939 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
16940 record.child_owners = vec![Principal::Reserved {
16941 module_id: MAGIC.to_string(),
16942 }];
16943 rig.sync(vec![record]).await;
16944 let scope = || Some(rig_selector("s", Some(1)));
16945
16946 rig.bound(Some(OWNER), PLEXUS, scope()).await;
16949 rig.bound(Some(AFT), OTHER, scope()).await;
16950 rig.bound(Some(BROCA), PLEXUS, scope()).await;
16951
16952 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
16956 assert_eq!(
16957 rig.refused(opener, target, scope()).await,
16958 error_codes::SCOPE_NOT_CARRIER,
16959 "{opener:?} -> {target}"
16960 );
16961 }
16962 }
16963
16964 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16965 ScopeSelector::new(
16966 Principal::Reserved {
16967 module_id: OWNER.to_string(),
16968 },
16969 scope_ref.to_string(),
16970 scope_epoch,
16971 )
16972 }
16973
16974 #[tokio::test]
16975 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
16976 let mut rig = rig().await;
16977 rig.sync(vec![session(1)]).await;
16978 for opener in [OWNER, AFT] {
16979 assert_eq!(
16980 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
16981 .await,
16982 error_codes::SCOPE_EPOCH_REQUIRED,
16983 "{opener}"
16984 );
16985 }
16986 }
16987
16988 #[tokio::test]
16989 async fn admission_separates_not_synced_not_live_and_ended() {
16990 let mut rig = rig().await;
16991 let code = rig
16993 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16994 .await;
16995 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
16996 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
16997
16998 let ghost = ScopeSelector::new(
17000 Principal::Reserved {
17001 module_id: "ghost".to_string(),
17002 },
17003 "s".to_string(),
17004 Some(1),
17005 );
17006 assert_eq!(
17007 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
17008 error_codes::SCOPE_NOT_LIVE
17009 );
17010
17011 rig.sync(vec![session(2)]).await;
17012 assert_eq!(
17013 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
17014 .await,
17015 error_codes::SCOPE_NOT_LIVE
17016 );
17017 for epoch in [1, 3] {
17018 assert_eq!(
17019 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
17020 .await,
17021 error_codes::SCOPE_ENDED,
17022 "epoch {epoch}"
17023 );
17024 }
17025 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
17027 .await;
17028 }
17029
17030 #[tokio::test]
17031 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
17032 let mut rig = rig().await;
17033 rig.sync(vec![session(1)]).await;
17034 let route = rig
17035 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17036 .await;
17037 let stamp = route.stamp().expect("a scoped bind carries the stamp");
17038 assert_eq!(stamp.scope_ref, "s");
17039 assert_eq!(stamp.scope_epoch, 1);
17040 assert_eq!(stamp.kind, ScopeKind::Head);
17041 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
17042 assert!(stamp.attributes.delegates);
17043 assert!(stamp.owner_authorized);
17044 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17045 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
17046
17047 let (broca, mut broca_rx) = wide_ctx(50);
17050 hello_via_sink(
17051 &rig.handler,
17052 &broca,
17053 &mut broca_rx,
17054 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
17055 )
17056 .await;
17057 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
17058 .await
17059 .unwrap();
17060 let own = ScopeSelector::new(
17061 Principal::Reserved {
17062 module_id: BROCA.to_string(),
17063 },
17064 "b".to_string(),
17065 Some(1),
17066 );
17067 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
17068 assert!(!route.stamp().unwrap().owner_authorized);
17069 }
17070
17071 #[tokio::test]
17072 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
17073 let mut rig = rig().await;
17074 let mut record = head("s", 1);
17075 record.carriers = vec![carrier(AFT, None)];
17076 let flow_id = "Flow:run-7/step_2!~";
17077 record.attributes.flow_id = Some(flow_id.to_string());
17078 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
17079 .await
17080 .unwrap();
17081 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17082 panic!("not a sync reply");
17083 };
17084 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
17085 let route = rig
17086 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17087 .await;
17088 assert!(rig.live(&route), "the stamped bind committed");
17089 let stamp = route.stamp().expect("a flow scope carries a stamp");
17090 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
17091 assert_eq!(stamp.attributes.agent_id, None);
17092 assert!(!stamp.attributes.delegates);
17093 assert!(stamp.owner_authorized);
17094 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17095 assert_eq!(unscoped.stamp(), None);
17096 }
17097
17098 #[tokio::test]
17099 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
17100 let mut rig = rig_with_flow_support(false).await;
17101 let mut record = session(1);
17102 record.attributes.flow_id = Some("flow:7".to_string());
17103 rig.sync(vec![record]).await;
17104 for opener in [OWNER, AFT] {
17105 let body = rig
17106 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17107 .await;
17108 assert_eq!(body["code"], "target_flow_unsupported");
17109 let message = body["message"].as_str().unwrap();
17110 for required in [PLEXUS, "flow-scopes/v1"] {
17111 assert!(message.contains(required), "{message}");
17112 }
17113 }
17114 }
17115
17116 #[tokio::test]
17117 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
17118 let mut rig = rig_with_flow_support(true).await;
17119 let mut record = session(1);
17120 record.attributes.flow_id = Some("flow:7".to_string());
17121 rig.sync(vec![record]).await;
17122 for opener in [OWNER, AFT] {
17123 let route = rig
17124 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17125 .await;
17126 assert!(rig.live(&route));
17127 assert_eq!(
17128 route.stamp().unwrap().attributes.flow_id.as_deref(),
17129 Some("flow:7")
17130 );
17131 }
17132 }
17133
17134 #[tokio::test]
17135 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
17136 use std::future::Future;
17137
17138 let mut rig = rig_with_flow_support(true).await;
17139 let mut record = session(1);
17140 record.attributes.flow_id = Some("flow:7".to_string());
17141 rig.sync(vec![record]).await;
17142 let (client, mut client_rx, frame) =
17143 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
17144 for _ in 0..64 {
17147 client.egress.try_send(route_bind_ack(1)).unwrap();
17148 }
17149 let handler = rig.handler.clone();
17150 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17151 std::future::poll_fn(|cx| {
17152 assert!(open.as_mut().poll(cx).is_pending());
17153 std::task::Poll::Ready(())
17154 })
17155 .await;
17156 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17157
17158 let old_connection = rig.modules[PLEXUS].0.connection_id;
17159 rig.handler.cleanup_connection(old_connection).unwrap();
17160 let (replacement, mut replacement_rx) = wide_ctx(200);
17161 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
17162 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17163 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17164 let hello = Frame::build(
17165 FrameType::Hello,
17166 control_flags(),
17167 0,
17168 0,
17169 200,
17170 serde_json::to_vec(&body).unwrap(),
17171 )
17172 .unwrap();
17173 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
17174
17175 client_rx.try_recv().unwrap();
17176 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17177 .await
17178 .expect("the replacement is refused without waiting for a bind ack")
17179 .unwrap();
17180 assert_eq!(replies.len(), 1);
17181 let body = parse_error(&replies[0]);
17182 assert_eq!(body["code"], "target_flow_unsupported");
17183 for required in [PLEXUS, "flow-scopes/v1"] {
17184 assert!(body["message"].as_str().unwrap().contains(required));
17185 }
17186 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
17187 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17188 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17189 assert!(rig
17190 .handler
17191 .registry
17192 .get_module_by_connection(replacement.connection_id)
17193 .unwrap()
17194 .is_some());
17195 }
17196
17197 #[tokio::test]
17198 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
17199 use std::future::Future;
17200 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17201 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17202 const NEXT_HASH: &str =
17203 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17204 let mut rig = rig().await;
17205 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
17206 record.carriers.push(ScopeCarrier::new(Principal::Direct));
17207 rig.sync(vec![record.clone()]).await;
17208 let (client, mut rx, frame) = rig.open_frame(
17209 None,
17210 PLEXUS,
17211 Some(
17212 rig.selector("s", Some(1))
17213 .with_bind_proof(Some(PROOF.into())),
17214 ),
17215 );
17216 for _ in 0..64 {
17219 client.egress.try_send(route_bind_ack(1)).unwrap();
17220 }
17221 let handler = rig.handler.clone();
17222 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17223 std::future::poll_fn(|cx| {
17224 assert!(open.as_mut().poll(cx).is_pending());
17225 std::task::Poll::Ready(())
17226 })
17227 .await;
17228 record.bind_proof_sha256 = Some(NEXT_HASH.into());
17229 rig.sync(vec![record]).await;
17230 rx.try_recv().unwrap();
17231 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17232 .await
17233 .unwrap()
17234 .unwrap();
17235 assert_eq!(
17236 parse_error(&replies[0])["code"],
17237 error_codes::SCOPE_BIND_PROOF_MISMATCH
17238 );
17239 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
17240 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17241 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17242 }
17243
17244 #[tokio::test]
17245 async fn scope_bind_proof_reserved_openers_ignore_proof() {
17246 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17247 let mut rig = rig().await;
17248 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17249 .await;
17250 for opener in [OWNER, AFT] {
17251 for proof in [None, Some("malformed+proof=")] {
17252 let route = rig
17253 .bound(
17254 Some(opener),
17255 PLEXUS,
17256 Some(
17257 rig.selector("s", Some(1))
17258 .with_bind_proof(proof.map(str::to_string)),
17259 ),
17260 )
17261 .await;
17262 assert!(rig.live(&route));
17263 }
17264 }
17265 }
17266
17267 #[tokio::test]
17268 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
17269 use std::future::Future;
17270 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17271 const NEXT_HASH: &str =
17272 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17273 let mut rig = rig().await;
17274 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17275 .await;
17276 let (client, mut rx, frame) =
17277 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
17278 for _ in 0..64 {
17279 client.egress.try_send(route_bind_ack(1)).unwrap();
17280 }
17281 let handler = rig.handler.clone();
17282 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17283 std::future::poll_fn(|cx| {
17284 assert!(open.as_mut().poll(cx).is_pending());
17285 std::task::Poll::Ready(())
17286 })
17287 .await;
17288 rig.sync(vec![
17289 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
17290 ])
17291 .await;
17292 rx.try_recv().unwrap();
17293 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17294 let bind = std::future::poll_fn(|cx| {
17295 assert!(open.as_mut().poll(cx).is_pending());
17296 module_rx.poll_recv(cx)
17297 })
17298 .await
17299 .unwrap()
17300 .frame;
17301 handler
17302 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
17303 .await
17304 .unwrap();
17305 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17306 .await
17307 .unwrap()
17308 .unwrap();
17309 assert_eq!(replies.len(), 1);
17310 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
17313 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17314 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
17315 let route = rig
17316 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
17317 .await;
17318 assert!(rig.live(&route));
17319 }
17320
17321 #[tokio::test]
17322 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
17323 {
17324 let mut rig = rig_with_flow_support(false).await;
17325 rig.sync(vec![session(1)]).await;
17326 let route = rig
17327 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17328 .await;
17329 assert!(rig.live(&route));
17330 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
17331 let mut record = session(1);
17332 record.attributes.flow_id = Some("flow:7".to_string());
17333 rig.sync(vec![record]).await;
17334 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
17335 assert!(rig.live(&unscoped));
17336 assert_eq!(unscoped.stamp(), None);
17337 }
17338
17339 #[tokio::test]
17340 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
17341 let mut rig = rig().await;
17342 let mut record = session(1);
17343 record.attributes.flow_id = Some("flow:7".to_string());
17344 rig.sync(vec![record.clone()]).await;
17345 let before = rig.forwarding.published_scope_tag(OWNER, "s");
17346 let mut owner_route = rig
17347 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17348 .await;
17349 let mut carrier_route = rig
17350 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
17351 .await;
17352 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17353 rig.sync(vec![record.clone()]).await;
17354 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17355 assert!(rig.live(&owner_route) && owner_route.untouched());
17356 assert!(rig.live(&carrier_route) && carrier_route.untouched());
17357
17358 record.attributes.flow_id = Some("flow:8".to_string());
17359 rig.sync(vec![record]).await;
17360 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
17361 let before = before.unwrap();
17362 assert_eq!(after.scope_epoch, before.scope_epoch);
17363 assert!(after.version > before.version);
17364 for route in [&mut owner_route, &mut carrier_route] {
17365 assert!(!rig.live(route));
17366 assert_eq!(
17367 route.closed_reason(),
17368 RouteCloseReason::ScopeDelegationChanged
17369 );
17370 }
17371 assert!(rig.live(&unscoped) && unscoped.untouched());
17372 for target in [PLEXUS, OTHER] {
17375 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
17376 let goodbye = module_rx
17377 .try_recv()
17378 .expect("the provider sees the drain")
17379 .frame;
17380 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17381 assert!(module_rx.try_recv().is_err());
17382 }
17383 let rebound = rig
17384 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17385 .await;
17386 assert_eq!(
17387 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
17388 Some("flow:8")
17389 );
17390 }
17391
17392 #[tokio::test]
17397 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
17398 let mut rig = rig().await;
17399 rig.sync(vec![session(1)]).await;
17400 let mut cotenant = rig
17401 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17402 .await;
17403
17404 let mut changed = session(1);
17405 changed.child_owners.push(Principal::Reserved {
17406 module_id: MAGIC.to_string(),
17407 });
17408 let mut ended = None;
17409 for (code, next) in [
17410 (error_codes::SCOPE_CHANGED, vec![changed]),
17411 (error_codes::SCOPE_ENDED, Vec::new()),
17412 ] {
17413 let relayed = rig
17414 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17415 .await;
17416 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
17417 ended = Some(next.is_empty());
17418 rig.sync(next).await;
17419 rig.ack(&relayed).await;
17420 let replies = relayed.task.await.unwrap();
17421 assert_eq!(replies.len(), 1, "{replies:?}");
17422 assert_eq!(parse_error(&replies[0])["code"], code);
17423 assert_eq!(
17424 subc_protocol::error_codes::is_retryable_route_open(code),
17425 code == error_codes::SCOPE_CHANGED
17426 );
17427 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17428 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17430 let mut goodbyes = Vec::new();
17433 while let Ok(outbound) = plexus_rx.try_recv() {
17434 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
17435 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
17436 }
17437 assert!(
17438 goodbyes.contains(&(bind_channel, bind_epoch)),
17439 "{goodbyes:?}"
17440 );
17441 assert!(rig
17442 .handler
17443 .registry
17444 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
17445 .unwrap()
17446 .is_some());
17447 }
17448 assert_eq!(ended, Some(true));
17449 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
17452 }
17453
17454 #[tokio::test]
17457 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
17458 struct Case {
17459 name: &'static str,
17460 change: fn(&mut ScopeRecord),
17461 closed: [Option<RouteCloseReason>; 4],
17464 }
17465 use RouteCloseReason::*;
17466 let cases = [
17467 Case {
17468 name: "a carrier entry removed",
17469 change: |r| {
17470 r.carriers.retain(|c| {
17471 c.principal
17472 != Principal::Reserved {
17473 module_id: AFT.to_string(),
17474 }
17475 })
17476 },
17477 closed: [None, Some(ScopeCarrierRemoved), None, None],
17478 },
17479 Case {
17480 name: "a target removed from a carrier",
17481 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
17482 closed: [None, None, None, Some(ScopeCarrierRemoved)],
17483 },
17484 Case {
17485 name: "a bare carrier narrowed to targets",
17486 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
17487 closed: [None, Some(ScopeCarrierRemoved), None, None],
17488 },
17489 Case {
17490 name: "delegates turned off",
17491 change: |r| r.attributes.delegates = false,
17492 closed: [Some(ScopeDelegationChanged); 4],
17493 },
17494 Case {
17495 name: "agent_id changed",
17496 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
17497 closed: [Some(ScopeDelegationChanged); 4],
17498 },
17499 Case {
17500 name: "a carrier added, child owners changed, the record re-sent",
17501 change: |r| {
17502 r.carriers.push(carrier(MAGIC, None));
17503 r.child_owners.push(Principal::Reserved {
17504 module_id: MAGIC.to_string(),
17505 });
17506 },
17507 closed: [None; 4],
17508 },
17509 Case {
17510 name: "a target added",
17511 change: |r| {
17512 r.carriers[1]
17513 .targets
17514 .as_mut()
17515 .unwrap()
17516 .push("third".to_string())
17517 },
17518 closed: [None; 4],
17519 },
17520 Case {
17521 name: "delegates turned on",
17522 change: |r| r.attributes.delegates = true,
17523 closed: [None; 4],
17524 },
17525 ];
17526 for case in cases {
17527 let mut rig = rig().await;
17528 rig.sync(vec![session(1)]).await;
17529 let scope = || Some(rig_selector("s", Some(1)));
17530 let mut routes = [
17531 rig.bound(Some(OWNER), PLEXUS, scope()).await,
17532 rig.bound(Some(AFT), PLEXUS, scope()).await,
17533 rig.bound(Some(BROCA), PLEXUS, scope()).await,
17534 rig.bound(Some(BROCA), OTHER, scope()).await,
17535 ];
17536 let mut record = session(1);
17537 (case.change)(&mut record);
17538 rig.sync(vec![record]).await;
17539 for (index, expected) in case.closed.iter().enumerate() {
17540 let route = &mut routes[index];
17541 match expected {
17542 Some(reason) => {
17543 assert!(!rig.live(route), "{}: route {index} still live", case.name);
17544 assert_eq!(
17545 route.closed_reason(),
17546 *reason,
17547 "{}: route {index}",
17548 case.name
17549 );
17550 }
17551 None => {
17552 assert!(rig.live(route), "{}: route {index} closed", case.name);
17553 assert!(
17554 route.untouched(),
17555 "{}: route {index} was told something",
17556 case.name
17557 );
17558 }
17559 }
17560 }
17561 }
17562 }
17563
17564 #[tokio::test]
17565 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
17566 for next in [Vec::new(), vec![session(2)]] {
17568 let mut rig = rig().await;
17569 rig.sync(vec![session(1)]).await;
17570 let mut route = rig
17571 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17572 .await;
17573 rig.sync(next).await;
17574 assert!(!rig.live(&route));
17575 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
17576 }
17577
17578 let mut rig = rig().await;
17581 let mut child = session(1);
17582 child.scope_ref = "child".to_string();
17583 child.kind = ScopeKind::Worker;
17584 child.parent = Some(ScopeParent::new(
17585 Principal::Reserved {
17586 module_id: OWNER.to_string(),
17587 },
17588 "s".to_string(),
17589 1,
17590 ));
17591 rig.sync(vec![session(1), child.clone()]).await;
17592 let mut child_route = rig
17593 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
17594 .await;
17595 assert_eq!(
17596 child_route.stamp().unwrap().parent_state,
17597 Some(ParentState::Linked)
17598 );
17599 rig.sync(vec![child]).await;
17600 assert!(!rig.live(&child_route));
17601 assert_eq!(
17602 child_route.closed_reason(),
17603 RouteCloseReason::ScopeParentEnded
17604 );
17605 }
17606
17607 #[tokio::test]
17608 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
17609 ) {
17610 let mut rig = rig().await;
17611 rig.sync(vec![session(1)]).await;
17612 let mut route = rig
17613 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17614 .await;
17615 let before = rig.forwarding.published_scope_tag(OWNER, "s");
17616 rig.sync(vec![session(1)]).await;
17617 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17618 assert!(rig.live(&route) && route.untouched());
17619
17620 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
17624 .forwarding
17625 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17626 .unwrap()
17627 else {
17628 panic!("the route is bound");
17629 };
17630 binding.flow.acquire_tagged(9, false).await.unwrap();
17631 let mut widened = session(1);
17632 widened.carriers.push(carrier(MAGIC, None));
17633 rig.sync(vec![widened]).await;
17634 assert!(rig.live(&route) && route.untouched());
17635 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17636 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
17637 assert_eq!(binding.flow.in_flight(), 1);
17640 binding
17641 .flow
17642 .acquire_tagged(10, false)
17643 .await
17644 .expect("the flow is still open");
17645 }
17646
17647 #[tokio::test]
17650 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
17651 let mut rig = rig().await;
17652 rig.sync(vec![session(1)]).await;
17653 let mut on_incumbent = rig
17654 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17655 .await;
17656
17657 let (candidate, _candidate_rx) = wide_ctx(9);
17660 let registration = rig
17661 .handler
17662 .registry
17663 .register_candidate_with_control_ops(
17664 manifest(PLEXUS, PROTOCOL_VERSION),
17665 PROTOCOL_VERSION,
17666 candidate.connection_id,
17667 module_baseline_control_ops(),
17668 )
17669 .unwrap();
17670 rig.forwarding
17671 .register_candidate_module_connection(
17672 candidate.connection_id,
17673 PLEXUS.to_string(),
17674 PROTOCOL_VERSION,
17675 manifest_concurrency(®istration.manifest),
17676 candidate.egress.clone(),
17677 )
17678 .unwrap();
17679 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
17680 rig.handler
17681 .registry
17682 .promote_candidate(PLEXUS)
17683 .unwrap()
17684 .unwrap();
17685 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
17686
17687 rig.sync(Vec::new()).await;
17688 assert!(!rig.live(&on_incumbent));
17689 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
17690 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17691 let goodbye = incumbent_rx
17692 .try_recv()
17693 .expect("the superseded endpoint is told")
17694 .frame;
17695 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17696 }
17697 }
17698}
17699
17700#[cfg(test)]
17701mod concurrency_default_exposure_tests {
17702 use super::*;
17703
17704 fn hello_body(role_json: &str) -> Vec<u8> {
17705 format!(
17706 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
17707 )
17708 .into_bytes()
17709 }
17710
17711 fn manifest_from(body: &[u8]) -> ModuleManifest {
17712 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
17713 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
17714 .expect("manifest parses")
17715 }
17716
17717 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
17718
17719 #[test]
17720 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
17721 let body = hello_body(&format!(
17722 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
17723 ));
17724 let manifest = manifest_from(&body);
17725 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17728 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
17729 }
17730
17731 #[test]
17732 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
17733 let body = hello_body(&format!(
17734 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
17735 ));
17736 let manifest = manifest_from(&body);
17737 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17738 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17739 }
17740
17741 #[test]
17742 fn non_management_roles_are_never_reported() {
17743 let body = hello_body(
17744 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
17745 );
17746 let manifest = manifest_from(&body);
17747 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17748 }
17749}