1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_event_declarations,
25 validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
26 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
27 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
28 },
29 scope::{
30 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
31 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
32 SCOPE_SYNC_OP,
33 },
34 session::{
35 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
36 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
37 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
38 ROLE_VERSIONS_FIELD,
39 },
40 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
41 Priority, RouteTarget, PROTOCOL_VERSION,
42};
43use tokio::time::{timeout_at, Instant};
44use tracing::{debug, info, warn};
45
46use crate::{
47 caller_identity::CallerIdentity,
48 capability_requirements::{
49 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
50 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
51 RuntimeModule,
52 },
53 child_roster::DaemonShutdownFlag,
54 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
55 forwarding::{
56 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
57 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
58 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
59 },
60 observability::{
61 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
62 },
63 privileged_prompt::{prompt_text, PrivilegedOperation},
64 provenance::{
65 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
66 },
67 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
68 router::{RouteCtx, RouterError},
69 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
70 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
71 stderr_tail::{CaptureState, TailEntry},
72 supervise::{
73 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
74 SupervisorHandle, SwapHelloAdmission,
75 },
76 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
77};
78
79pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
85
86const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
87const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
88const CAP_PING_PONG: &str = "ping_pong_v1";
89const CAP_SESSION_ATTACH: &str = "session_attach_v1";
90const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
91
92const SUBC_CONTROL_OPS: &[&str] = &[
93 ops::ACTIVITY_RECORD,
94 ops::ACTIVITY_QUERY,
95 ops::ACTIVITY_SUBSCRIBE,
96 ops::SERVER_DESCRIBE,
97 ops::CATALOG_LIST,
98 ops::ROUTE_OPEN,
99 ops::ROUTE_POLL,
100 ops::ROUTE_CLOSING,
101 ops::ROUTE_CLOSED,
102 ops::SUPERVISOR_LIST,
103 ops::SUPERVISOR_RESTART,
104 ops::SUPERVISOR_SWAP,
105 ops::SUPERVISOR_RELOAD,
106 ops::SUPERVISOR_RESCAN,
107 ops::SUPERVISOR_RELEASE_RESERVED,
108 ops::SUPERVISOR_SET_ENABLED,
109 ops::SUPERVISOR_HEALTH_PROBE,
110 ops::SUPERVISOR_HEALTH,
111 ops::SUPERVISOR_STDERR_TAIL,
112 ops::SUPERVISOR_TERMINALS,
113 ops::SUPERVISOR_ROUTES,
114 ops::SUPERVISOR_PROVENANCE,
115 ops::SUPERVISOR_SPAWN_SNAPSHOT,
116 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
117];
118
119const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
126
127mod rescan_checked;
128use rescan_checked::{RescanMode, RescanPlan};
129
130const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
131 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
132 "supervisor.live_roots",
133 SCOPE_SYNC_OP,
134 SCOPE_APPLY_OP,
135 SCOPE_DESCRIBE_OP,
136 "operator.confirm",
137];
138
139const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
143
144const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
145
146pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
154
155pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
170
171pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
189
190const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
191const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
192
193fn reload_verdict(
194 configured: &Path,
195 spawned_from: Option<&Path>,
196 image: subc_control::RunningImageAgreement,
197) -> PendingReloadVerdict {
198 let path = match spawned_from {
199 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
200 ReloadPathAgreement::Match
201 }
202 Some(spawned_from) => ReloadPathAgreement::Mismatch {
203 configured: configured.to_path_buf(),
204 spawned_from: spawned_from.to_path_buf(),
205 },
206 None => ReloadPathAgreement::Unavailable {
207 reason: if matches!(
208 image,
209 subc_control::RunningImageAgreement::Unavailable {
210 reason: subc_control::RunningImageUnavailableReason::NotRunning
211 }
212 ) {
213 ReloadPathUnavailableReason::NotRunning
214 } else {
215 ReloadPathUnavailableReason::SpawnedPathUnavailable
216 },
217 },
218 };
219 PendingReloadVerdict {
220 path,
221 image,
222 spec: None,
223 }
224}
225
226fn launch_spec_agreement(
227 configured: &crate::supervise::ModuleSpec,
228 running: bool,
229 spawned: Option<&crate::supervise::LaunchSpecDigest>,
230) -> subc_control::LaunchSpecAgreement {
231 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
232 if !running {
233 return Agreement::Unavailable {
234 reason: Reason::NotRunning,
235 };
236 }
237 match spawned {
238 None => Agreement::Unavailable {
239 reason: Reason::SpawnedSpecUnavailable,
240 },
241 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
242 Agreement::Match
243 }
244 Some(_) => Agreement::Mismatch,
245 }
246}
247
248fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
249 if configured == spawned_from {
250 return true;
251 }
252 #[cfg(windows)]
253 {
254 match (
260 std::fs::canonicalize(configured),
261 std::fs::canonicalize(spawned_from),
262 ) {
263 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
264 _ => false,
265 }
266 }
267 #[cfg(not(windows))]
268 {
269 false
270 }
271}
272
273#[derive(Clone)]
274struct DaemonProvenanceFacts {
275 build: DaemonBuildProvenance,
276 pid: Option<u32>,
277 started_at_ms: Option<u64>,
278 start_clock: Option<crate::clock::StartClock>,
279 executable_path: Option<PathBuf>,
280 executable_identity: Option<SpawnedFileIdentity>,
281 process_start_time: Option<u64>,
282 probe: ExecutableIdentityProbe,
283}
284
285impl Default for DaemonProvenanceFacts {
286 fn default() -> Self {
287 Self {
288 build: DaemonBuildProvenance {
289 build_git_sha: None,
290 build_lock_digest: None,
291 },
292 pid: None,
293 started_at_ms: None,
294 start_clock: None,
295 executable_path: None,
296 executable_identity: None,
297 process_start_time: None,
298 probe: ExecutableIdentityProbe::default(),
299 }
300 }
301}
302
303#[derive(Debug, Clone)]
304struct SupervisorRescanContext {
305 supervisor: Supervisor,
306 config_path: PathBuf,
307 configured_port: Option<u16>,
308 storage_config: Option<crate::daemon_config::StorageConfig>,
309 admission_facts_carrier_module_id: Option<String>,
310 admission_facts_targets: Option<Vec<String>>,
311 scope_authority_owners: Vec<String>,
312 privileged_operator_prompt: PrivilegedOperatorPrompt,
313 generation: Arc<std::sync::atomic::AtomicU64>,
319}
320
321const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
342 "reloading",
343 "supervisor_not_live",
344 "registration_not_active",
345 "no_forwarding_connection",
346 "relay_send_failed",
347];
348
349#[derive(Clone)]
351pub struct ControlHandler {
352 registry: Arc<Registry>,
353 activity: Option<crate::activity::ActivityLedger>,
354 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
355 forwarding: Arc<ForwardingTable>,
356 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
357 supervisor: SupervisorHandle,
358 subc_capabilities: Arc<[String]>,
359 route_bind_relay_timeout: Duration,
363 route_bind_relay_timeouts: BTreeMap<String, Duration>,
367 route_bind_breakers: RouteBindBreakers,
370 route_bind_concurrency: RouteBindConcurrency,
373 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
378 route_bind_breaker_threshold: u32,
380 route_bind_breaker_cooldown: Duration,
382 health_probe_timeout: Duration,
383 storage_config: Option<crate::daemon_config::StorageConfig>,
386 machine_id: Option<crate::machine_id::MachineId>,
390 admission_facts_carrier_module_id: Option<String>,
391 admission_facts_targets: Option<Vec<String>>,
392 scopes: Arc<RwLock<ScopeTable>>,
396 scope_deadline_added: Arc<tokio::sync::Notify>,
400 scope_authority_owners: Vec<String>,
403 privileged_operator_prompt: PrivilegedOperatorPrompt,
405 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
408 #[cfg(test)]
409 caller_lookup: Option<TestCallerLookup>,
410 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
413 rescan: Option<SupervisorRescanContext>,
414 connected_clients: ConnectedClients,
415 counters: DaemonCounters,
416 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
417 daemon_shutdown: DaemonShutdownFlag,
422 daemon_provenance: DaemonProvenanceFacts,
423 #[cfg(test)]
424 control_dispatch_delay: Option<Duration>,
425 #[cfg(test)]
426 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
427 #[cfg(test)]
431 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
432 #[cfg(test)]
438 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
439}
440
441impl fmt::Debug for ControlHandler {
442 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
443 f.debug_struct("ControlHandler")
444 .field("registry", &self.registry)
445 .field("forwarding", &self.forwarding)
446 .field("process_liveness", &self.process_liveness.is_some())
447 .field("supervisor", &self.supervisor)
448 .field("subc_capabilities", &self.subc_capabilities)
449 .finish()
450 }
451}
452
453struct RouteOpenRequest {
454 target: RouteTarget,
455 identity: BindIdentity,
456 consumer_identity: Option<ConsumerIdentity>,
457 consumer_capabilities: Option<Vec<String>>,
458 role_versions: Option<BTreeMap<String, String>>,
459 admission_facts: Option<serde_json::Value>,
460 scope: Option<ScopeSelector>,
461}
462
463#[derive(Clone, Copy)]
464enum RouteOpenPhase {
465 Admission,
466 ProjectRoot,
467 EgressReservation,
468 BindSend,
469 BindAckWait,
470}
471
472struct RouteOpenTimings {
476 module_id: String,
477 principal: String,
478 admission: Duration,
479 project_root: Duration,
480 egress_reservation: Duration,
481 bind_send: Duration,
482 bind_ack_wait: Duration,
483 phase: Option<RouteOpenPhase>,
484 phase_started_at: Instant,
485}
486
487impl RouteOpenTimings {
488 fn new() -> Self {
489 Self {
490 module_id: String::new(),
491 principal: "unresolved".to_string(),
493 admission: Duration::ZERO,
494 project_root: Duration::ZERO,
495 egress_reservation: Duration::ZERO,
496 bind_send: Duration::ZERO,
497 bind_ack_wait: Duration::ZERO,
498 phase: Some(RouteOpenPhase::Admission),
499 phase_started_at: Instant::now(),
500 }
501 }
502
503 fn finish(&mut self) {
504 let elapsed = self.phase_started_at.elapsed();
505 match self.phase.take() {
506 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
507 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
508 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
509 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
510 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
511 None => {}
512 }
513 }
514
515 fn enter(&mut self, phase: RouteOpenPhase) {
516 self.finish();
517 self.phase = Some(phase);
518 self.phase_started_at = Instant::now();
519 }
520}
521
522struct RouteBindReservationGuard {
523 forwarding: Arc<ForwardingTable>,
524 endpoint: ModuleEndpointId,
525 relay_corr: u64,
526 armed: bool,
527}
528
529struct ModuleControlRpcGuard {
530 forwarding: Arc<ForwardingTable>,
531 endpoint: ModuleEndpointId,
532 corr: u64,
533 armed: bool,
534}
535
536impl ModuleControlRpcGuard {
537 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
538 Self {
539 forwarding,
540 endpoint,
541 corr,
542 armed: true,
543 }
544 }
545
546 fn disarm(&mut self) {
547 self.armed = false;
548 }
549}
550
551impl Drop for ModuleControlRpcGuard {
552 fn drop(&mut self) {
553 if self.armed {
554 let _ = self
555 .forwarding
556 .cancel_module_control_rpc(self.endpoint, self.corr);
557 }
558 }
559}
560
561impl RouteBindReservationGuard {
562 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
563 Self {
564 forwarding,
565 endpoint,
566 relay_corr,
567 armed: true,
568 }
569 }
570
571 fn release_and_disarm(&mut self) {
572 if !self.armed {
573 return;
574 }
575 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
576 self.endpoint,
577 self.relay_corr,
578 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
579 ) {
580 send_goodbye_target_best_effort(
581 &self.forwarding.counters(),
582 &target,
583 "canceled route.bind",
584 );
585 }
586 self.armed = false;
587 }
588
589 fn disarm(&mut self) {
590 self.armed = false;
591 }
592}
593
594impl Drop for RouteBindReservationGuard {
595 fn drop(&mut self) {
596 self.release_and_disarm();
597 }
598}
599
600#[derive(Debug, Clone, Default)]
630pub(crate) struct RouteBindBreakers {
631 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
632}
633
634#[derive(Debug, Clone, Default)]
635pub(crate) struct RouteBindConcurrency {
636 modules: Arc<Mutex<HashMap<String, usize>>>,
637}
638
639struct RouteBindConcurrencyGuard {
640 concurrency: RouteBindConcurrency,
641 module_id: String,
642}
643
644impl RouteBindConcurrency {
645 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
648 let mut modules = self
649 .modules
650 .lock()
651 .expect("route.bind concurrency mutex poisoned");
652 let in_flight = modules.entry(module_id.to_string()).or_default();
653 if *in_flight >= limit {
654 return Err(*in_flight);
655 }
656 *in_flight += 1;
657 Ok(RouteBindConcurrencyGuard {
658 concurrency: self.clone(),
659 module_id: module_id.to_string(),
660 })
661 }
662}
663
664impl Drop for RouteBindConcurrencyGuard {
665 fn drop(&mut self) {
666 let mut modules = self
667 .concurrency
668 .modules
669 .lock()
670 .expect("route.bind concurrency mutex poisoned");
671 let remove = {
672 let in_flight = modules
673 .get_mut(&self.module_id)
674 .expect("admitted route.bind has a concurrency entry");
675 *in_flight -= 1;
676 *in_flight == 0
677 };
678 if remove {
679 modules.remove(&self.module_id);
680 }
681 }
682}
683
684#[derive(Debug, Default)]
685struct ModuleBreakerState {
686 consecutive_timeouts: u32,
688 cooldown_until: Option<Instant>,
691 probe_in_flight: Option<Arc<()>>,
696}
697
698enum RouteBindAdmission<'a> {
700 Admitted {
701 guard: RouteBindBreakerGuard<'a>,
702 probe: bool,
705 },
706 Refused {
707 consecutive_timeouts: u32,
708 retry_in: Duration,
712 probe_in_flight: bool,
713 },
714}
715
716struct RouteBindBreakerGuard<'a> {
723 breakers: RouteBindBreakers,
724 module_id: &'a str,
725 probe_token: Option<Arc<()>>,
726 settled: bool,
727}
728
729impl RouteBindBreakerGuard<'_> {
730 fn record_accepted(&mut self) -> bool {
734 self.settled = true;
735 self.breakers.record_accepted(self.module_id)
736 }
737
738 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
741 self.settled = true;
742 self.breakers.record_timeout(
743 self.module_id,
744 self.probe_token.as_ref(),
745 threshold,
746 cooldown,
747 )
748 }
749
750 fn record_inconclusive(&mut self) {
759 self.settled = true;
760 self.breakers
761 .record_inconclusive(self.module_id, self.probe_token.as_ref());
762 }
763}
764
765impl Drop for RouteBindBreakerGuard<'_> {
766 fn drop(&mut self) {
767 if !self.settled {
768 self.breakers
769 .record_inconclusive(self.module_id, self.probe_token.as_ref());
770 }
771 }
772}
773
774struct BreakerOpened {
778 consecutive_timeouts: u32,
779 reopened_after_probe: bool,
782}
783
784impl RouteBindBreakers {
785 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
786 self.modules
787 .lock()
788 .expect("route.bind breaker mutex poisoned")
789 }
790
791 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
794 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
795 probe: probe_token.is_some(),
796 guard: RouteBindBreakerGuard {
797 breakers: self.clone(),
798 module_id,
799 probe_token,
800 settled: false,
801 },
802 };
803
804 let mut modules = self.lock();
805 let Some(state) = modules.get_mut(module_id) else {
806 return admitted(None);
807 };
808 let Some(cooldown_until) = state.cooldown_until else {
809 return admitted(None);
810 };
811 if state.probe_in_flight.is_some() {
812 return RouteBindAdmission::Refused {
813 consecutive_timeouts: state.consecutive_timeouts,
814 retry_in: Duration::ZERO,
815 probe_in_flight: true,
816 };
817 }
818 let now = Instant::now();
819 if now < cooldown_until {
820 return RouteBindAdmission::Refused {
821 consecutive_timeouts: state.consecutive_timeouts,
822 retry_in: cooldown_until - now,
823 probe_in_flight: false,
824 };
825 }
826 let token = Arc::new(());
827 state.probe_in_flight = Some(Arc::clone(&token));
828 admitted(Some(token))
829 }
830
831 fn record_accepted(&self, module_id: &str) -> bool {
832 self.lock()
833 .remove(module_id)
834 .is_some_and(|state| state.cooldown_until.is_some())
835 }
836
837 fn record_timeout(
838 &self,
839 module_id: &str,
840 probe_token: Option<&Arc<()>>,
841 threshold: u32,
842 cooldown: Duration,
843 ) -> Option<BreakerOpened> {
844 let mut modules = self.lock();
845 let state = modules.entry(module_id.to_string()).or_default();
846 let was_open = state.cooldown_until.is_some();
847 let was_probe = Self::owns_probe(state, probe_token);
848 if was_probe {
849 state.probe_in_flight = None;
850 }
851 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
852 if state.consecutive_timeouts < threshold {
853 return None;
854 }
855 state.cooldown_until = Some(Instant::now() + cooldown);
856 Some(BreakerOpened {
857 consecutive_timeouts: state.consecutive_timeouts,
858 reopened_after_probe: was_open && was_probe,
859 })
860 }
861
862 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
863 state
867 .probe_in_flight
868 .as_ref()
869 .zip(token)
870 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
871 }
872
873 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
874 if let Some(state) = self.lock().get_mut(module_id) {
875 if Self::owns_probe(state, probe_token) {
876 state.probe_in_flight = None;
877 }
878 }
879 }
880
881 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
896 self.lock()
897 .remove(module_id)
898 .map(|state| state.consecutive_timeouts)
899 .filter(|discarded| *discarded > 0)
900 }
901
902 fn open_snapshot(&self) -> Option<serde_json::Value> {
909 let now = Instant::now();
910 let modules = self.lock();
911 let open = modules
912 .iter()
913 .filter_map(|(module_id, state)| {
914 let cooldown_until = state.cooldown_until?;
915 Some((
916 module_id.clone(),
917 serde_json::json!({
918 "consecutive_timeouts": state.consecutive_timeouts,
919 "cooldown_remaining_ms":
920 cooldown_until.saturating_duration_since(now).as_millis() as u64,
921 "probe_in_flight": state.probe_in_flight.is_some(),
922 }),
923 ))
924 })
925 .collect::<serde_json::Map<String, serde_json::Value>>();
926 (!open.is_empty()).then_some(serde_json::Value::Object(open))
927 }
928}
929
930impl ControlHandler {
931 pub fn new(registry: Arc<Registry>) -> Self {
932 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
933 }
934
935 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
936 let counters = forwarding.counters();
937 let route_bind_breakers = forwarding.route_bind_breakers();
941 let route_bind_concurrency = forwarding.route_bind_concurrency();
942 let route_outages = forwarding.route_outages();
943 Self {
944 registry,
945 activity: None,
946 activity_streams: Arc::new(Mutex::new(HashMap::new())),
947 forwarding,
948 process_liveness: None,
949 supervisor: SupervisorHandle::new(),
950 subc_capabilities: Arc::from([
951 CAP_MANIFEST_REGISTRATION.to_string(),
952 CAP_CHANNEL_LIFECYCLE.to_string(),
953 CAP_PING_PONG.to_string(),
954 CAP_SESSION_ATTACH.to_string(),
955 CAP_ADMISSION_FACTS_RELAY.to_string(),
956 CAP_SCOPES_V1.to_string(),
957 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
958 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
959 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
960 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
961 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
962 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
963 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
964 subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
965 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
966 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
967 ]),
968 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
969 route_bind_relay_timeouts: BTreeMap::new(),
970 route_bind_breakers,
971 route_bind_concurrency,
972 route_outages,
973 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
974 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
975 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
976 storage_config: None,
977 machine_id: None,
978 admission_facts_carrier_module_id: None,
979 admission_facts_targets: None,
980 scopes: Arc::new(RwLock::new(ScopeTable::new(
981 crate::daemon_config::default_scope_authority_owners(),
982 ))),
983 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
984 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
985 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
986 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
987 #[cfg(test)]
988 caller_lookup: None,
989 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
990 rescan: None,
991 connected_clients: ConnectedClients::new(),
992 counters,
993 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
994 daemon_shutdown: DaemonShutdownFlag::default(),
995 daemon_provenance: DaemonProvenanceFacts::default(),
996 #[cfg(test)]
997 control_dispatch_delay: None,
998 #[cfg(test)]
999 provenance_probe_override: None,
1000 #[cfg(test)]
1001 route_bind_relay_sent_hold: None,
1002 #[cfg(test)]
1003 hello_registry_to_forwarding_hold: None,
1004 }
1005 }
1006
1007 pub fn with_storage_config(
1010 mut self,
1011 storage_config: Option<crate::daemon_config::StorageConfig>,
1012 ) -> Self {
1013 self.storage_config = storage_config;
1014 self
1015 }
1016
1017 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1020 self.activity = Some(crate::activity::ActivityLedger::open(
1021 run_dir.join("activity.jsonl"),
1022 incarnation,
1023 ));
1024 self.supervisor.set_activity(self.activity.clone());
1025 self
1026 }
1027
1028 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1031 self.machine_id = machine_id;
1032 self
1033 }
1034
1035 pub fn with_admission_facts_config(
1039 mut self,
1040 carrier_module_id: Option<String>,
1041 targets: Option<Vec<String>>,
1042 ) -> Self {
1043 self.admission_facts_carrier_module_id = carrier_module_id;
1044 self.admission_facts_targets = targets;
1045 self
1046 }
1047
1048 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1052 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1053 self.scope_authority_owners = owners;
1054 self
1055 }
1056
1057 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1059 self.privileged_operator_prompt = mode;
1060 self
1061 }
1062
1063 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1066 self.route_bind_relay_timeout = timeout;
1067 self
1068 }
1069
1070 pub fn with_route_bind_relay_timeouts(
1076 mut self,
1077 timeouts: impl IntoIterator<Item = (String, Duration)>,
1078 ) -> Self {
1079 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1080 self
1081 }
1082
1083 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1089 self.route_bind_relay_timeouts
1090 .get(module_id)
1091 .copied()
1092 .unwrap_or(self.route_bind_relay_timeout)
1093 }
1094
1095 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1103 self.route_bind_breaker_threshold = threshold.max(1);
1104 self.route_bind_breaker_cooldown = cooldown;
1105 self
1106 }
1107
1108 #[cfg(test)]
1109 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1110 self.health_probe_timeout = timeout;
1111 self
1112 }
1113
1114 #[cfg(test)]
1115 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1116 self.control_dispatch_delay = Some(delay);
1117 self
1118 }
1119
1120 #[cfg(test)]
1124 pub(crate) fn with_route_bind_relay_sent_hold(
1125 mut self,
1126 hold: Arc<tokio::sync::Barrier>,
1127 ) -> Self {
1128 self.route_bind_relay_sent_hold = Some(hold);
1129 self
1130 }
1131
1132 #[cfg(test)]
1137 pub(crate) fn with_hello_registry_to_forwarding_hold(
1138 mut self,
1139 hold: Arc<std::sync::Barrier>,
1140 ) -> Self {
1141 self.hello_registry_to_forwarding_hold = Some(hold);
1142 self
1143 }
1144
1145 #[cfg(test)]
1146 fn hold_between_registry_and_forwarding(&self) {
1147 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1148 hold.wait();
1149 hold.wait();
1150 }
1151 }
1152
1153 pub fn with_process_liveness(
1154 mut self,
1155 process_liveness: Arc<dyn ModuleProcessLiveness>,
1156 ) -> Self {
1157 self.process_liveness = Some(process_liveness);
1158 self
1159 }
1160
1161 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1162 self.supervisor = supervisor;
1163 self.supervisor.set_activity(self.activity.clone());
1164 self
1165 }
1166
1167 pub fn with_daemon_provenance(
1168 mut self,
1169 pid: u32,
1170 started_at_ms: u64,
1171 executable_path: Option<PathBuf>,
1172 build_git_sha: Option<String>,
1173 build_lock_digest: Option<String>,
1174 ) -> Self {
1175 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1176 let process_start_time = process_start_time(pid);
1177 self.daemon_provenance = DaemonProvenanceFacts {
1178 build: DaemonBuildProvenance {
1179 build_git_sha,
1180 build_lock_digest,
1181 },
1182 pid: Some(pid),
1183 started_at_ms: Some(started_at_ms),
1184 start_clock: None,
1185 executable_path,
1186 executable_identity,
1187 process_start_time,
1188 probe: ExecutableIdentityProbe::default(),
1189 };
1190 self
1191 }
1192
1193 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1194 self.daemon_provenance.start_clock = Some(clock);
1195 self
1196 }
1197
1198 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1200 self.scopes
1201 .write()
1202 .unwrap_or_else(|poisoned| poisoned.into_inner())
1203 .set_wall_clock(Arc::new(clock));
1204 self
1205 }
1206
1207 #[cfg(test)]
1208 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1209 self.provenance_probe_override = Some(result);
1210 self
1211 }
1212
1213 pub fn with_capability_config(
1217 self,
1218 modules: impl IntoIterator<Item = (String, bool)>,
1219 reserved_capabilities: BTreeMap<String, String>,
1220 ) -> Self {
1221 self.capability_evaluator
1222 .configure(modules, reserved_capabilities);
1223 self
1224 }
1225
1226 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1230 self.daemon_shutdown = flag;
1231 self
1232 }
1233
1234 pub fn with_supervisor_rescan(
1235 mut self,
1236 supervisor: Supervisor,
1237 config_path: impl Into<PathBuf>,
1238 configured_port: Option<u16>,
1239 ) -> Self {
1240 self.rescan = Some(SupervisorRescanContext {
1241 supervisor,
1242 config_path: config_path.into(),
1243 configured_port,
1244 storage_config: self.storage_config.clone(),
1245 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1246 admission_facts_targets: self.admission_facts_targets.clone(),
1247 scope_authority_owners: self.scope_authority_owners.clone(),
1248 privileged_operator_prompt: self.privileged_operator_prompt,
1249 generation: Arc::default(),
1250 });
1251 self
1252 }
1253
1254 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1255 self.connected_clients = connected_clients;
1256 self
1257 }
1258
1259 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1260 Arc::clone(&self.forwarding)
1261 }
1262
1263 pub(crate) fn counters(&self) -> DaemonCounters {
1264 self.counters.clone()
1265 }
1266
1267 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1270 tokio::spawn(async move {
1271 loop {
1272 self.capability_evaluator
1273 .wait_for_change_or_deadline()
1274 .await;
1275 self.refresh_capability_requirements();
1276 }
1277 });
1278 }
1279
1280 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1286 tokio::spawn(async move {
1287 loop {
1288 let has_deadlines = self
1289 .scopes
1290 .read()
1291 .unwrap_or_else(|poisoned| poisoned.into_inner())
1292 .has_deadlines();
1293 if !has_deadlines {
1294 self.scope_deadline_added.notified().await;
1295 continue;
1296 }
1297 tokio::time::sleep(Duration::from_secs(1)).await;
1298 if let Err(error) = self.sweep_expired_scopes() {
1299 warn!(%error, "scope expiry sweep failed");
1300 }
1301 }
1302 });
1303 }
1304
1305 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1306 let mut table = self
1307 .scopes
1308 .write()
1309 .unwrap_or_else(|poisoned| poisoned.into_inner());
1310 let swept = table.sweep_expired();
1311 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1312 drop(table);
1313 self.close_scope_drained_routes(drained);
1314 Ok(())
1315 }
1316
1317 fn publish_scope_changes(
1318 &self,
1319 changes: &[crate::scopes::ScopeTagChange],
1320 expired: &[crate::scopes::ScopeExpired],
1321 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1322 if changes.is_empty() {
1323 return Ok(Vec::new());
1324 }
1325 let drained = self
1326 .forwarding
1327 .publish_scope_changes(changes)
1328 .map_err(RouterError::Forwarding)?;
1329 let mut counts = HashMap::new();
1330 for route in &drained {
1331 *counts
1332 .entry((
1333 route.scope.owner.as_str(),
1334 route.scope.scope_ref.as_str(),
1335 route.scope.tag.scope_epoch,
1336 ))
1337 .or_insert(0usize) += 1;
1338 }
1339 for scope in expired {
1340 let routes_closed = counts
1341 .get(&(
1342 scope.owner.as_str(),
1343 scope.scope_ref.as_str(),
1344 scope.scope_epoch,
1345 ))
1346 .copied()
1347 .unwrap_or(0);
1348 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1349 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1350 routes_closed, "scope expired");
1351 }
1352 Ok(drained)
1353 }
1354
1355 fn runtime_capability_snapshot(
1356 &self,
1357 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1358 let runtime = self
1359 .supervisor
1360 .list()
1361 .into_iter()
1362 .map(|module| {
1363 let status = module.status().map_err(|err| {
1364 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1365 })?;
1366 Ok(RuntimeModule {
1367 module_id: status.module_id,
1368 state: status.state,
1369 enabled: status.enabled,
1370 })
1371 })
1372 .collect::<Result<Vec<_>, RouterError>>()?;
1373 let registrations = self.registry.list_registered_modules().map_err(|err| {
1379 RouterError::backend(
1380 0,
1381 0,
1382 format!("failed to list capability registrations: {err}"),
1383 )
1384 })?;
1385 let registrations = registrations
1386 .into_iter()
1387 .map(|registration| RegisteredModule {
1388 module_id: registration.manifest.module_id,
1389 module_version: registration.manifest.module_version,
1390 capabilities: registration.manifest.capabilities,
1391 })
1392 .collect();
1393 Ok((runtime, registrations))
1394 }
1395
1396 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1403 let cached_registration = RegisteredModule {
1404 module_id: registration.manifest.module_id.clone(),
1405 module_version: registration.manifest.module_version.clone(),
1406 capabilities: registration.manifest.capabilities.clone(),
1407 };
1408 if self.capability_evaluator.record_hello(&cached_registration) {
1409 warn!(
1410 module_id = %cached_registration.module_id,
1411 "capability claims drifted from the cached manifest"
1412 );
1413 }
1414 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1415 self.enforce_capability_denies();
1416 }
1417 self.refresh_capability_requirements();
1418 }
1419
1420 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1424 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1425 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1426 self.supervisor.set_swap_promotion_observer(observer);
1427 }
1428
1429 pub fn refresh_capability_requirements(&self) {
1430 match self.runtime_capability_snapshot() {
1431 Ok((runtime, registrations)) => {
1432 log_requirement_events(
1433 self.capability_evaluator
1434 .evaluate_now(&runtime, ®istrations),
1435 &self.daemon_shutdown,
1436 );
1437 }
1438 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1439 }
1440 }
1441
1442 fn enforce_capability_denies(&self) {
1446 let registrations = match self.registry.list_registered_modules() {
1447 Ok(snapshot) => snapshot,
1448 Err(err) => {
1449 warn!(error = %err, "failed to read registrations for capability deny census");
1450 return;
1451 }
1452 };
1453 let manifests = registrations
1454 .into_iter()
1455 .map(|registration| {
1456 (
1457 registration.manifest.module_id.clone(),
1458 registration.manifest,
1459 )
1460 })
1461 .collect::<BTreeMap<_, _>>();
1462 let census = match self.forwarding.route_census(None) {
1463 Ok(census) => census,
1464 Err(err) => {
1465 warn!(error = %err, "failed to read route census for capability deny enforcement");
1466 return;
1467 }
1468 };
1469
1470 for (target_module_id, routes) in census {
1471 let Some(target_manifest) = manifests.get(&target_module_id) else {
1472 continue;
1473 };
1474 let mut closed_routes = Vec::new();
1475 let mut goodbyes = Vec::new();
1476 for route in routes {
1477 let Principal::Reserved {
1478 module_id: opening_module_id,
1479 } = &route.principal
1480 else {
1481 continue;
1482 };
1483 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1484 continue;
1485 };
1486 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1487 continue;
1488 };
1489
1490 match self.forwarding.release_client_route(
1491 route.goodbye_target.connection_id,
1492 route.goodbye_target.channel,
1493 route.goodbye_target.epoch,
1494 ) {
1495 Ok(RouteRelease::Removed(module_goodbye)) => {
1496 warn!(
1497 opening_module_id,
1498 target_module_id,
1499 capability,
1500 "force-closing route because an attested capability deny edge now matches"
1501 );
1502 goodbyes.push(module_goodbye);
1503 goodbyes.push(route.goodbye_target.clone());
1504 closed_routes.push(route);
1505 }
1506 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1507 Err(err) => warn!(
1508 opening_module_id,
1509 target_module_id,
1510 capability,
1511 error = %err,
1512 "failed to force-close capability-denied route"
1513 ),
1514 }
1515 }
1516
1517 if closed_routes.is_empty() {
1518 continue;
1519 }
1520 send_route_control_pushes(
1521 &self.forwarding,
1522 closed_routes,
1523 ClientControlPush::RouteClosed {
1524 module_id: target_module_id,
1525 channels: Vec::new(),
1526 reason: RouteCloseReason::CapabilityDenied,
1527 drained: false,
1528 abandoned: 0,
1529 excluded_subscriptions: 0,
1530 terminal: Some(false),
1531 },
1532 );
1533 self.emit_route_goodbyes(goodbyes);
1536 }
1537 }
1538
1539 fn not_ready_reason(
1545 &self,
1546 registration: &crate::registry::ModuleRegistration,
1547 ) -> Option<NotReadyReason> {
1548 if !registration.ready {
1549 return Some(NotReadyReason {
1550 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1551 capability: None,
1552 });
1553 }
1554 self.first_unprovided_required_capability(registration)
1555 .map(|capability| NotReadyReason {
1556 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1557 capability: Some(capability),
1558 })
1559 }
1560
1561 fn first_unprovided_required_capability(
1579 &self,
1580 registration: &crate::registry::ModuleRegistration,
1581 ) -> Option<String> {
1582 let required = registration
1583 .manifest
1584 .capabilities
1585 .iter()
1586 .flat_map(|declarations| declarations.requires.iter())
1587 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1588 .map(|requirement| requirement.capability.as_str())
1589 .collect::<BTreeSet<_>>();
1590 if required.is_empty() {
1591 return None;
1592 }
1593 let module_id = registration.manifest.module_id.as_str();
1594 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1595 if required
1596 .iter()
1597 .any(|capability| verdict(capability).is_none())
1598 {
1599 self.refresh_capability_requirements();
1600 }
1601 required
1602 .into_iter()
1603 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1604 .map(str::to_string)
1605 }
1606
1607 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1608 self.capability_evaluator
1609 .statuses()
1610 .into_iter()
1611 .map(capability_requirement_status)
1612 .collect()
1613 }
1614
1615 fn deregister_connection(
1622 &self,
1623 connection_id: ConnectionId,
1624 reason: RegistrationEndReason,
1625 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1626 self.registry
1627 .deregister_connection_with_reason(connection_id, reason)
1628 }
1629
1630 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1631 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1632 return None;
1633 }
1634 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1635 parse_client_control_request(&frame.body)
1636 else {
1637 return None;
1638 };
1639 Some(target_module_id(&target).to_string())
1640 }
1641
1642 pub(crate) fn route_open_capacity_refusal(
1643 &self,
1644 ctx: &RouteCtx,
1645 frame: &Frame,
1646 target_module_id: &str,
1647 in_flight: usize,
1648 limit: usize,
1649 ) -> Result<Frame, RouterError> {
1650 let caller = self.route_open_caller_from_frame(frame);
1653 self.route_open_admission_refusal_frame(
1654 ctx,
1655 &caller,
1656 frame,
1657 target_module_id,
1658 "open_admission_full",
1659 (in_flight, limit),
1660 format!(
1661 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1662 ),
1663 )
1664 }
1665
1666 fn route_open_target_capacity_refusal(
1667 &self,
1668 ctx: &RouteCtx,
1669 caller: &RouteOpenCaller,
1670 frame: &Frame,
1671 target_module_id: &str,
1672 in_flight: usize,
1673 ) -> Result<Frame, RouterError> {
1674 self.route_open_admission_refusal_frame(
1675 ctx,
1676 caller,
1677 frame,
1678 target_module_id,
1679 "target_binds_full",
1680 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1681 format!(
1682 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1683 ),
1684 )
1685 }
1686
1687 #[allow(clippy::too_many_arguments)]
1695 fn route_open_admission_refusal_frame(
1696 &self,
1697 ctx: &RouteCtx,
1698 caller: &RouteOpenCaller,
1699 frame: &Frame,
1700 target_module_id: &str,
1701 reason: &'static str,
1702 (in_flight, limit): (usize, usize),
1703 message: impl Into<String>,
1704 ) -> Result<Frame, RouterError> {
1705 let code = error_codes::TARGET_UNAVAILABLE;
1706 self.counters.increment_route_open_refused(code);
1707 info!(
1708 target: "control",
1709 code,
1710 reason,
1711 module_id = ?target_module_id,
1712 connection_id = ctx.connection_id.get(),
1713 principal = %caller.principal,
1714 scope = %caller.scope,
1715 harness = %caller.harness,
1716 in_flight,
1717 limit,
1718 "route.open refused"
1719 );
1720 control_error_frame(frame, code, message.into())
1721 }
1722
1723 #[cfg(test)]
1729 pub fn handle_control(
1730 &self,
1731 connection_id: ConnectionId,
1732 frame: Frame,
1733 ) -> Result<Vec<Frame>, RouterError> {
1734 match frame.header.ty {
1735 FrameType::Ping => Ok(vec![pong(&frame)?]),
1736 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1737 FrameType::Goodbye => self.handle_goodbye(connection_id),
1738 ty => Ok(vec![control_error_frame(
1739 &frame,
1740 "unsupported_control_frame",
1741 format!("unsupported channel-0 frame {ty:?}"),
1742 )?]),
1743 }
1744 }
1745
1746 pub async fn handle_control_frame(
1747 &self,
1748 ctx: &RouteCtx,
1749 frame: Frame,
1750 ) -> Result<Vec<Frame>, RouterError> {
1751 self.handle_control_frame_timed(ctx, frame, None).await
1752 }
1753
1754 pub(crate) fn record_caller_endpoints(
1755 &self,
1756 connection: ConnectionId,
1757 endpoints: Option<(SocketAddr, SocketAddr)>,
1758 ) {
1759 let mut callers = self
1760 .caller_endpoints
1761 .lock()
1762 .unwrap_or_else(|p| p.into_inner());
1763 callers.remove(&connection);
1764 if let Some(endpoints) = endpoints {
1765 callers.insert(connection, endpoints);
1766 }
1767 }
1768
1769 #[cfg(test)]
1770 pub(crate) fn with_caller_lookup_for_test(
1771 mut self,
1772 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1773 ) -> Self {
1774 self.caller_lookup = Some(Arc::new(lookup));
1775 self
1776 }
1777
1778 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1779 let endpoints = self
1780 .caller_endpoints
1781 .lock()
1782 .unwrap_or_else(|p| p.into_inner())
1783 .get(&connection)
1784 .copied();
1785 #[cfg(test)]
1786 let lookup = self.caller_lookup.clone();
1787 tokio::task::spawn_blocking(move || {
1788 #[cfg(test)]
1789 if let Some(lookup) = lookup {
1790 return lookup(endpoints);
1791 }
1792 endpoints
1793 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1794 .unwrap_or_default()
1795 })
1796 .await
1797 .unwrap_or_default()
1798 }
1799
1800 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1801 frame.header.channel == 0
1802 && frame.header.ty == FrameType::Request
1803 && self
1804 .forwarding
1805 .module_endpoint_for_connection(ctx.connection_id)
1806 .is_ok_and(|endpoint| endpoint.is_none())
1807 && parse_client_control_request(&frame.body)
1808 .ok()
1809 .is_some_and(|request| privileged_operation(&request).is_some())
1810 }
1811
1812 pub(crate) async fn handle_control_frame_timed(
1813 &self,
1814 ctx: &RouteCtx,
1815 frame: Frame,
1816 dispatch_started_at: Option<StdInstant>,
1817 ) -> Result<Vec<Frame>, RouterError> {
1818 match frame.header.ty {
1819 FrameType::Ping => Ok(vec![pong(&frame)?]),
1820 FrameType::Hello => {
1821 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1822 }
1823 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1824 FrameType::Cancel => {
1825 if self
1828 .forwarding
1829 .operator_confirms()
1830 .cancel(ctx.connection_id, frame.header.corr)
1831 || self
1832 .supervisor
1833 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1834 || self.cancel_activity_subscription(ctx, &frame)?
1835 {
1836 Ok(Vec::new())
1837 } else {
1838 Ok(vec![control_error_frame(
1839 &frame,
1840 "unknown_subscription",
1841 "no subscription has this correlation id",
1842 )?])
1843 }
1844 }
1845 FrameType::Request => {
1846 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1849 if op
1850 .as_ref()
1851 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1852 {
1853 return self.handle_activity_record(ctx, frame).await;
1854 }
1855 if op.as_ref().is_some_and(|probe| {
1856 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1857 }) {
1858 return self.handle_activity_read(ctx, frame).await;
1859 }
1860 if op
1861 .as_ref()
1862 .is_some_and(|probe| probe.op == "operator.confirm")
1863 {
1864 return self.handle_operator_confirm(ctx, frame);
1865 }
1866 if op
1867 .as_ref()
1868 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1869 {
1870 return self.handle_scope_served_by(ctx.connection_id, frame);
1871 }
1872 if self
1873 .forwarding
1874 .module_endpoint_for_connection(ctx.connection_id)
1875 .map_err(RouterError::Forwarding)?
1876 .is_some()
1877 {
1878 if !is_known_module_request_op(&frame.body) {
1879 return Ok(vec![control_error_frame(
1880 &frame,
1881 "unsupported_control_frame",
1882 "module-originated channel-0 REQUEST is not supported",
1883 )?]);
1884 }
1885 let request = match parse_module_control_request_from_module(&frame.body) {
1886 Ok(request) => request,
1887 Err((err, ControlRequestBodyError::UnknownOp)) => {
1888 return Ok(vec![control_error_frame(
1889 &frame,
1890 "unsupported_control_frame",
1891 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1892 )?])
1893 }
1894 Err((err, ControlRequestBodyError::InvalidBody)) => {
1895 return Ok(vec![control_error_frame(
1896 &frame,
1897 "invalid_control_body",
1898 format!("malformed module control body: {err}"),
1899 )?])
1900 }
1901 };
1902 let op = module_control_request_op(&request);
1903 let corr = frame.header.corr;
1904 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1905 let result =
1906 self.handle_module_control_request(ctx.connection_id, frame, request);
1907 log_slow_control_dispatch(
1908 dispatch_started_at,
1909 op,
1910 ctx.connection_id,
1911 corr,
1912 None,
1913 );
1914 return result;
1915 }
1916
1917 if is_known_module_request_op(&frame.body) {
1918 return Ok(vec![control_error_frame(
1919 &frame,
1920 "not_registered",
1921 "catalog.update requires an active module registration owned by this connection",
1922 )?]);
1923 }
1924
1925 let request = match parse_client_control_request(&frame.body) {
1926 Ok(request) => request,
1927 Err((err, ControlRequestBodyError::UnknownOp)) => {
1928 return Ok(vec![control_error_frame(
1929 &frame,
1930 "unknown_control_op",
1931 format!("unknown client control op: {err}"),
1932 )?])
1933 }
1934 Err((err, ControlRequestBodyError::InvalidBody)) => {
1935 return Ok(vec![control_error_frame(
1936 &frame,
1937 "invalid_control_body",
1938 format!("malformed client control body: {err}"),
1939 )?])
1940 }
1941 };
1942 let op = client_control_request_op(&request);
1943 let corr = frame.header.corr;
1944 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1945 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1946 #[cfg(test)]
1947 if let Some(delay) = self.control_dispatch_delay {
1948 tokio::time::sleep(delay).await;
1949 }
1950 let result = self
1951 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1952 .await;
1953 if let Some(timings) = route_open_timings.as_mut() {
1954 timings.finish();
1955 }
1956 log_slow_control_dispatch(
1957 dispatch_started_at,
1958 op,
1959 ctx.connection_id,
1960 corr,
1961 route_open_timings.as_ref(),
1962 );
1963 result
1964 }
1965 FrameType::Push => {
1966 let Some(endpoint) = self
1967 .forwarding
1968 .module_endpoint_for_connection(ctx.connection_id)
1969 .map_err(RouterError::Forwarding)?
1970 else {
1971 return Ok(vec![control_error_frame(
1972 &frame,
1973 "unsupported_control_frame",
1974 "client-originated channel-0 PUSH is not supported",
1975 )?]);
1976 };
1977 self.handle_status_update(endpoint, frame)
1978 }
1979 FrameType::Response | FrameType::Error
1980 if self
1981 .forwarding
1982 .module_endpoint_for_connection(ctx.connection_id)
1983 .map_err(RouterError::Forwarding)?
1984 .is_some() =>
1985 {
1986 self.handle_module_relay_response(ctx.connection_id, frame)
1987 }
1988 ty => Ok(vec![control_error_frame(
1989 &frame,
1990 "unsupported_control_frame",
1991 format!("unsupported channel-0 frame {ty:?}"),
1992 )?]),
1993 }
1994 }
1995
1996 pub fn cleanup_connection(
1997 &self,
1998 connection_id: ConnectionId,
1999 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2000 self.caller_endpoints
2001 .lock()
2002 .unwrap_or_else(|p| p.into_inner())
2003 .remove(&connection_id);
2004 self.forwarding
2005 .operator_confirms()
2006 .connection_closed(connection_id);
2007 self.cleanup_connection_with_end_reason(
2008 connection_id,
2009 RegistrationEndReason::ConnectionClosed,
2010 )
2011 }
2012
2013 fn cleanup_connection_with_end_reason(
2014 &self,
2015 connection_id: ConnectionId,
2016 requested_reason: RegistrationEndReason,
2017 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2018 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2019 self.registry
2020 .get_module_by_connection(connection_id)?
2021 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2022 .and_then(|module| module.registration_end_reason().ok().flatten())
2023 .unwrap_or(requested_reason)
2024 } else {
2025 requested_reason
2026 };
2027 let crash_closed = self
2028 .registry
2029 .get_module_by_connection(connection_id)?
2030 .and_then(|registration| {
2031 self.forwarding
2032 .module_endpoint_for_connection(connection_id)
2033 .ok()
2034 .flatten()
2035 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2036 .map(|routes| (registration.manifest.module_id, routes))
2037 });
2038 let crash_closed = crash_closed.map(|(module_id, routes)| {
2039 let terminal = match self.supervisor.get(&module_id) {
2040 None => false,
2041 Some(module) => match module.will_recover_after_connection_loss() {
2042 Ok(will_recover) => !will_recover,
2043 Err(err) => {
2044 warn!(
2045 %module_id,
2046 error = %err,
2047 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2048 );
2049 false
2050 }
2051 },
2052 };
2053 let (reason, drained) = match self
2057 .forwarding
2058 .daemon_drain_result_for_connection(connection_id)
2059 {
2060 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2061 Ok(None) => (RouteCloseReason::Crash, false),
2062 Err(err) => {
2063 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2064 (RouteCloseReason::Crash, false)
2065 }
2066 };
2067 (module_id, routes, reason, drained, terminal)
2068 });
2069 let registrations = self.deregister_connection(connection_id, end_reason);
2070 let cleanup = if crash_closed.is_some() {
2071 self.forwarding.cleanup_connection_counted(connection_id)
2072 } else {
2073 self.forwarding
2079 .cleanup_connection(connection_id)
2080 .map(|released| crate::forwarding::ConnectionCleanup {
2081 released,
2082 abandoned_relays: 0,
2083 })
2084 };
2085 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2090 let abandoned = cleanup
2091 .as_ref()
2092 .map_or(0, |cleanup| cleanup.abandoned_relays);
2093 send_route_control_pushes(
2094 &self.forwarding,
2095 routes,
2096 ClientControlPush::RouteClosed {
2097 module_id,
2098 channels: Vec::new(),
2099 reason,
2100 drained,
2101 abandoned,
2102 excluded_subscriptions: 0,
2103 terminal: Some(terminal),
2104 },
2105 );
2106 }
2107 if let Ok(cleanup) = cleanup {
2108 self.emit_route_goodbyes(cleanup.released);
2109 }
2110 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2114 crate::supervise::notify_registration_release();
2115 self.capability_evaluator.wake_deadline_loop();
2116 self.refresh_capability_requirements();
2117 }
2118 self.supervisor.remove_spawn_subscribers(connection_id);
2119 self.activity_streams
2120 .lock()
2121 .unwrap_or_else(|p| p.into_inner())
2122 .retain(|(id, _), task| {
2123 if *id == connection_id {
2124 task.abort();
2125 false
2126 } else {
2127 true
2128 }
2129 });
2130 self.hello_launch_nonces
2133 .lock()
2134 .unwrap_or_else(|poisoned| poisoned.into_inner())
2135 .forget(connection_id);
2136 self.scopes
2137 .write()
2138 .unwrap_or_else(|poisoned| poisoned.into_inner())
2139 .release_connection(connection_id);
2140 registrations
2141 }
2142
2143 pub(crate) fn handle_route_goodbye(
2144 &self,
2145 connection_id: ConnectionId,
2146 route_channel: u16,
2147 route_epoch: u32,
2148 ) -> Result<bool, RouterError> {
2149 debug!(
2150 connection_id = connection_id.get(),
2151 route_channel, route_epoch, "handling route GOODBYE"
2152 );
2153 let RouteRelease::Removed(released_route) = self
2154 .forwarding
2155 .release_client_route(connection_id, route_channel, route_epoch)
2156 .map_err(RouterError::Forwarding)?
2157 else {
2158 return Ok(false);
2159 };
2160 self.emit_route_goodbyes(vec![released_route]);
2161 Ok(true)
2162 }
2163
2164 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2165 for released in released_routes {
2166 let frame = match Frame::build_with_version(
2167 released.negotiated_ver,
2168 FrameType::Goodbye,
2169 control_flags(),
2170 released.channel,
2171 released.epoch,
2172 0,
2173 Vec::new(),
2174 ) {
2175 Ok(frame) => frame,
2176 Err(err) => {
2177 warn!(
2178 route_channel = released.channel,
2179 error = %err,
2180 "failed to build route GOODBYE frame"
2181 );
2182 continue;
2183 }
2184 };
2185 if !released.close_on_delivery_failure() {
2186 crate::forwarding::send_module_route_goodbye(
2187 &self.counters,
2188 &released.sink,
2189 frame,
2190 released.module_id.as_deref(),
2191 "client route released",
2192 );
2193 continue;
2194 }
2195 if let Err(err) = released.sink.try_send(frame) {
2196 warn!(
2197 target_connection_id = released.connection_id.get(),
2198 route_channel = released.channel,
2199 error = %err,
2200 "route GOODBYE was not delivered to client; closing target connection"
2201 );
2202 if self
2203 .forwarding
2204 .escalate_client_delivery_failure(
2205 released.connection_id,
2206 released.channel,
2207 released.epoch,
2208 CloseReason::new(
2209 "route_goodbye_delivery_failed",
2210 format!(
2211 "failed to enqueue route GOODBYE for channel {}: {err}",
2212 released.channel
2213 ),
2214 ),
2215 crate::forwarding::UndeliveredFrame {
2216 module_id: released.module_id.as_deref(),
2217 sink: &released.sink,
2218 },
2219 )
2220 .unwrap_or(false)
2221 {
2222 self.counters.increment_goodbye_relay_client_failed();
2223 }
2224 }
2225 }
2226 }
2227
2228 fn send_abandoned_route_bind_goodbye(
2240 &self,
2241 module_sink: &crate::FrameSink,
2242 negotiated_ver: u8,
2243 module_channel: u16,
2244 module_epoch: u32,
2245 ) {
2246 let frame = match Frame::build_with_version(
2247 negotiated_ver,
2248 FrameType::Goodbye,
2249 control_flags(),
2250 module_channel,
2251 module_epoch,
2252 0,
2253 Vec::new(),
2254 ) {
2255 Ok(frame) => frame,
2256 Err(err) => {
2257 warn!(
2258 route_channel = module_channel,
2259 error = %err,
2260 "failed to build GOODBYE for abandoned route.bind"
2261 );
2262 return;
2263 }
2264 };
2265 crate::forwarding::send_module_route_goodbye(
2266 &self.counters,
2267 module_sink,
2268 frame,
2269 None,
2270 "abandoned route.bind",
2271 );
2272 }
2273
2274 fn handle_hello(
2275 &self,
2276 connection_id: ConnectionId,
2277 sink: Option<crate::FrameSink>,
2278 frame: Frame,
2279 ) -> Result<Vec<Frame>, RouterError> {
2280 debug!(
2281 connection_id = connection_id.get(),
2282 corr = frame.header.corr,
2283 "handling HELLO"
2284 );
2285 if self
2289 .registry
2290 .get_module_by_connection(connection_id)
2291 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2292 .is_some()
2293 {
2294 return Ok(vec![control_error_frame(
2295 &frame,
2296 "invalid_hello",
2297 "connection is already registered as a module",
2298 )?]);
2299 }
2300 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2301 Ok(value) => value,
2302 Err(err) => {
2303 return Ok(vec![control_error_frame(
2304 &frame,
2305 "invalid_hello",
2306 format!("malformed HELLO body: {err}"),
2307 )?])
2308 }
2309 };
2310 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2311 return Ok(vec![control_error_frame(
2312 &frame,
2313 "invalid_capability_grammar",
2314 err.to_string(),
2315 )?]);
2316 }
2317 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2318 return Ok(vec![control_error_frame(
2319 &frame,
2320 "invalid_manifest",
2321 err.to_string(),
2322 )?]);
2323 }
2324 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2325 return Ok(vec![control_error_frame(
2326 &frame,
2327 "invalid_event_declaration",
2328 err.to_string(),
2329 )?]);
2330 }
2331 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2332 return Ok(vec![control_error_frame(
2333 &frame,
2334 "invalid_manifest",
2335 err.to_string(),
2336 )?]);
2337 }
2338 if let Some(provenance) = hello_value
2339 .get("manifest")
2340 .and_then(|manifest| manifest.get("provenance"))
2341 {
2342 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2343 return Ok(vec![control_error_frame(
2344 &frame,
2345 "invalid_manifest",
2346 format!("malformed manifest provenance: {err}"),
2347 )?]);
2348 }
2349 }
2350 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2351 let module_id = hello_value["manifest"]["module_id"]
2352 .as_str()
2353 .unwrap_or("<unknown>");
2354 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2355 }
2356 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2357 Ok(hello) => hello,
2358 Err(err) => {
2359 return Ok(vec![control_error_frame(
2360 &frame,
2361 "invalid_hello",
2362 format!("malformed HELLO body: {err}"),
2363 )?])
2364 }
2365 };
2366
2367 if hello.protocol_ver != hello.manifest.protocol_ver {
2368 return Ok(vec![control_error_frame(
2369 &frame,
2370 "invalid_manifest",
2371 format!(
2372 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2373 hello.protocol_ver, hello.manifest.protocol_ver
2374 ),
2375 )?]);
2376 }
2377
2378 if hello.manifest.module_id.trim().is_empty() {
2379 return Ok(vec![control_error_frame(
2380 &frame,
2381 "invalid_manifest",
2382 "manifest module_id must not be empty",
2383 )?]);
2384 }
2385
2386 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2387 Ok(negotiated_ver) => negotiated_ver,
2388 Err(message) => {
2389 return Ok(vec![control_error_frame(
2390 &frame,
2391 "version_unsupported",
2392 message,
2393 )?])
2394 }
2395 };
2396
2397 let swap_admission = self
2405 .supervisor
2406 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2407 if swap_admission == SwapHelloAdmission::Refused {
2408 warn!(
2409 module_id = %hello.manifest.module_id,
2410 connection_id = connection_id.get(),
2411 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2412 );
2413 return Ok(vec![control_error_frame(
2414 &frame,
2415 "swap_token_invalid",
2416 format!(
2417 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2418 hello.manifest.module_id
2419 ),
2420 )?]);
2421 }
2422 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2423
2424 let nonce_verified = if swap_candidate {
2429 true
2430 } else {
2431 match self.supervisor.reserved_hello_verification(
2432 &hello.manifest.module_id,
2433 hello.launch_nonce.as_deref(),
2434 ) {
2435 Ok(verified) => verified,
2436 Err(rejection) => {
2437 let message = match rejection {
2438 ReservedHelloRejection::Exact { module_id } => format!(
2439 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2440 ),
2441 ReservedHelloRejection::Prefix {
2442 prefix,
2443 owner_module_id,
2444 } => format!(
2445 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2446 hello.manifest.module_id
2447 ),
2448 };
2449 return Ok(vec![control_error_frame(
2450 &frame,
2451 "reserved_module",
2452 message,
2453 )?]);
2454 }
2455 }
2456 };
2457
2458 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2459 &hello.manifest.module_id,
2460 hello.manifest.capabilities.as_ref(),
2461 );
2462 if let Some(refusal) = reserved_capability_refusals.first() {
2463 let capability = refusal.capability.clone();
2464 let bound_module = refusal.claimants[0].clone();
2465 log_duplicate_claim_events(reserved_capability_refusals);
2466 return Ok(vec![control_error_frame(
2467 &frame,
2468 "reserved_capability",
2469 format!(
2470 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2471 capability, bound_module, hello.manifest.module_id
2472 ),
2473 )?]);
2474 }
2475
2476 if self
2479 .forwarding
2480 .connection_has_client_routes(connection_id)
2481 .map_err(RouterError::Forwarding)?
2482 {
2483 return Ok(vec![control_error_frame(
2484 &frame,
2485 "invalid_hello",
2486 "connection has open client routes and cannot also register as a module",
2487 )?]);
2488 }
2489
2490 self.hello_launch_nonces
2495 .lock()
2496 .unwrap_or_else(|poisoned| poisoned.into_inner())
2497 .record(connection_id, hello.launch_nonce.as_deref());
2498 let control_ops = effective_module_control_ops(hello.control_ops);
2499 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2502 if swap_candidate {
2503 return self.register_swap_candidate(
2504 connection_id,
2505 sink,
2506 &frame,
2507 hello.manifest,
2508 negotiated_ver,
2509 control_ops,
2510 hello_ack,
2511 );
2512 }
2513 let registration = match self
2520 .registry
2521 .register_unpublished_verified_with_control_ops(
2522 hello.manifest,
2523 negotiated_ver,
2524 connection_id,
2525 control_ops,
2526 nonce_verified,
2527 ) {
2528 Ok(registration) => registration,
2529 Err(RegistryError::DuplicateModuleId { module_id }) => {
2530 return Ok(vec![control_error_frame(
2531 &frame,
2532 "duplicate_module_id",
2533 format!(
2534 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2535 ),
2536 )?])
2537 }
2538 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2539 return Ok(vec![control_error_frame(
2540 &frame,
2541 "invalid_module_id",
2542 err.to_string(),
2543 )?])
2544 }
2545 Err(err) => {
2546 return Ok(vec![control_error_frame(
2547 &frame,
2548 "registry_error",
2549 err.to_string(),
2550 )?])
2551 }
2552 };
2553 #[cfg(test)]
2554 self.hold_between_registry_and_forwarding();
2555
2556 let reply = if let Some(sink) = sink {
2557 let concurrency = manifest_concurrency(®istration.manifest);
2573 if let Err(err) = self.forwarding.register_module_connection_acked(
2574 connection_id,
2575 registration.manifest.module_id.clone(),
2576 negotiated_ver,
2577 concurrency,
2578 sink,
2579 hello_ack,
2580 ) {
2581 if matches!(
2586 self.deregister_connection(
2587 connection_id,
2588 RegistrationEndReason::RegistrationFailed,
2589 ),
2590 Ok(r) if !r.is_empty()
2591 ) {
2592 crate::supervise::notify_registration_release();
2593 }
2594 return Ok(vec![control_error_frame(
2595 &frame,
2596 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2597 "invalid_hello"
2598 } else {
2599 forwarding_error_code(&err)
2600 },
2601 err.to_string(),
2602 )?]);
2603 }
2604 Vec::new()
2605 } else {
2606 vec![hello_ack]
2609 };
2610
2611 if !self
2616 .registry
2617 .publish_registration(connection_id)
2618 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2619 {
2620 debug!(
2621 module_id = %registration.manifest.module_id,
2622 connection_id = connection_id.get(),
2623 "registration was removed before it could be published"
2624 );
2625 }
2626
2627 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2636 info!(
2637 module_id = %registration.manifest.module_id,
2638 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2639 );
2640 }
2641
2642 self.apply_registration_capabilities(®istration);
2643
2644 self.supervisor
2645 .record_activity_registration(®istration.manifest.module_id);
2646
2647 info!(
2648 module_id = %registration.manifest.module_id,
2649 module_version = %registration.manifest.module_version,
2650 negotiated_ver,
2651 routable_provider = manifest_provides_routable_role(®istration.manifest),
2652 connection_id = connection_id.get(),
2653 "module registered"
2654 );
2655
2656 Ok(reply)
2657 }
2658
2659 #[allow(clippy::too_many_arguments)]
2676 fn register_swap_candidate(
2677 &self,
2678 connection_id: ConnectionId,
2679 sink: Option<crate::FrameSink>,
2680 frame: &Frame,
2681 manifest: ModuleManifest,
2682 negotiated_ver: u8,
2683 control_ops: Vec<String>,
2684 hello_ack: Frame,
2685 ) -> Result<Vec<Frame>, RouterError> {
2686 let module_id = manifest.module_id.clone();
2687 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2688 manifest,
2689 negotiated_ver,
2690 connection_id,
2691 control_ops,
2692 ) {
2693 Ok(registration) => registration,
2694 Err(RegistryError::DuplicateModuleId { module_id }) => {
2695 return Ok(vec![control_error_frame(
2696 frame,
2697 "duplicate_module_id",
2698 format!(
2699 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2700 ),
2701 )?])
2702 }
2703 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2704 return Ok(vec![control_error_frame(
2705 frame,
2706 "invalid_module_id",
2707 err.to_string(),
2708 )?])
2709 }
2710 Err(err) => {
2711 return Ok(vec![control_error_frame(
2712 frame,
2713 "registry_error",
2714 err.to_string(),
2715 )?])
2716 }
2717 };
2718 #[cfg(test)]
2719 self.hold_between_registry_and_forwarding();
2720 let reply = if let Some(sink) = sink {
2721 let concurrency = manifest_concurrency(®istration.manifest);
2725 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2726 connection_id,
2727 module_id.clone(),
2728 negotiated_ver,
2729 concurrency,
2730 sink,
2731 hello_ack,
2732 ) {
2733 if matches!(
2734 self.deregister_connection(
2735 connection_id,
2736 RegistrationEndReason::RegistrationFailed,
2737 ),
2738 Ok(r) if !r.is_empty()
2739 ) {
2740 crate::supervise::notify_registration_release();
2741 }
2742 return Ok(vec![control_error_frame(
2743 frame,
2744 forwarding_error_code(&err),
2745 err.to_string(),
2746 )?]);
2747 }
2748 Vec::new()
2749 } else {
2750 vec![hello_ack]
2751 };
2752 if !self
2753 .registry
2754 .publish_registration(connection_id)
2755 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2756 {
2757 debug!(
2758 module_id = %module_id,
2759 connection_id = connection_id.get(),
2760 "swap candidate registration was removed before it could be published"
2761 );
2762 }
2763 self.supervisor.mark_swap_candidate_admitted(&module_id);
2764 info!(
2765 module_id = %module_id,
2766 module_version = %registration.manifest.module_version,
2767 negotiated_ver,
2768 ready = registration.ready,
2769 connection_id = connection_id.get(),
2770 "swap candidate registered; not routable until cutover"
2771 );
2772 Ok(reply)
2773 }
2774
2775 fn build_hello_ack(
2776 &self,
2777 frame: &Frame,
2778 negotiated_ver: u8,
2779 module_id: &str,
2780 ) -> Result<Frame, RouterError> {
2781 let ack = ModuleHelloAckBody {
2782 negotiated_ver,
2783 subc_ops: module_subc_ops(),
2784 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2785 storage: self
2786 .storage_config
2787 .as_ref()
2788 .map(|cfg| cfg.descriptor_for(module_id)),
2789 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2790 };
2791 let body = serde_json::to_vec(&ack).map_err(|err| {
2792 RouterError::backend(
2793 0,
2794 frame.header.corr,
2795 format!("failed to encode HELLO_ACK: {err}"),
2796 )
2797 })?;
2798
2799 Frame::build_with_version(
2800 negotiated_ver,
2801 FrameType::HelloAck,
2802 control_flags(),
2803 0,
2804 0,
2805 frame.header.corr,
2806 body,
2807 )
2808 .map_err(RouterError::FrameBuild)
2809 }
2810
2811 async fn handle_client_control_request(
2812 &self,
2813 ctx: &RouteCtx,
2814 frame: Frame,
2815 request: ClientControlRequest,
2816 route_open_timings: &mut Option<RouteOpenTimings>,
2817 ) -> Result<Vec<Frame>, RouterError> {
2818 if let Some(operation) = privileged_operation(&request) {
2819 let caller = self.resolve_caller(ctx.connection_id).await;
2820 let op = client_control_request_op(&request);
2821 let text = prompt_text(operation, &caller);
2822 match self.privileged_operator_prompt {
2823 PrivilegedOperatorPrompt::Log => {
2824 info!(target: "subc_daemon::privileged_prompt", operation = op,
2825 target = ?operation, caller = %caller.description(), summary = %text,
2826 "privileged request would prompt the operator");
2827 }
2828 PrivilegedOperatorPrompt::Enforce => {
2829 let key =
2830 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2831 let outcome = self
2832 .forwarding
2833 .operator_confirms()
2834 .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2835 .await;
2836 if outcome.code != "confirmed" {
2837 return Ok(vec![outcome.frame(&frame)]);
2838 }
2839 }
2840 }
2841 }
2842 match request {
2843 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2844 ClientControlRequest::CatalogList { module_id } => {
2845 self.handle_catalog_list(frame, module_id)
2846 }
2847 ClientControlRequest::RouteOpen {
2848 target,
2849 identity,
2850 consumer_identity,
2851 consumer_capabilities,
2852 role_versions,
2853 admission_facts,
2854 scope,
2855 } => {
2856 self.handle_route_open(
2857 ctx,
2858 frame,
2859 RouteOpenRequest {
2860 target,
2861 identity,
2862 consumer_identity,
2863 consumer_capabilities,
2864 role_versions,
2865 admission_facts,
2866 scope,
2867 },
2868 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2869 )
2870 .await
2871 }
2872 ClientControlRequest::RoutePoll {
2873 route_channel,
2874 route_epoch,
2875 kind,
2876 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2877 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2878 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2879 self.handle_supervisor_spawn_snapshot(frame)
2880 }
2881 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2882 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2883 }
2884 ClientControlRequest::SupervisorRestart {
2885 module_id,
2886 drain_timeout_ms,
2887 } => {
2888 self.log_supervisor_request_received(
2889 ctx,
2890 frame.header.corr,
2891 ops::SUPERVISOR_RESTART,
2892 Some(&module_id),
2893 None,
2894 )?;
2895 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2896 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2897 .await
2898 }
2899 ClientControlRequest::SupervisorSwap {
2900 module_id,
2901 ready_timeout_ms,
2902 } => {
2903 self.log_supervisor_request_received(
2904 ctx,
2905 frame.header.corr,
2906 ops::SUPERVISOR_SWAP,
2907 Some(&module_id),
2908 None,
2909 )?;
2910 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2911 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2912 .await
2913 }
2914 ClientControlRequest::SupervisorReload { module_id } => {
2915 self.log_supervisor_request_received(
2916 ctx,
2917 frame.header.corr,
2918 ops::SUPERVISOR_RELOAD,
2919 Some(&module_id),
2920 None,
2921 )?;
2922 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2923 self.handle_supervisor_reload(frame, module_id, requester)
2924 .await
2925 }
2926 ClientControlRequest::SupervisorRescan { preview } => {
2927 if !preview {
2928 self.log_supervisor_request_received(
2929 ctx,
2930 frame.header.corr,
2931 ops::SUPERVISOR_RESCAN,
2932 None,
2933 None,
2934 )?;
2935 }
2936 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2937 self.handle_supervisor_rescan(
2938 frame,
2939 if preview {
2940 RescanMode::Preview
2941 } else {
2942 RescanMode::Execute
2943 },
2944 requester,
2945 )
2946 .await
2947 }
2948 ClientControlRequest::SupervisorRescanChecked {
2949 preview,
2950 candidate,
2951 expect,
2952 } => {
2953 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
2954 .await
2955 }
2956 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2957 self.log_supervisor_request_received(
2958 ctx,
2959 frame.header.corr,
2960 ops::SUPERVISOR_RELEASE_RESERVED,
2961 Some(&module_id),
2962 None,
2963 )?;
2964 self.handle_supervisor_release_reserved(frame, module_id)
2965 .await
2966 }
2967 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2968 self.log_supervisor_request_received(
2969 ctx,
2970 frame.header.corr,
2971 ops::SUPERVISOR_SET_ENABLED,
2972 Some(&module_id),
2973 Some(enabled),
2974 )?;
2975 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2976 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2977 .await
2978 }
2979 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2980 self.handle_supervisor_health_probe(frame, module_id).await
2981 }
2982 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2983 ClientControlRequest::SupervisorRoutes { module_id } => {
2984 self.handle_supervisor_routes(frame, module_id)
2985 }
2986 ClientControlRequest::SupervisorProvenance { module_id } => {
2987 self.handle_supervisor_provenance(frame, module_id).await
2988 }
2989 ClientControlRequest::SupervisorStderrTail {
2990 module_id,
2991 max_lines,
2992 max_bytes,
2993 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2994 ClientControlRequest::SupervisorTerminals { module_id } => {
2995 self.handle_supervisor_terminals(frame, module_id).await
2996 }
2997 }
2998 }
2999
3000 fn log_supervisor_request_received(
3001 &self,
3002 ctx: &RouteCtx,
3003 corr: u64,
3004 op: &'static str,
3005 module_id: Option<&str>,
3006 enabled: Option<bool>,
3007 ) -> Result<(), RouterError> {
3008 let caller = self.supervisor_requester(ctx, corr)?;
3009 let caller = principal_label(&caller);
3010 info!(
3013 target: "control",
3014 op,
3015 module_id,
3016 enabled,
3017 connection_id = ctx.connection_id.get(),
3018 caller = %caller,
3019 "supervisor request received"
3020 );
3021 Ok(())
3022 }
3023
3024 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3025 Ok(self
3026 .registry
3027 .get_module_by_connection(ctx.connection_id)
3028 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3029 .map(|registration| Principal::Reserved {
3030 module_id: registration.manifest.module_id,
3031 })
3032 .unwrap_or(Principal::Direct))
3033 }
3034
3035 fn handle_module_control_request(
3036 &self,
3037 connection_id: ConnectionId,
3038 frame: Frame,
3039 request: ModuleControlRequestFromModule,
3040 ) -> Result<Vec<Frame>, RouterError> {
3041 match request {
3042 ModuleControlRequestFromModule::CatalogUpdate {
3043 provides,
3044 capabilities,
3045 ready,
3046 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3047 ModuleControlRequestFromModule::LiveRoots {} => {
3048 let registered = self
3049 .registry
3050 .get_module_by_connection(connection_id)
3051 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3052 let Some(registration) = registered else {
3053 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3054 };
3055 let response = self
3056 .forwarding
3057 .live_roots(®istration.manifest.module_id)
3058 .map_err(RouterError::Forwarding)?;
3059 Ok(vec![control_response_body_frame(
3060 &frame,
3061 &response,
3062 "ModuleControlResponseToModule::LiveRoots",
3063 )?])
3064 }
3065 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3066 self.handle_scope_sync(connection_id, frame, generation, scopes)
3067 }
3068 ModuleControlRequestFromModule::ScopeApply {
3069 generation,
3070 upsert,
3071 end,
3072 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3073 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3074 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3075 }
3076 }
3077 }
3078
3079 fn handle_scope_served_by(
3080 &self,
3081 connection_id: ConnectionId,
3082 frame: Frame,
3083 ) -> Result<Vec<Frame>, RouterError> {
3084 let registered = self
3085 .registry
3086 .get_module_by_connection(connection_id)
3087 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3088 let Some(registered) = registered else {
3089 return Ok(vec![control_error_frame(
3090 &frame,
3091 "not_registered",
3092 "scope.served_by requires an active module registration owned by this connection",
3093 )?]);
3094 };
3095 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3096 {
3097 Ok(request) => request,
3098 Err(err) => {
3099 return Ok(vec![control_error_frame(
3100 &frame,
3101 "invalid_control_body",
3102 format!("malformed scope.served_by body: {err}"),
3103 )?])
3104 }
3105 };
3106 let answer = self
3107 .scopes
3108 .read()
3109 .unwrap_or_else(|poisoned| poisoned.into_inner())
3110 .served_by(
3111 ®istered.manifest.module_id,
3112 &request.scope_ref,
3113 request.scope_epoch,
3114 &request.module_id,
3115 );
3116 let reply = subc_control::ScopeServedByReply::new(
3117 answer,
3118 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3119 );
3120 Ok(vec![control_response_body_frame(
3121 &frame,
3122 &reply,
3123 "ScopeServedByReply",
3124 )?])
3125 }
3126
3127 fn handle_operator_confirm(
3128 &self,
3129 ctx: &RouteCtx,
3130 frame: Frame,
3131 ) -> Result<Vec<Frame>, RouterError> {
3132 use crate::operator_confirm::{audit, Outcome};
3133 let registration = self
3134 .registry
3135 .get_module_by_connection(ctx.connection_id)
3136 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3137 let Some(registration) = registration else {
3138 let outcome = Outcome::refusal("not_registered");
3139 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3140 return Ok(vec![outcome.frame(&frame)]);
3141 };
3142 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3143 Ok(request) => request,
3144 Err(_) => {
3145 let outcome = Outcome::refusal("invalid_control_body");
3146 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3147 return Ok(vec![outcome.frame(&frame)]);
3148 }
3149 };
3150 let module_id = registration.manifest.module_id;
3151 let nonce = self
3155 .hello_launch_nonces
3156 .lock()
3157 .unwrap_or_else(|p| p.into_inner())
3158 .nonce(ctx.connection_id)
3159 .map(str::to_owned);
3160 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3161 self.supervisor
3162 .spawned_consumer_authorized(&module_id, nonce)
3163 });
3164 let confirms = self.forwarding.operator_confirms();
3165 self.forwarding
3166 .with_operator_route(
3167 ctx.connection_id,
3168 request.route_channel,
3169 request.route_epoch,
3170 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3171 )
3172 .map_err(RouterError::Forwarding)
3173 }
3174
3175 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3178 let mut scopes = ScopeRecords::from(scopes);
3179 for (index, record) in scopes.records.iter_mut().enumerate() {
3180 if let Some(refusal) = Self::canonicalize_workdir(record) {
3181 scopes.refusals.insert(index, refusal);
3182 }
3183 }
3184 scopes
3185 }
3186
3187 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3188 let workdir = record.workdir.as_mut()?;
3189 let path = std::path::Path::new(workdir);
3190 if !path.is_absolute() {
3191 return Some((
3192 error_codes::INVALID_WORKDIR,
3193 "workdir must be absolute".to_string(),
3194 ));
3195 }
3196 match ProjectRootId::from_path_allowing_missing(path) {
3197 Ok(root) => match root.as_path().to_str() {
3198 Some(canonical) => {
3199 *workdir = canonical.to_string();
3200 None
3201 }
3202 None => Some((
3203 error_codes::INVALID_WORKDIR,
3204 "canonical workdir is not UTF-8".to_string(),
3205 )),
3206 },
3207 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3208 }
3209 }
3210
3211 fn handle_scope_sync(
3215 &self,
3216 connection_id: ConnectionId,
3217 frame: Frame,
3218 generation: u64,
3219 scopes: Vec<ScopeRecord>,
3220 ) -> Result<Vec<Frame>, RouterError> {
3221 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3222 }
3223
3224 fn handle_scope_change(
3225 &self,
3226 connection_id: ConnectionId,
3227 frame: Frame,
3228 generation: u64,
3229 scopes: Vec<ScopeRecord>,
3230 end: Option<Vec<ScopeEnd>>,
3231 ) -> Result<Vec<Frame>, RouterError> {
3232 let is_apply = end.is_some();
3233 let Some(registration) = self
3234 .registry
3235 .get_module_by_connection(connection_id)
3236 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3237 else {
3238 return Ok(vec![control_error_frame(
3239 &frame,
3240 "not_registered",
3241 "scope.sync and scope.apply require an active module registration owned by this connection",
3242 )?]);
3243 };
3244 let owner = registration.manifest.module_id;
3245 let scopes = Self::prepare_scope_records(scopes);
3246 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3247 let is_current_launch = |connection: ConnectionId| {
3248 self.hello_launch_nonces
3249 .lock()
3250 .unwrap_or_else(|poisoned| poisoned.into_inner())
3251 .presented(connection, current_nonce.as_deref())
3252 };
3253 let mut table = self
3258 .scopes
3259 .write()
3260 .unwrap_or_else(|poisoned| poisoned.into_inner());
3261 let outcome = match end {
3262 Some(end) => table.apply(
3263 &owner,
3264 connection_id,
3265 is_current_launch,
3266 generation,
3267 scopes,
3268 end,
3269 ),
3270 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3271 };
3272 let drained = match &outcome {
3273 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3274 Err(_) => Vec::new(),
3275 };
3276 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3277 drop(table);
3278 if has_deadlines {
3279 self.scope_deadline_added.notify_one();
3280 }
3281 match outcome {
3282 Ok(applied) => {
3283 let counts = ScopeOutcomeCounts::of(&applied.results);
3284 info!(
3285 owner = %owner,
3286 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3287 generation,
3288 records = applied.results.len(),
3289 created = counts.created,
3290 replaced = counts.replaced,
3291 updated = counts.updated,
3292 unchanged = counts.unchanged,
3293 refused = counts.refused,
3294 ended = applied.ended.len(),
3295 tag_changes = applied.tag_changes.len(),
3296 routes_closed = drained.len(),
3297 "scope change accepted"
3298 );
3299 for refused in applied
3305 .results
3306 .iter()
3307 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3308 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3309 {
3310 warn!(
3311 owner = %owner,
3312 generation,
3313 scope_ref = %refused.scope_ref,
3314 scope_epoch = refused.scope_epoch,
3315 code = refused.code.as_deref().unwrap_or(""),
3316 "scope record refused"
3317 );
3318 }
3319 self.close_scope_drained_routes(drained);
3320 let response = if is_apply {
3321 ModuleControlResponseToModule::ScopeApply {
3322 generation,
3323 results: applied.results,
3324 end_results: applied.end_results,
3325 ended: applied.ended,
3326 }
3327 } else {
3328 ModuleControlResponseToModule::ScopeSync {
3329 generation,
3330 results: applied.results,
3331 ended: applied.ended,
3332 }
3333 };
3334 Ok(vec![control_response_body_frame(
3335 &frame,
3336 &response,
3337 "scope change response",
3338 )?])
3339 }
3340 Err(refusal) => {
3341 info!(
3342 owner = %owner,
3343 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3344 generation,
3345 code = refusal.code,
3346 "scope change refused"
3347 );
3348 Ok(vec![control_error_frame(
3349 &frame,
3350 refusal.code,
3351 refusal.message,
3352 )?])
3353 }
3354 }
3355 }
3356
3357 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3363 if drained.is_empty() {
3364 return;
3365 }
3366 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3367 BTreeMap::new();
3368 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3369 for route in drained {
3370 warn!(
3371 module_id = %route.module_id,
3372 reason = ?route.reason,
3373 client_connection_id = route.client.connection_id.get(),
3374 route_channel = route.client.channel,
3375 "closing route because its scope changed"
3376 );
3377 pushes
3378 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3379 .or_insert_with(|| (route.reason, Vec::new()))
3380 .1
3381 .push(EndpointRoute {
3382 goodbye_target: route.client.clone(),
3383 principal: Principal::Unverified,
3384 bound_at: Instant::now(),
3385 draining: false,
3386 drain_reason: None,
3387 });
3388 goodbyes.push(route.module);
3389 goodbyes.push(route.client);
3390 }
3391 for ((module_id, _), (reason, routes)) in pushes {
3392 send_route_control_pushes(
3393 &self.forwarding,
3394 routes,
3395 ClientControlPush::RouteClosed {
3396 module_id,
3397 channels: Vec::new(),
3398 reason,
3399 drained: false,
3400 abandoned: 0,
3401 excluded_subscriptions: 0,
3402 terminal: Some(false),
3403 },
3404 );
3405 }
3406 self.emit_route_goodbyes(goodbyes);
3407 }
3408
3409 fn handle_scope_describe(
3412 &self,
3413 connection_id: ConnectionId,
3414 frame: Frame,
3415 owner: Principal,
3416 scope_ref: String,
3417 ) -> Result<Vec<Frame>, RouterError> {
3418 let registered = self
3419 .registry
3420 .get_module_by_connection(connection_id)
3421 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3422 let Some(registered) = registered else {
3423 return Ok(vec![control_error_frame(
3424 &frame,
3425 "not_registered",
3426 "scope.describe requires an active module registration owned by this connection",
3427 )?]);
3428 };
3429 let mut description = self
3430 .scopes
3431 .read()
3432 .unwrap_or_else(|poisoned| poisoned.into_inner())
3433 .describe(&owner, &scope_ref);
3434 if !registered
3437 .manifest
3438 .capabilities
3439 .as_ref()
3440 .is_some_and(|caps| {
3441 caps.provides
3442 .iter()
3443 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3444 })
3445 {
3446 if let Some(stamp) = description.stamp.as_mut() {
3447 stamp.attributes.on_behalf_of_agent = None;
3448 }
3449 }
3450 if !registered
3451 .manifest
3452 .capabilities
3453 .as_ref()
3454 .is_some_and(|caps| {
3455 caps.provides
3456 .iter()
3457 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
3458 })
3459 {
3460 if let Some(stamp) = description.stamp.as_mut() {
3461 stamp.attributes.flow_version = None;
3462 stamp.attributes.flow_code_hash = None;
3463 stamp.attributes.flow_install_id = None;
3464 }
3465 }
3466 let owner_configured = match &owner {
3467 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3472 _ => false,
3473 };
3474 let response = ModuleControlResponseToModule::ScopeDescribe {
3475 status: description.status,
3476 scope_epoch: description.scope_epoch,
3477 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3478 owner_synced: description.owner_synced,
3479 owner_configured,
3480 scope: description.stamp,
3481 };
3482 Ok(vec![control_response_body_frame(
3483 &frame,
3484 &response,
3485 "ModuleControlResponseToModule::ScopeDescribe",
3486 )?])
3487 }
3488
3489 fn handle_catalog_update(
3490 &self,
3491 connection_id: ConnectionId,
3492 frame: Frame,
3493 provides: Vec<ProviderRole>,
3494 capabilities: Option<CapabilityDeclarations>,
3495 ready: Option<bool>,
3496 ) -> Result<Vec<Frame>, RouterError> {
3497 self.refresh_capability_requirements();
3498 let Some(registration) = self
3499 .registry
3500 .get_module_by_connection(connection_id)
3501 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3502 else {
3503 return Ok(vec![control_error_frame(
3504 &frame,
3505 "not_registered",
3506 "catalog.update requires an active module registration owned by this connection",
3507 )?]);
3508 };
3509
3510 if let Some(message) =
3511 catalog_update_frozen_field_message(®istration.manifest, &provides)
3512 {
3513 return Ok(vec![control_error_frame(
3514 &frame,
3515 "catalog_update_frozen_field",
3516 message,
3517 )?]);
3518 }
3519
3520 let mut candidate = registration.manifest.clone();
3521 candidate.provides = provides.clone();
3522 candidate.capabilities = capabilities
3523 .clone()
3524 .or_else(|| registration.manifest.capabilities.clone());
3525 if let Some(rich_ui) = &candidate.rich_ui {
3526 if let Err(err) = rich_ui.validate() {
3527 return Ok(vec![control_error_frame(
3528 &frame,
3529 "invalid_manifest",
3530 err.to_string(),
3531 )?]);
3532 }
3533 }
3534 if let Err(err) = candidate.validate_capability_grammar() {
3535 return Ok(vec![control_error_frame(
3536 &frame,
3537 "invalid_capability_grammar",
3538 err.to_string(),
3539 )?]);
3540 }
3541
3542 let mut conflicts = self
3545 .capability_evaluator
3546 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3547 if let Some(conflict) = conflicts.first() {
3548 let message = format!(
3549 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3550 conflict.capability, conflict.claimants[0], candidate.module_id
3551 );
3552 for conflict in &mut conflicts {
3553 conflict.source = DuplicateClaimSource::CatalogUpdate;
3554 }
3555 log_duplicate_claim_events(conflicts);
3556 return Ok(vec![control_error_frame(
3557 &frame,
3558 "reserved_capability",
3559 message,
3560 )?]);
3561 }
3562
3563 let updated = self
3564 .registry
3565 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3566 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3567 if updated.is_none() {
3568 return Ok(vec![control_error_frame(
3569 &frame,
3570 "not_registered",
3571 "catalog.update requires an active module registration owned by this connection",
3572 )?]);
3573 }
3574 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3575 log_duplicate_claim_events(
3576 self.capability_evaluator
3577 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3578 );
3579 }
3580 if capability_census_trigger(
3581 registration.manifest.capabilities.as_ref(),
3582 updated
3583 .as_ref()
3584 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3585 ) {
3586 self.enforce_capability_denies();
3587 }
3588 self.refresh_capability_requirements();
3589
3590 let response = ModuleControlResponseToModule::CatalogUpdate {};
3591 control_response_body_frame(
3592 &frame,
3593 &response,
3594 "ModuleControlResponseToModule::CatalogUpdate",
3595 )
3596 .map(|frame| vec![frame])
3597 }
3598
3599 pub(crate) fn with_activity_ledger_config(
3600 mut self,
3601 run_dir: &Path,
3602 incarnation: String,
3603 config: crate::activity::ActivityConfig,
3604 ) -> Self {
3605 self.activity = Some(crate::activity::ActivityLedger::open_config(
3606 run_dir.join("activity.jsonl"),
3607 incarnation,
3608 config,
3609 ));
3610 self.supervisor.set_activity(self.activity.clone());
3611 self
3612 }
3613
3614 pub(crate) async fn configure_activity(
3615 &self,
3616 config: crate::activity::ActivityConfig,
3617 ) -> Result<(), &'static str> {
3618 match &self.activity {
3619 Some(ledger) => ledger.configure(config).await,
3620 None => Ok(()),
3621 }
3622 }
3623
3624 fn cancel_activity_subscription(
3625 &self,
3626 ctx: &RouteCtx,
3627 frame: &Frame,
3628 ) -> Result<bool, RouterError> {
3629 let task = self
3630 .activity_streams
3631 .lock()
3632 .unwrap_or_else(|p| p.into_inner())
3633 .remove(&(ctx.connection_id, frame.header.corr));
3634 let Some(task) = task else {
3635 return Ok(false);
3636 };
3637 task.abort();
3638 let end = Frame::build_with_version(
3639 response_version(frame),
3640 FrameType::StreamEnd,
3641 control_flags(),
3642 0,
3643 0,
3644 frame.header.corr,
3645 Vec::new(),
3646 )
3647 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3648 let sink = ctx.egress.clone();
3649 tokio::spawn(async move {
3650 let _ = sink.send(end).await;
3651 });
3652 Ok(true)
3653 }
3654
3655 async fn handle_activity_read(
3656 &self,
3657 ctx: &RouteCtx,
3658 frame: Frame,
3659 ) -> Result<Vec<Frame>, RouterError> {
3660 use subc_control::{
3661 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3662 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3663 };
3664 #[derive(Deserialize)]
3665 struct Identity {
3666 #[serde(default)]
3667 consumer_identity: Option<ConsumerIdentity>,
3668 }
3669 let principal = match self
3673 .forwarding
3674 .module_id_for_connection(ctx.connection_id)
3675 .map_err(RouterError::Forwarding)?
3676 {
3677 Some(module_id) => Ok(Principal::Reserved { module_id }),
3678 None => {
3679 let identity: Identity = match serde_json::from_slice(&frame.body) {
3680 Ok(identity) => identity,
3681 Err(_) => {
3682 return Ok(vec![control_error_frame(
3683 &frame,
3684 ACTIVITY_MALFORMED,
3685 "malformed activity read",
3686 )?])
3687 }
3688 };
3689 self.route_open_principal(&frame, identity.consumer_identity)?
3690 }
3691 };
3692 let admitted = matches!(principal, Ok(Principal::Direct))
3693 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3694 if !admitted {
3695 return Ok(vec![control_error_frame(
3696 &frame,
3697 ACTIVITY_READ_FORBIDDEN,
3698 "activity reads require an operator or trusted mirror principal",
3699 )?]);
3700 }
3701 let Some(ledger) = &self.activity else {
3702 return Ok(vec![control_error_frame(
3703 &frame,
3704 ACTIVITY_UNAVAILABLE,
3705 "activity ledger unavailable",
3706 )?]);
3707 };
3708 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3709 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3710 {
3711 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3712 Ok(request) => ledger.query(request).await,
3713 Err(_) => Err(ACTIVITY_MALFORMED),
3714 };
3715 return match result {
3716 Ok(reply) => Ok(vec![control_response_body_frame(
3717 &frame,
3718 &reply,
3719 ops::ACTIVITY_QUERY,
3720 )?]),
3721 Err(code) => Ok(vec![control_error_frame(
3722 &frame,
3723 code,
3724 "activity query refused",
3725 )?]),
3726 };
3727 }
3728 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3729 Ok(request) => request,
3730 Err(_) => {
3731 return Ok(vec![control_error_frame(
3732 &frame,
3733 ACTIVITY_MALFORMED,
3734 "malformed activity subscription",
3735 )?])
3736 }
3737 };
3738 let mut subscription = match ledger.subscribe(request.since).await {
3739 Ok(subscription) => subscription,
3740 Err(code) => {
3741 return Ok(vec![control_error_frame(
3742 &frame,
3743 code,
3744 "activity subscription refused",
3745 )?])
3746 }
3747 };
3748 let key = (ctx.connection_id, frame.header.corr);
3749 let sink = ctx.egress.clone();
3750 let mut streams = self
3751 .activity_streams
3752 .lock()
3753 .unwrap_or_else(|p| p.into_inner());
3754 streams.retain(|_, task| !task.is_finished());
3755 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3756 return Ok(vec![control_error_frame(
3757 &frame,
3758 "duplicate_subscription",
3759 "activity subscription correlation id already in use",
3760 )?]);
3761 }
3762 let task = tokio::spawn(async move {
3763 loop {
3764 let entry = if let Some(entry) = subscription.replay.pop_front() {
3765 Some(entry)
3766 } else {
3767 subscription.receiver.recv().await
3768 };
3769 let Some(entry) = entry else {
3770 break;
3771 };
3772 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3773 return;
3774 };
3775 let Ok(next) = Frame::build_with_version(
3776 response_version(&frame),
3777 FrameType::StreamData,
3778 control_flags(),
3779 0,
3780 0,
3781 frame.header.corr,
3782 body,
3783 ) else {
3784 return;
3785 };
3786 if sink.send(next).await.is_err() {
3787 return;
3788 }
3789 }
3790 if let Some(code) = subscription.terminal_code() {
3793 if let Ok(end) =
3794 control_error_frame(&frame, code, "activity subscriber fell behind")
3795 {
3796 let _ = sink.send(end).await;
3797 }
3798 }
3799 });
3800 streams.insert(key, task.abort_handle());
3801 Ok(Vec::new())
3802 }
3803
3804 async fn handle_activity_record(
3805 &self,
3806 ctx: &RouteCtx,
3807 frame: Frame,
3808 ) -> Result<Vec<Frame>, RouterError> {
3809 use subc_control::{
3810 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3811 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3812 };
3813 let result = async {
3814 let module = self
3815 .forwarding
3816 .module_id_for_connection(ctx.connection_id)
3817 .ok()
3818 .flatten()
3819 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3820 let request: ActivityRecordRequest =
3821 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3822 crate::activity::validate(&module, &request)?;
3823 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3824 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3825 let at_ms = std::time::SystemTime::now()
3826 .duration_since(std::time::UNIX_EPOCH)
3827 .unwrap_or_default()
3828 .as_millis() as u64;
3829 ledger.record(module, request, actor, at_ms).await
3830 }
3831 .await;
3832 match result {
3833 Ok(entry) => Ok(vec![control_response_body_frame(
3834 &frame,
3835 &ActivityRecordResponse { seq: entry.seq },
3836 "activity.record",
3837 )?]),
3838 Err(code) => Ok(vec![control_error_frame(
3839 &frame,
3840 code,
3841 "activity recording refused",
3842 )?]),
3843 }
3844 }
3845
3846 fn activity_actor(
3847 &self,
3848 connection: ConnectionId,
3849 module: &str,
3850 on: Option<&subc_control::ActivityOn>,
3851 ) -> Result<subc_control::ActivityActor, &'static str> {
3852 use crate::forwarding::{DataRoute, DataRouteState};
3853 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3854 let principal = |id: &str| Principal::Reserved {
3855 module_id: id.to_owned(),
3856 };
3857 let mut actor = ActivityActor {
3858 principal: principal(module),
3859 scope_owner: None,
3860 agent_id: None,
3861 run_id: None,
3862 flow_id: None,
3863 project_root: None,
3864 workdir: None,
3865 project_id: None,
3866 };
3867 let scope = match on {
3868 None => return Ok(actor),
3869 Some(ActivityOn::Scope(scope)) => {
3870 let selector = subc_protocol::scope::ScopeSelector::new(
3871 principal(module),
3872 scope.scope_ref.clone(),
3873 Some(scope.epoch),
3874 );
3875 self.scopes
3876 .read()
3877 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3878 .attribute(&principal(module), module, &selector, true)
3879 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3880 }
3881 Some(ActivityOn::Route(route)) => {
3882 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3883 .forwarding
3884 .lookup_data_route(connection, route.channel, route.epoch)
3885 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3886 else {
3887 return Err(ACTIVITY_ACTOR_UNKNOWN);
3888 };
3889 if binding.module_id != module {
3890 return Err(ACTIVITY_ACTOR_UNKNOWN);
3891 }
3892 actor.principal = binding.principal.clone();
3893 actor.project_root = binding
3894 .project_root
3895 .as_ref()
3896 .map(|root| root.as_path().to_path_buf());
3897 let Some(bound) = &binding.scope else {
3898 return Ok(actor);
3899 };
3900 let owner = principal(&bound.owner);
3901 let selector = subc_protocol::scope::ScopeSelector::new(
3902 owner.clone(),
3903 bound.scope_ref.clone(),
3904 Some(bound.tag.scope_epoch),
3905 );
3906 let admitted = self
3907 .scopes
3908 .read()
3909 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3910 .attribute(&owner, module, &selector, true)
3911 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3912 if admitted.tag != bound.tag {
3913 return Err(ACTIVITY_ACTOR_UNKNOWN);
3914 }
3915 admitted
3916 }
3917 };
3918 actor.scope_owner = Some(scope.stamp.owner);
3919 actor.agent_id = scope.stamp.attributes.agent_id;
3920 actor.run_id = scope.stamp.attributes.run_id;
3921 actor.flow_id = scope.stamp.attributes.flow_id;
3922 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
3923 actor.project_id = scope.stamp.project_id;
3924 Ok(actor)
3925 }
3926
3927 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3928 self.refresh_capability_requirements();
3929 let mut counters = self.counters.snapshot();
3935 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3936 obj.insert(
3937 "activity_unparseable_lines".into(),
3938 ledger.read_errors().into(),
3939 );
3940 obj.insert(
3941 "activity_lifecycle_append_failures".into(),
3942 ledger.lifecycle_failures().into(),
3943 );
3944 obj.insert(
3945 "activity_started_digest_skips".into(),
3946 ledger.lifecycle_digest_skips().into(),
3947 );
3948 }
3949 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3950 self.forwarding.client_route_concentration(),
3951 counters.as_object_mut(),
3952 ) {
3953 obj.insert(
3954 "client_connections_with_routes".into(),
3955 connections_with_routes.into(),
3956 );
3957 obj.insert("max_routes_on_one_connection".into(), max.into());
3958 }
3959 if let (Some(open_breakers), Some(obj)) = (
3965 self.route_bind_breakers.open_snapshot(),
3966 counters.as_object_mut(),
3967 ) {
3968 obj.insert("route_bind_breakers_open".into(), open_breakers);
3969 }
3970 let response = ClientControlResponse::ServerDescribe {
3971 protocol_ver: PROTOCOL_VERSION,
3972 subc_ops: subc_ops(),
3973 capabilities: self.describe_capabilities(),
3974 connected_clients: self.connected_clients.count(),
3975 counters: Some(counters),
3976 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3977 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3978 capability_requirements: self.capability_requirement_statuses(),
3979 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3980 config_path: self
3981 .rescan
3982 .as_ref()
3983 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
3984 };
3985 Ok(vec![control_response_body_frame(
3986 &frame,
3987 &response,
3988 "ClientControlResponse::ServerDescribe",
3989 )?])
3990 }
3991
3992 fn handle_catalog_list(
3993 &self,
3994 frame: Frame,
3995 module_id: Option<String>,
3996 ) -> Result<Vec<Frame>, RouterError> {
3997 let (generation, modules) = self
3998 .registry
3999 .list_modules_with_nonce_verification()
4000 .map_err(|err| {
4001 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4002 })?;
4003 let entries = modules
4004 .into_iter()
4005 .filter(|(registration, _)| {
4006 module_id
4007 .as_deref()
4008 .map(|wanted| registration.manifest.module_id == wanted)
4009 .unwrap_or(true)
4010 })
4011 .map(|(registration, nonce_verified)| {
4012 let not_ready = self.not_ready_reason(®istration);
4013 let roles = registration.manifest.provides;
4014 CatalogEntry::new(
4015 registration.manifest.module_id,
4016 roles,
4017 registration.control_ops,
4018 )
4019 .with_ready(not_ready.is_none())
4020 .with_not_ready(not_ready)
4021 .with_module_version(Some(registration.manifest.module_version))
4022 .with_capabilities(registration.manifest.capabilities)
4023 .with_self_signals(registration.manifest.self_signals)
4024 .with_events(registration.manifest.events)
4025 .with_operation_delivery(registration.manifest.operation_delivery)
4026 .with_operation_payload(registration.manifest.operation_payload)
4027 .with_rich_ui(registration.manifest.rich_ui)
4028 .with_role_declarations(registration.manifest.role_declarations)
4029 .with_nonce_verified(nonce_verified)
4030 })
4031 .collect();
4032 let response = ClientControlResponse::CatalogList {
4033 generation,
4034 modules: entries,
4035 subc_ops: subc_ops(),
4036 };
4037 Ok(vec![control_response_body_frame(
4038 &frame,
4039 &response,
4040 "ClientControlResponse::CatalogList",
4041 )?])
4042 }
4043
4044 fn attest_route_open_consumer(
4049 &self,
4050 consumer_identity: Option<&ConsumerIdentity>,
4051 ) -> Option<Principal> {
4052 let Some(consumer_identity) = consumer_identity else {
4053 return Some(Principal::Direct);
4054 };
4055 self.supervisor
4056 .spawned_consumer_authorized(
4057 &consumer_identity.module_id,
4058 &consumer_identity.launch_nonce,
4059 )
4060 .then(|| Principal::Reserved {
4061 module_id: consumer_identity.module_id.clone(),
4062 })
4063 }
4064
4065 fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4069 let Ok(ClientControlRequest::RouteOpen {
4070 identity,
4071 consumer_identity,
4072 scope,
4073 ..
4074 }) = parse_client_control_request(&frame.body)
4075 else {
4076 return RouteOpenCaller::unknown();
4077 };
4078 let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4079 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4080 }
4081
4082 fn route_open_principal(
4083 &self,
4084 frame: &Frame,
4085 consumer_identity: Option<ConsumerIdentity>,
4086 ) -> Result<Result<Principal, Frame>, RouterError> {
4087 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4088 self.route_open_principal_attested(frame, consumer_identity, attested)
4089 }
4090
4091 fn route_open_principal_attested(
4095 &self,
4096 frame: &Frame,
4097 consumer_identity: Option<ConsumerIdentity>,
4098 attested: Option<Principal>,
4099 ) -> Result<Result<Principal, Frame>, RouterError> {
4100 if let Some(principal) = attested {
4101 return Ok(Ok(principal));
4102 }
4103 let Some(consumer_identity) = consumer_identity else {
4104 return Ok(Ok(Principal::Direct));
4105 };
4106
4107 Ok(Err(control_error_frame(
4108 frame,
4109 "bad_consumer_identity",
4110 format!(
4111 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4112 consumer_identity.module_id
4113 ),
4114 )?))
4115 }
4116
4117 #[allow(clippy::too_many_arguments)]
4124 fn route_open_refusal_frame(
4125 &self,
4126 ctx: &RouteCtx,
4127 caller: &RouteOpenCaller,
4128 frame: &Frame,
4129 module_id: &str,
4130 reason: &'static str,
4131 code: &'static str,
4132 message: impl Into<String>,
4133 ) -> Result<Frame, RouterError> {
4134 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4135 control_error_frame(frame, code, message.into())
4136 }
4137
4138 #[allow(clippy::too_many_arguments)]
4156 fn route_open_breaker_refusal_frame(
4157 &self,
4158 ctx: &RouteCtx,
4159 caller: &RouteOpenCaller,
4160 frame: &Frame,
4161 module_id: &str,
4162 consecutive_timeouts: u32,
4163 retry_in: Duration,
4164 probe_in_flight: bool,
4165 ) -> Result<Frame, RouterError> {
4166 self.counters
4167 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4168 debug!(
4169 target: "control",
4170 code = "module_timeout",
4171 module_id = ?module_id,
4172 connection_id = ctx.connection_id.get(),
4173 principal = %caller.principal,
4174 scope = %caller.scope,
4175 harness = %caller.harness,
4176 consecutive_timeouts,
4177 retry_in_ms = retry_in.as_millis() as u64,
4178 probe_in_flight,
4179 "route.open refused by open bind-relay breaker"
4180 );
4181 let detail = if probe_in_flight {
4188 "one test route is already being tried; retry once it settles".to_string()
4189 } else {
4190 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4191 };
4192 control_error_frame(
4193 frame,
4194 "module_timeout",
4195 format!(
4196 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4197 timed out in a row); {detail}; its established routes are unaffected"
4198 ),
4199 )
4200 }
4201
4202 fn observe_route_open_refusal(
4215 &self,
4216 ctx: &RouteCtx,
4217 caller: &RouteOpenCaller,
4218 module_id: &str,
4219 reason: &'static str,
4220 code: &'static str,
4221 ) {
4222 self.counters.increment_route_open_refused(code);
4223 info!(
4224 target: "control",
4225 code,
4226 reason,
4227 module_id = ?module_id,
4228 connection_id = ctx.connection_id.get(),
4229 principal = %caller.principal,
4230 scope = %caller.scope,
4231 harness = %caller.harness,
4232 "route.open refused"
4233 );
4234 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4235 self.route_outages.record_not_serving(module_id, reason);
4236 }
4237 }
4238
4239 fn observe_route_open_accept(
4284 &self,
4285 ctx: &RouteCtx,
4286 module_id: &str,
4287 principal: &str,
4288 scope: &str,
4289 ) {
4290 self.route_outages.record_accepted(module_id);
4291 self.counters.increment_route_open_accepted(principal);
4292 info!(
4293 target: "control",
4294 principal,
4295 module_id,
4296 connection_id = ctx.connection_id.get(),
4297 scope = %scope,
4298 "route.open accepted"
4299 );
4300 }
4301
4302 fn supervised_absent_route_open_refusal_frame(
4303 &self,
4304 ctx: &RouteCtx,
4305 caller: &RouteOpenCaller,
4306 frame: &Frame,
4307 module_id: &str,
4308 code: &'static str,
4309 status: &crate::supervise::ModuleStatus,
4310 ) -> Result<Frame, RouterError> {
4311 self.counters.increment_route_open_refused(code);
4312 info!(
4313 target: "control",
4314 code,
4315 reason = "supervised_not_registered",
4316 module_id = ?module_id,
4317 connection_id = ctx.connection_id.get(),
4318 principal = %caller.principal,
4319 scope = %caller.scope,
4320 harness = %caller.harness,
4321 state = %status.state,
4322 enabled = status.enabled,
4323 live = status.live,
4324 "route.open refused"
4325 );
4326 self.route_outages
4330 .record_not_serving(module_id, "supervised_not_registered");
4331 let message = if let Some(pid) = status.waiting_for_previous_pid {
4332 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4333 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4334 })
4335 } else {
4336 format!(
4337 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4338 status.state, status.enabled, status.live
4339 )
4340 };
4341 control_error_frame(frame, code, message)
4342 }
4343
4344 async fn handle_route_open(
4345 &self,
4346 ctx: &RouteCtx,
4347 frame: Frame,
4348 request: RouteOpenRequest,
4349 timings: &mut RouteOpenTimings,
4350 ) -> Result<Vec<Frame>, RouterError> {
4351 let RouteOpenRequest {
4352 target,
4353 mut identity,
4354 consumer_identity,
4355 consumer_capabilities,
4356 role_versions,
4357 admission_facts,
4358 scope,
4359 } = request;
4360 let target_module_id = target_module_id(&target).to_string();
4361 timings.module_id.clone_from(&target_module_id);
4362 if self
4363 .registry
4364 .get_module_by_connection(ctx.connection_id)
4365 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4366 .is_some()
4367 {
4368 return Ok(vec![control_error_frame(
4369 &frame,
4370 "invalid_request",
4371 "module connections cannot open client routes",
4372 )?]);
4373 }
4374 debug!(
4375 connection_id = ctx.connection_id.get(),
4376 corr = frame.header.corr,
4377 module_id = %target_module_id,
4378 "handling route.open"
4379 );
4380
4381 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4386 let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4387
4388 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4394 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4395 self.observe_route_open_refusal(
4396 ctx,
4397 &caller,
4398 &target_module_id,
4399 "invalid_role_versions",
4400 error_codes::INVALID_REQUEST,
4401 );
4402 return Ok(vec![control_error_body_frame(
4403 &frame,
4404 ErrorBody {
4405 code: error_codes::INVALID_REQUEST.to_string(),
4406 message: error.to_string(),
4407 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4408 },
4409 )?]);
4410 }
4411
4412 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4413 if status.waiting_for_previous_pid.is_some() {
4414 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4415 ctx,
4416 &caller,
4417 &frame,
4418 &target_module_id,
4419 "module_warming",
4420 &status,
4421 )?]);
4422 }
4423 }
4424
4425 let Some(registration) = self
4444 .registry
4445 .get_module(&target_module_id)
4446 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4447 else {
4448 if let Some((status, warming)) =
4449 self.supervisor_status(&target_module_id, frame.header.corr)?
4450 {
4451 if status.protocol == ModuleProtocol::None {
4458 return Ok(vec![self.route_open_refusal_frame(
4459 ctx,
4460 &caller,
4461 &frame,
4462 &target_module_id,
4463 "protocol_none",
4464 error_codes::MODULE_NO_PROTOCOL,
4465 format!(
4466 "module_id '{target_module_id}' is declared protocol: none; \
4467 it speaks no subc wire and serves no routes"
4468 ),
4469 )?]);
4470 }
4471 let code = if warming {
4472 "module_warming"
4473 } else {
4474 "target_unavailable"
4475 };
4476 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4477 ctx,
4478 &caller,
4479 &frame,
4480 &target_module_id,
4481 code,
4482 &status,
4483 )?]);
4484 }
4485 if let Some(removed_ago_ms) =
4486 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4487 {
4488 return Ok(vec![self.route_open_refusal_frame(
4489 ctx,
4490 &caller,
4491 &frame,
4492 &target_module_id,
4493 "removed",
4494 error_codes::MODULE_REMOVED,
4495 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4496 )?]);
4497 }
4498 return Ok(vec![self.route_open_refusal_frame(
4499 ctx,
4500 &caller,
4501 &frame,
4502 &target_module_id,
4503 "not_registered",
4504 error_codes::UNKNOWN_MODULE,
4505 format!("module_id '{target_module_id}' is not registered"),
4506 )?]);
4507 };
4508
4509 if !registration.ready {
4513 self.counters
4514 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4515 info!(
4516 target: "control",
4517 code = error_codes::MODULE_WARMING,
4518 module_id = ?target_module_id,
4519 connection_id = ctx.connection_id.get(),
4520 principal = %caller.principal,
4521 scope = %caller.scope,
4522 harness = %caller.harness,
4523 reason = "declared_not_ready",
4524 "route.open refused"
4525 );
4526 self.route_outages
4529 .record_not_serving(&target_module_id, "declared_not_ready");
4530 return Ok(vec![control_error_body_frame(
4531 &frame,
4532 ErrorBody {
4533 code: error_codes::MODULE_WARMING.to_string(),
4534 message: format!(
4535 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4536 ),
4537 detail: Some(serde_json::json!({
4538 "reason": "declared_not_ready"
4539 })),
4540 },
4541 )?]);
4542 }
4543
4544 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4563 self.counters
4564 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4565 info!(
4566 target: "control",
4567 code = error_codes::MODULE_WARMING,
4568 module_id = ?target_module_id,
4569 connection_id = ctx.connection_id.get(),
4570 principal = %caller.principal,
4571 scope = %caller.scope,
4572 harness = %caller.harness,
4573 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4574 capability = %capability,
4575 "route.open refused"
4576 );
4577 return Ok(vec![control_error_body_frame(
4578 &frame,
4579 ErrorBody {
4580 code: error_codes::MODULE_WARMING.to_string(),
4581 message: format!(
4582 "module_id '{target_module_id}' requires capability '{capability}', \
4583 which no registered module provides; retry"
4584 ),
4585 detail: Some(serde_json::json!({
4586 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4587 "capability": capability,
4588 })),
4589 },
4590 )?]);
4591 }
4592
4593 if !target_has_required_role(&target, ®istration.manifest.provides) {
4594 return Ok(vec![self.route_open_refusal_frame(
4595 ctx,
4596 &caller,
4597 &frame,
4598 &target_module_id,
4599 "role_not_provided",
4600 "target_unavailable",
4601 format!("module_id '{target_module_id}' does not provide the requested target"),
4602 )?]);
4603 }
4604
4605 if registration.state != ChannelState::Active {
4606 return Ok(vec![self.route_open_refusal_frame(
4607 ctx,
4608 &caller,
4609 &frame,
4610 &target_module_id,
4611 "registration_not_active",
4612 "target_unavailable",
4613 format!("module_id '{target_module_id}' is not active"),
4614 )?]);
4615 }
4616
4617 if self
4618 .forwarding
4619 .module_is_draining(&target_module_id)
4620 .map_err(RouterError::Forwarding)?
4621 {
4622 return Ok(vec![self.route_open_refusal_frame(
4623 ctx,
4624 &caller,
4625 &frame,
4626 &target_module_id,
4627 "reloading",
4628 "module_reloading",
4629 format!("module_id '{target_module_id}' is reloading"),
4630 )?]);
4631 }
4632
4633 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4634 process_liveness.process_live(&target_module_id) == Some(false)
4635 }) {
4636 if process_liveness.process_replacing(&target_module_id) {
4643 return Ok(vec![self.route_open_refusal_frame(
4644 ctx,
4645 &caller,
4646 &frame,
4647 &target_module_id,
4648 "reloading",
4649 "module_reloading",
4650 format!("module_id '{target_module_id}' is reloading"),
4651 )?]);
4652 }
4653 return Ok(vec![self.route_open_refusal_frame(
4654 ctx,
4655 &caller,
4656 &frame,
4657 &target_module_id,
4658 "supervisor_not_live",
4659 "target_unavailable",
4660 format!("module_id '{target_module_id}' is not live"),
4661 )?]);
4662 }
4663
4664 if !self
4665 .forwarding
4666 .has_live_module_connection(&target_module_id)
4667 .map_err(RouterError::Forwarding)?
4668 {
4669 return Ok(vec![self.route_open_refusal_frame(
4670 ctx,
4671 &caller,
4672 &frame,
4673 &target_module_id,
4674 "no_forwarding_connection",
4675 "target_unavailable",
4676 format!("module_id '{target_module_id}' has no live forwarding connection"),
4677 )?]);
4678 }
4679
4680 if let Some(error) =
4681 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4682 {
4683 self.observe_route_open_refusal(
4684 ctx,
4685 &caller,
4686 &target_module_id,
4687 "op_not_allowed",
4688 "op_not_allowed",
4689 );
4690 return Ok(vec![error]);
4691 }
4692
4693 let principal =
4694 match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
4695 Ok(principal) => principal,
4696 Err(error) => {
4697 self.observe_route_open_refusal(
4698 ctx,
4699 &caller,
4700 &target_module_id,
4701 "bad_consumer_identity",
4702 "bad_consumer_identity",
4703 );
4704 return Ok(vec![error]);
4705 }
4706 };
4707 timings.principal = principal_label(&principal);
4708
4709 if let Principal::Reserved {
4713 module_id: opening_module_id,
4714 } = &principal
4715 {
4716 if let Some(opening_registration) = self
4717 .registry
4718 .get_module(opening_module_id)
4719 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4720 {
4721 if let Some(capability) =
4722 denied_capability(&opening_registration.manifest, ®istration.manifest)
4723 {
4724 warn!(
4725 opening_module_id,
4726 target_module_id,
4727 capability,
4728 "refusing route.open because an attested capability deny edge matches"
4729 );
4730 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4731 &frame,
4732 &target_module_id,
4733 "capability_deny_edge",
4734 "capability_forbidden",
4735 format!(
4736 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4737 ),
4738 )?]);
4739 }
4740 }
4741 }
4742
4743 if admission_facts.is_some() {
4744 let carrier_matches = matches!(
4745 &principal,
4746 Principal::Reserved { module_id }
4747 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4748 );
4749 if !carrier_matches {
4750 return Ok(vec![self.route_open_refusal_frame(
4751 ctx,
4752 &caller,
4753 &frame,
4754 &target_module_id,
4755 "admission_facts_carrier_not_permitted",
4756 "admission_facts_not_permitted",
4757 "admission facts may only be carried by the configured reserved module",
4758 )?]);
4759 }
4760
4761 let target_allowed = self
4762 .admission_facts_targets
4763 .as_ref()
4764 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4765 if !target_allowed {
4766 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4767 &frame,
4768 &target_module_id,
4769 "admission_facts_target_not_listed",
4770 "admission_facts_target_not_allowed",
4771 format!(
4772 "admission facts are not permitted for target module_id '{target_module_id}'"
4773 ),
4774 )?]);
4775 }
4776
4777 }
4781
4782 let (bound_scope, mut scope_stamp) = match &scope {
4788 None => (None, None),
4789 Some(selector) => {
4790 let owner_configured = match &selector.owner {
4791 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4797 _ => false,
4798 };
4799 let mut scopes = self
4800 .scopes
4801 .write()
4802 .unwrap_or_else(|poisoned| poisoned.into_inner());
4803 let admitted = scopes
4804 .admit(&principal, &target_module_id, selector, owner_configured)
4805 .and_then(|admission| {
4806 crate::scopes::check_target_flow_support(
4807 &admission.stamp,
4808 &target_module_id,
4809 registration.manifest.capabilities.as_ref(),
4810 )?;
4811 crate::scopes::check_target_agent_run_support(
4812 &admission.stamp,
4813 &target_module_id,
4814 registration.manifest.capabilities.as_ref(),
4815 )?;
4816 let bound = scopes.reserve_bind(
4817 &admission,
4818 &principal,
4819 ctx.connection_id,
4820 selector.hold_presence,
4821 );
4822 Ok((admission, bound))
4823 });
4824 match admitted {
4825 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4826 Err(refusal) => {
4827 return Ok(vec![self.route_open_refusal_frame(
4828 ctx,
4829 &caller,
4830 &frame,
4831 &target_module_id,
4832 refusal.code,
4833 refusal.code,
4834 refusal.message,
4835 )?]);
4836 }
4837 }
4838 }
4839 };
4840 if let Some(stamp) = &scope_stamp {
4843 caller.scope = route_open_scope_label(stamp);
4844 }
4845
4846 timings.enter(RouteOpenPhase::ProjectRoot);
4868 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4869 Ok(project_root) => project_root,
4870 Err(err) => {
4871 return Ok(vec![control_error_frame(
4872 &frame,
4873 "invalid_project_root",
4874 err.to_string(),
4875 )?])
4876 }
4877 };
4878 timings.enter(RouteOpenPhase::Admission);
4879 identity.project_root = project_root.as_path().to_path_buf();
4880
4881 let _concurrency_guard = match self
4892 .route_bind_concurrency
4893 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4894 {
4895 Ok(guard) => guard,
4896 Err(in_flight) => {
4897 return Ok(vec![self.route_open_target_capacity_refusal(
4898 ctx,
4899 &caller,
4900 &frame,
4901 &target_module_id,
4902 in_flight,
4903 )?]);
4904 }
4905 };
4906
4907 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4910 RouteBindAdmission::Admitted { guard, probe } => {
4911 if probe {
4912 info!(
4913 module_id = %target_module_id,
4914 connection_id = ctx.connection_id.get(),
4915 "route.bind breaker half-open: admitting one probe"
4916 );
4917 }
4918 guard
4919 }
4920 RouteBindAdmission::Refused {
4921 consecutive_timeouts,
4922 retry_in,
4923 probe_in_flight,
4924 } => {
4925 return Ok(vec![self.route_open_breaker_refusal_frame(
4926 ctx,
4927 &caller,
4928 &frame,
4929 &target_module_id,
4930 consecutive_timeouts,
4931 retry_in,
4932 probe_in_flight,
4933 )?]);
4934 }
4935 };
4936
4937 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4942 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4943 timings.enter(RouteOpenPhase::EgressReservation);
4944 let pending = match self
4945 .forwarding
4946 .begin_route_bind_relay_for(
4947 ctx.connection_id,
4948 ctx.egress.clone(),
4949 response_version(&frame),
4950 frame.header.corr,
4951 &target_module_id,
4952 principal.clone(),
4953 bound_scope,
4954 Some(project_root),
4955 relay_deadline,
4956 )
4957 .await
4958 {
4959 Ok(pending) => pending,
4960 Err(err) => {
4961 return Ok(vec![self.route_open_refusal_frame(
4962 ctx,
4963 &caller,
4964 &frame,
4965 &target_module_id,
4966 "relay_reservation_failed",
4967 forwarding_error_code(&err),
4968 err.to_string(),
4969 )?])
4970 }
4971 };
4972 timings.enter(RouteOpenPhase::Admission);
4973 let crate::forwarding::PendingRouteBindRelay {
4974 endpoint,
4975 module_sink,
4976 negotiated_ver,
4977 client_channel,
4978 client_epoch,
4979 module_channel,
4980 module_epoch,
4981 corr: relay_corr,
4982 receiver,
4983 } = pending;
4984 let mut reservation =
4985 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4986
4987 let relay_registration = self
4992 .registry
4993 .get_module_by_connection(endpoint.connection_id)
4994 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4995 let relay_capabilities = relay_registration
4996 .as_ref()
4997 .and_then(|registration| registration.manifest.capabilities.as_ref());
4998 if let Some(selector) = &scope {
4999 if let Err(refusal) = self
5002 .scopes
5003 .read()
5004 .unwrap_or_else(|poisoned| poisoned.into_inner())
5005 .check_bind_proof(&principal, selector)
5006 {
5007 reservation.release_and_disarm();
5008 return Ok(vec![self.route_open_refusal_frame(
5009 ctx,
5010 &caller,
5011 &frame,
5012 &target_module_id,
5013 refusal.code,
5014 refusal.code,
5015 refusal.message,
5016 )?]);
5017 }
5018 }
5019 if let Some(stamp) = scope_stamp
5020 .as_ref()
5021 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5022 {
5023 if let Err(refusal) = crate::scopes::check_target_flow_support(
5024 stamp,
5025 &target_module_id,
5026 relay_capabilities,
5027 )
5028 .and_then(|()| {
5029 crate::scopes::check_target_agent_run_support(
5030 stamp,
5031 &target_module_id,
5032 relay_capabilities,
5033 )
5034 }) {
5035 reservation.release_and_disarm();
5036 return Ok(vec![self.route_open_refusal_frame(
5037 ctx,
5038 &caller,
5039 &frame,
5040 &target_module_id,
5041 refusal.code,
5042 refusal.code,
5043 refusal.message,
5044 )?]);
5045 }
5046 }
5047
5048 if !relay_capabilities.is_some_and(|caps| {
5051 caps.provides
5052 .iter()
5053 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5054 }) {
5055 if let Some(stamp) = scope_stamp.as_mut() {
5056 stamp.workdir = None;
5057 stamp.project_id = None;
5058 }
5059 }
5060 if !relay_capabilities.is_some_and(|caps| {
5061 caps.provides
5062 .iter()
5063 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5064 }) {
5065 if let Some(stamp) = scope_stamp.as_mut() {
5066 stamp.attributes.on_behalf_of_agent = None;
5067 }
5068 }
5069 if !relay_capabilities.is_some_and(|caps| {
5070 caps.provides
5071 .iter()
5072 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5073 }) {
5074 if let Some(stamp) = scope_stamp.as_mut() {
5075 stamp.attributes.flow_version = None;
5076 stamp.attributes.flow_code_hash = None;
5077 stamp.attributes.flow_install_id = None;
5078 }
5079 }
5080 let call_mode = if relay_capabilities
5081 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5082 {
5083 Some(derive_call_mode(scope_stamp.as_ref()))
5084 } else {
5085 if let Some(stamp) = scope_stamp.as_mut() {
5086 stamp.expires_at_ms = None;
5087 }
5088 None
5089 };
5090
5091 debug!(
5092 connection_id = ctx.connection_id.get(),
5093 client_channel,
5094 client_epoch,
5095 module_channel,
5096 module_epoch,
5097 "reserved route handle pair"
5098 );
5099 let route_principal_label = principal_label(&principal);
5102 let relay = ModuleControlRequest::RouteBind {
5103 route_channel: module_channel,
5104 epoch: module_epoch,
5105 target,
5106 identity,
5107 principal: Some(principal),
5108 consumer_capabilities,
5109 role_versions,
5110 admission_facts,
5111 scope: scope_stamp,
5112 call_mode,
5113 };
5114 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
5115 RouterError::backend(
5116 0,
5117 frame.header.corr,
5118 format!("failed to encode route.bind request: {err}"),
5119 )
5120 })?;
5121 let relay_frame = Frame::build_with_version(
5122 negotiated_ver,
5123 FrameType::Request,
5124 control_flags(),
5125 0,
5126 0,
5127 relay_corr,
5128 relay_body,
5129 )
5130 .map_err(RouterError::FrameBuild)?;
5131
5132 timings.enter(RouteOpenPhase::BindSend);
5133 let sent = module_sink.send(relay_frame).await;
5134 timings.enter(RouteOpenPhase::Admission);
5135 if let Err(err) = sent {
5136 reservation.release_and_disarm();
5137 return Ok(vec![self.route_open_refusal_frame(
5138 ctx,
5139 &caller,
5140 &frame,
5141 &target_module_id,
5142 "relay_send_failed",
5143 "target_unavailable",
5144 err.to_string(),
5145 )?]);
5146 }
5147
5148 #[cfg(test)]
5149 if let Some(hold) = &self.route_bind_relay_sent_hold {
5150 hold.wait().await;
5154 hold.wait().await;
5155 }
5156
5157 if self
5162 .forwarding
5163 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5164 .map_err(RouterError::Forwarding)?
5165 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5166 {
5167 self.send_abandoned_route_bind_goodbye(
5168 &module_sink,
5169 negotiated_ver,
5170 module_channel,
5171 module_epoch,
5172 );
5173 }
5174
5175 timings.enter(RouteOpenPhase::BindAckWait);
5176 let outcome = timeout_at(relay_deadline, receiver).await;
5177 timings.enter(RouteOpenPhase::Admission);
5178 match outcome {
5179 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5180 reservation.disarm();
5181 if breaker.record_accepted() {
5182 info!(
5183 module_id = %target_module_id,
5184 "route.bind breaker closed: the probe was accepted"
5185 );
5186 }
5187 self.observe_route_open_accept(
5188 ctx,
5189 &target_module_id,
5190 &route_principal_label,
5191 &caller.scope,
5192 );
5193 Ok(Vec::new())
5194 }
5195 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5196 reservation.release_and_disarm();
5197 breaker.record_inconclusive();
5201 let scope_code = match body.code.as_str() {
5205 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
5206 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
5207 _ => None,
5208 };
5209 if let Some(code) = scope_code {
5210 self.observe_route_open_refusal(
5211 ctx,
5212 &caller,
5213 &target_module_id,
5214 "scope_changed_before_commit",
5215 code,
5216 );
5217 return Ok(vec![control_error_body_frame(&frame, body)?]);
5218 }
5219 self.counters
5220 .increment_route_open_refused("module_rejected");
5221 info!(
5222 target: "control",
5223 code = "module_rejected",
5224 module_code = ?body.code,
5225 module_id = ?target_module_id,
5226 connection_id = ctx.connection_id.get(),
5227 principal = %caller.principal,
5228 scope = %caller.scope,
5229 harness = %caller.harness,
5230 "route.open refused"
5231 );
5232 Ok(vec![control_error_body_frame(&frame, body)?])
5233 }
5234 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5235 reservation.release_and_disarm();
5236 breaker.record_inconclusive();
5237 tracing::warn!(
5244 module_id = %target_module_id,
5245 "route.bind relay abandoned: {message}"
5246 );
5247 Ok(vec![self.route_open_refusal_frame(
5248 ctx,
5249 &caller,
5250 &frame,
5251 &target_module_id,
5252 "relay_abandoned",
5253 "target_unavailable",
5254 message,
5255 )?])
5256 }
5257 Ok(Err(_)) => {
5258 reservation.release_and_disarm();
5259 breaker.record_inconclusive();
5260 Ok(vec![self.route_open_refusal_frame(
5261 ctx,
5262 &caller,
5263 &frame,
5264 &target_module_id,
5265 "relay_waiter_canceled",
5266 "target_unavailable",
5267 "route.bind relay waiter was canceled before the module responded",
5268 )?])
5269 }
5270 Err(_) => {
5271 reservation.release_and_disarm();
5272 if let Some(opened) = breaker.record_timeout(
5276 self.route_bind_breaker_threshold,
5277 self.route_bind_breaker_cooldown,
5278 ) {
5279 warn!(
5280 module_id = %target_module_id,
5281 consecutive_timeouts = opened.consecutive_timeouts,
5282 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5283 reopened_after_probe = opened.reopened_after_probe,
5284 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5285 );
5286 }
5287 tracing::warn!(
5295 module_id = %target_module_id,
5296 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5297 "route.bind relay timed out: module did not ack within budget"
5298 );
5299 Ok(vec![self.route_open_refusal_frame(
5300 ctx,
5301 &caller,
5302 &frame,
5303 &target_module_id,
5304 "relay_timed_out",
5305 "module_timeout",
5306 format!(
5307 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5308 route_bind_relay_timeout
5309 ),
5310 )?])
5311 }
5312 }
5313 }
5314
5315 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5316 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5317 snapshot: self.supervisor.spawn_snapshot(),
5318 };
5319 Ok(vec![control_response_body_frame(
5320 &frame,
5321 &response,
5322 "ClientControlResponse::SupervisorSpawnSnapshot",
5323 )?])
5324 }
5325
5326 fn handle_supervisor_spawn_subscribe(
5327 &self,
5328 ctx: &RouteCtx,
5329 frame: Frame,
5330 since: Option<SpawnCursor>,
5331 ) -> Result<Vec<Frame>, RouterError> {
5332 match self.supervisor.subscribe_spawns(
5333 ctx.connection_id,
5334 frame.header.corr,
5335 response_version(&frame),
5336 since,
5337 ctx.egress.clone(),
5338 ) {
5339 Ok(()) => Ok(Vec::new()),
5340 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5341 Ok(vec![control_error_body_frame(
5342 &frame,
5343 ErrorBody {
5344 code: "spawn_cursor_incarnation_mismatch".to_string(),
5345 message: "spawn cursor belongs to a different daemon incarnation"
5346 .to_string(),
5347 detail: Some(serde_json::json!({
5348 "current_daemon_incarnation": current
5349 })),
5350 },
5351 )?])
5352 }
5353 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5354 &frame,
5355 ErrorBody {
5356 code: "spawn_cursor_too_old".to_string(),
5357 message: "spawn cursor predates the retained event ring".to_string(),
5358 detail: Some(serde_json::json!({
5359 "oldest_retained_cursor": oldest
5360 })),
5361 },
5362 )?]),
5363 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5364 0,
5365 frame.header.corr,
5366 format!("failed to open supervisor spawn subscription: {error}"),
5367 )),
5368 }
5369 }
5370
5371 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5372 let generation = self
5373 .registry
5374 .generation()
5375 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5376 let mut modules = Vec::new();
5377 for module in self.supervisor.list() {
5378 let status = module.status_for_control("list").map_err(|err| {
5379 RouterError::backend(
5380 0,
5381 frame.header.corr,
5382 format!("failed to read supervisor status: {err}"),
5383 )
5384 })?;
5385 let (configured, _) = module.configuration().map_err(|err| {
5386 RouterError::backend(
5387 0,
5388 frame.header.corr,
5389 format!("failed to read module configuration: {err}"),
5390 )
5391 })?;
5392 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
5393 RouterError::backend(
5394 0,
5395 frame.header.corr,
5396 format!("failed to read retry time: {err}"),
5397 )
5398 })?;
5399 let image = module.running_image_agreement().await;
5401 let resources = Some(module.child_resource_usage());
5404 let mut verdict =
5405 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5406 verdict.spec = Some(launch_spec_agreement(
5407 &configured,
5408 status.process_alive,
5409 status.launch_spec_digest.as_ref(),
5410 ));
5411 let pending_reload = Some(verdict);
5412 let configured_enabled = self
5413 .capability_evaluator
5414 .configured_enabled(&status.module_id);
5415 modules.push(
5418 SupervisorEntry::new(
5419 status.module_id,
5420 status.state.to_string(),
5421 status.enabled,
5422 status.live,
5423 status.health.status,
5424 )
5425 .with_configured_enabled(configured_enabled)
5426 .with_next_retry_at_ms(next_retry_at_ms)
5427 .with_launch_nonce_env(Some(!cfg!(unix)))
5428 .with_launch_nonce_source(status.launch_nonce_source)
5429 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5430 .with_protocol(status.protocol)
5431 .with_pending_reload(pending_reload)
5432 .with_last_probe_ms(status.health.last_probe_ms)
5433 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5434 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5435 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5436 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5437 .with_restart_count(Some(status.restart_count))
5438 .with_max_restarts(Some(status.max_restarts))
5439 .with_lifetime_restarts(Some(status.lifetime_restarts))
5440 .with_spawn_generation(Some(status.spawn_generation))
5441 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5442 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5443 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5444 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5445 .with_resources(resources),
5446 );
5447 }
5448 let response = ClientControlResponse::SupervisorList {
5449 generation,
5450 modules,
5451 };
5452 Ok(vec![control_response_body_frame(
5453 &frame,
5454 &response,
5455 "ClientControlResponse::SupervisorList",
5456 )?])
5457 }
5458
5459 fn handle_supervisor_stderr_tail(
5460 &self,
5461 frame: Frame,
5462 module_id: String,
5463 max_lines: Option<u32>,
5464 max_bytes: Option<u32>,
5465 ) -> Result<Vec<Frame>, RouterError> {
5466 let Some(module) = self.supervisor.get(&module_id) else {
5467 return Ok(vec![control_error_frame(
5468 &frame,
5469 "unknown_module",
5470 format!("module_id '{module_id}' is not supervised"),
5471 )?]);
5472 };
5473
5474 let snapshot = module.stderr_tail(
5475 max_lines.map(|value| value as usize),
5476 max_bytes.map(|value| value as usize),
5477 );
5478
5479 let response = ClientControlResponse::SupervisorStderrTail {
5480 module_id,
5481 tail: StderrTail {
5482 capture: match snapshot.capture {
5483 CaptureState::Captured => StderrCaptureState::Captured,
5484 CaptureState::Incomplete { reason } => {
5485 StderrCaptureState::Incomplete { reason }
5486 }
5487 CaptureState::NotCaptured { reason } => {
5488 StderrCaptureState::NotCaptured { reason }
5489 }
5490 },
5491 entries: snapshot
5492 .entries
5493 .into_iter()
5494 .map(|entry| match entry {
5495 TailEntry::Line {
5496 text,
5497 truncated,
5498 at_ms,
5499 } => StderrTailEntry::Line {
5500 text,
5501 truncated,
5502 at_ms,
5503 },
5504 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5505 })
5506 .collect(),
5507 dropped_lines: snapshot.dropped_lines,
5508 },
5509 };
5510 Ok(vec![control_response_body_frame(
5511 &frame,
5512 &response,
5513 "ClientControlResponse::SupervisorStderrTail",
5514 )?])
5515 }
5516
5517 async fn handle_supervisor_terminals(
5518 &self,
5519 frame: Frame,
5520 module_id: String,
5521 ) -> Result<Vec<Frame>, RouterError> {
5522 let Some(module) = self.supervisor.get(&module_id) else {
5523 return Ok(vec![control_error_frame(
5524 &frame,
5525 "unknown_module",
5526 format!("module_id '{module_id}' is not supervised"),
5527 )?]);
5528 };
5529
5530 let terminals = module
5533 .read_durable_terminal_history()
5534 .await
5535 .map_err(|error| {
5536 RouterError::backend(
5537 0,
5538 frame.header.corr,
5539 format!("failed to read terminal history: {error}"),
5540 )
5541 })?;
5542 let response = ClientControlResponse::SupervisorTerminals {
5543 module_id,
5544 terminals,
5545 };
5546 Ok(vec![control_response_body_frame(
5547 &frame,
5548 &response,
5549 "ClientControlResponse::SupervisorTerminals",
5550 )?])
5551 }
5552
5553 fn handle_supervisor_routes(
5554 &self,
5555 frame: Frame,
5556 module_id: Option<String>,
5557 ) -> Result<Vec<Frame>, RouterError> {
5558 let modules = self
5559 .forwarding
5560 .route_census(module_id.as_deref())
5561 .map_err(RouterError::Forwarding)?
5562 .into_iter()
5563 .map(|(module_id, routes)| SupervisorRouteModule {
5564 module_id,
5565 routes: routes
5566 .into_iter()
5567 .map(|route| SupervisorRoute {
5568 consumer: match route.principal {
5569 Principal::Reserved { module_id } => {
5570 SupervisorRouteConsumer::Reserved { module_id }
5571 }
5572 Principal::Direct | Principal::Unverified => {
5573 SupervisorRouteConsumer::Direct {
5574 connection_id: route.goodbye_target.connection_id.get(),
5575 }
5576 }
5577 },
5578 age_ms: Instant::now()
5579 .saturating_duration_since(route.bound_at)
5580 .as_millis()
5581 .try_into()
5582 .unwrap_or(u64::MAX),
5583 draining: route.draining,
5584 drain_reason: route.drain_reason,
5585 })
5586 .collect(),
5587 })
5588 .collect();
5589 let response = ClientControlResponse::SupervisorRoutes { modules };
5590 Ok(vec![control_response_body_frame(
5591 &frame,
5592 &response,
5593 "ClientControlResponse::SupervisorRoutes",
5594 )?])
5595 }
5596
5597 async fn handle_supervisor_provenance(
5598 &self,
5599 frame: Frame,
5600 module_id: Option<String>,
5601 ) -> Result<Vec<Frame>, RouterError> {
5602 let mut selected = if let Some(module_id) = module_id {
5603 let Some(module) = self.supervisor.get(&module_id) else {
5604 return Ok(vec![control_error_frame(
5605 &frame,
5606 "unknown_module",
5607 format!("module_id '{module_id}' is not supervised"),
5608 )?]);
5609 };
5610 vec![module]
5611 } else {
5612 self.supervisor.list()
5613 };
5614
5615 let mut modules = Vec::with_capacity(selected.len());
5616 for module in selected.drain(..) {
5617 let (status, observed_image) = module
5618 .status_and_running_image_agreement()
5619 .await
5620 .map_err(|err| {
5621 RouterError::backend(
5622 0,
5623 frame.header.corr,
5624 format!("failed to read supervisor status: {err}"),
5625 )
5626 })?;
5627 let module_declared = self
5628 .registry
5629 .get_module(&status.module_id)
5630 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5631 .and_then(|registration| registration.manifest.provenance)
5632 .map(|build| ModuleDeclaredProvenance::Reported { build })
5633 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5634 #[cfg(test)]
5635 let running_image = match &self.provenance_probe_override {
5636 Some(result) => result.clone(),
5637 None => observed_image,
5638 };
5639 #[cfg(not(test))]
5640 let running_image = observed_image;
5641 modules.push(SupervisorModuleProvenance {
5642 module_id: status.module_id,
5643 module_declared,
5644 daemon_observed: SupervisorObservedProcess {
5645 launch_nonce_source: status.launch_nonce_source,
5646 pid: status.pid,
5647 spawned_at_ms: status.spawned_at_ms,
5648 spawned_from: status.spawned_from,
5649 running_image,
5650 },
5651 });
5652 }
5653 let daemon = SupervisorDaemonProvenance {
5654 daemon_build: self.daemon_provenance.build.clone(),
5655 daemon_observed: DaemonObservedProcess {
5656 pid: self.daemon_provenance.pid,
5657 started_at_ms: self
5658 .daemon_provenance
5659 .start_clock
5660 .map(|clock| clock.started_at_ms())
5661 .or(self.daemon_provenance.started_at_ms),
5662 running_image: self
5663 .daemon_provenance
5664 .probe
5665 .observe(
5666 self.daemon_provenance.pid,
5667 self.daemon_provenance.executable_path.as_deref(),
5668 self.daemon_provenance.executable_identity.clone(),
5669 self.daemon_provenance.process_start_time,
5670 )
5671 .await,
5672 },
5673 };
5674 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5675 Ok(vec![control_response_body_frame(
5676 &frame,
5677 &response,
5678 "ClientControlResponse::SupervisorProvenance",
5679 )?])
5680 }
5681
5682 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5683 self.refresh_capability_requirements();
5684 let generation = self
5685 .registry
5686 .generation()
5687 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5688 let modules = self
5689 .supervisor
5690 .list()
5691 .into_iter()
5692 .map(|module| {
5693 let status = module.status_for_control("health").map_err(|err| {
5694 RouterError::backend(
5695 0,
5696 frame.header.corr,
5697 format!("failed to read supervisor health: {err}"),
5698 )
5699 })?;
5700 let module_id = status.module_id;
5701 let capability_detail = self
5702 .capability_evaluator
5703 .required_problem_detail(&module_id);
5704 Ok(SupervisorHealthEntry {
5705 module_id,
5706 status: status.health.status,
5707 detail: append_capability_problem_detail(
5708 status.health.detail,
5709 capability_detail,
5710 ),
5711 metrics: status.health.metrics,
5712 consecutive_failures: status.health.consecutive_failures,
5713 late_answer_count: status.health.late_answer_count,
5714 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5715 last_action: status.health.last_action,
5716 last_action_ms: status.health.last_action_ms,
5717 last_probe_ms: status.health.last_probe_ms,
5718 })
5719 })
5720 .collect::<Result<Vec<_>, RouterError>>()?;
5721 let response = ClientControlResponse::SupervisorHealth {
5722 generation,
5723 modules,
5724 };
5725 Ok(vec![control_response_body_frame(
5726 &frame,
5727 &response,
5728 "ClientControlResponse::SupervisorHealth",
5729 )?])
5730 }
5731
5732 async fn handle_supervisor_restart(
5733 &self,
5734 frame: Frame,
5735 module_id: String,
5736 drain_timeout_ms: Option<u64>,
5737 requester: Principal,
5738 ) -> Result<Vec<Frame>, RouterError> {
5739 let operation_lock = self.supervisor.operation_lock();
5740 let _operation_guard = operation_lock.lock().await;
5741 let Some(module) = self.supervisor.get(&module_id) else {
5742 return Ok(vec![control_error_frame(
5743 &frame,
5744 "unknown_module",
5745 format!("module_id '{module_id}' is not supervised"),
5746 )?]);
5747 };
5748
5749 self.route_outages.mark_operator_action(&module_id);
5750 if let Err(err) = module
5751 .restart_requested(drain_timeout_ms, Some(requester))
5752 .await
5753 {
5754 self.route_outages
5755 .operator_action_ended_unrefused(&module_id);
5756 let (code, message) = match err {
5757 crate::supervise::SuperviseError::Disabled { .. } => {
5758 ("module_disabled", err.to_string())
5759 }
5760 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5761 ("swap_in_progress", err.to_string())
5762 }
5763 _ => (
5764 "target_unavailable",
5765 format!("failed to restart module_id '{module_id}': {err}"),
5766 ),
5767 };
5768 return Ok(vec![control_error_frame(&frame, code, message)?]);
5769 }
5770
5771 let response = ClientControlResponse::SupervisorAck {
5772 module_id,
5773 applied: true,
5774 };
5775 Ok(vec![control_response_body_frame(
5776 &frame,
5777 &response,
5778 "ClientControlResponse::SupervisorAck",
5779 )?])
5780 }
5781
5782 async fn handle_supervisor_swap(
5786 &self,
5787 frame: Frame,
5788 module_id: String,
5789 ready_timeout_ms: Option<u64>,
5790 requester: Principal,
5791 ) -> Result<Vec<Frame>, RouterError> {
5792 let module = {
5799 let operation_lock = self.supervisor.operation_lock();
5800 let _operation_guard = operation_lock.lock().await;
5801 self.supervisor.get(&module_id)
5802 };
5803 let Some(module) = module else {
5804 return Ok(vec![control_error_frame(
5805 &frame,
5806 "unknown_module",
5807 format!("module_id '{module_id}' is not supervised"),
5808 )?]);
5809 };
5810
5811 self.route_outages.mark_operator_action(&module_id);
5812 if let Err(err) = module
5813 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5814 .await
5815 {
5816 self.route_outages
5817 .operator_action_ended_unrefused(&module_id);
5818 use crate::supervise::SuperviseError;
5819 let message = err.to_string();
5820 let error = match err {
5821 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5822 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5823 code: "swap_refused".to_string(),
5824 message,
5825 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5826 },
5827 SuperviseError::SwapFailed {
5828 arm,
5829 candidate_exit,
5830 ..
5831 } => ErrorBody {
5832 code: "swap_failed".to_string(),
5833 message,
5834 detail: Some(serde_json::json!({
5835 "arm": arm.as_str(),
5836 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5837 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5838 })),
5839 },
5840 _ => ErrorBody::new(
5841 "target_unavailable",
5842 format!("failed to swap module_id '{module_id}': {message}"),
5843 ),
5844 };
5845 return Ok(vec![control_error_body_frame(&frame, error)?]);
5846 }
5847 self.route_outages
5851 .operator_action_ended_unrefused(&module_id);
5852
5853 let response = ClientControlResponse::SupervisorAck {
5854 module_id,
5855 applied: true,
5856 };
5857 Ok(vec![control_response_body_frame(
5858 &frame,
5859 &response,
5860 "ClientControlResponse::SupervisorAck",
5861 )?])
5862 }
5863
5864 async fn handle_supervisor_reload(
5865 &self,
5866 frame: Frame,
5867 module_id: String,
5868 requester: Principal,
5869 ) -> Result<Vec<Frame>, RouterError> {
5870 let operation_lock = self.supervisor.operation_lock();
5871 let _operation_guard = operation_lock.lock().await;
5872 let Some(module) = self.supervisor.get(&module_id) else {
5873 return Ok(vec![control_error_frame(
5874 &frame,
5875 "unknown_module",
5876 format!("module_id '{module_id}' is not supervised"),
5877 )?]);
5878 };
5879
5880 self.route_outages.mark_operator_action(&module_id);
5881 if let Err(err) = module.reload_requested(Some(requester)).await {
5882 self.route_outages
5883 .operator_action_ended_unrefused(&module_id);
5884 let (code, message) = match err {
5885 crate::supervise::SuperviseError::Disabled { .. } => {
5886 ("module_disabled", err.to_string())
5887 }
5888 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5889 ("swap_in_progress", err.to_string())
5890 }
5891 _ => (
5892 "reload_failed",
5893 format!("failed to reload module_id '{module_id}': {err}"),
5894 ),
5895 };
5896 return Ok(vec![control_error_frame(&frame, code, message)?]);
5897 }
5898
5899 let response = ClientControlResponse::SupervisorAck {
5900 module_id,
5901 applied: true,
5902 };
5903 Ok(vec![control_response_body_frame(
5904 &frame,
5905 &response,
5906 "ClientControlResponse::SupervisorAck",
5907 )?])
5908 }
5909
5910 async fn handle_supervisor_rescan(
5914 &self,
5915 frame: Frame,
5916 mode: RescanMode,
5917 requester: Principal,
5918 ) -> Result<Vec<Frame>, RouterError> {
5919 let preview = mode.previews();
5920 let Some(context) = self.rescan.clone() else {
5921 return Ok(vec![control_error_frame(
5922 &frame,
5923 "rescan_unavailable",
5924 "the daemon was not started with a reloadable config path".to_string(),
5925 )?]);
5926 };
5927
5928 let operation_lock = self.supervisor.operation_lock();
5929 let _operation_guard = operation_lock.lock().await;
5930 let loaded = match self.load_rescan_config(&mode, &context) {
5934 Ok(config) => config,
5935 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5936 };
5937 let Some((config, activity_config)) = loaded else {
5945 return Ok(vec![control_error_frame(
5946 &frame,
5947 "invalid_daemon_config",
5948 format!(
5949 "daemon config not found at {}; refusing to rescan (an absent config would \
5950 retire every supervised module)",
5951 context.config_path.display()
5952 ),
5953 )?]);
5954 };
5955 let (
5956 configured_port,
5957 storage_config,
5958 admission_facts_carrier_module_id,
5959 admission_facts_targets,
5960 scope_authority_owners,
5961 privileged_operator_prompt,
5962 modules,
5963 reserved_capabilities,
5964 ) = (
5965 config.port,
5966 config.storage,
5967 config.admission_facts_carrier_module_id,
5968 config.admission_facts_targets,
5969 config.scope_authority_owners,
5970 config.privileged_operator_prompt,
5971 config.modules,
5972 config.reserved_capabilities,
5973 );
5974
5975 let mut restart_required = Vec::new();
5984 for section in RestartRequiredSection::ALL {
5985 let changed = match section {
5986 RestartRequiredSection::Port => configured_port != context.configured_port,
5987 RestartRequiredSection::Storage => storage_config != context.storage_config,
5988 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5989 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5990 }
5991 RestartRequiredSection::AdmissionFactsTargets => {
5992 admission_facts_targets != context.admission_facts_targets
5993 }
5994 RestartRequiredSection::ScopeAuthorityOwners => {
5995 scope_authority_owners != context.scope_authority_owners
5996 }
5997 RestartRequiredSection::PrivilegedOperatorPrompt => {
5998 privileged_operator_prompt != context.privileged_operator_prompt
5999 }
6000 };
6001 if changed {
6002 restart_required.push(section.label().to_string());
6003 }
6004 }
6005 if !restart_required.is_empty() {
6006 warn!(
6007 config_path = %context.config_path.display(),
6008 sections = %restart_required.join(", "),
6009 "daemon config changed outside the modules section; restart the daemon to apply those changes"
6010 );
6011 }
6012
6013 for configured in &modules {
6014 if let Err(err) = validate_spec(&configured.module_spec()) {
6015 return Ok(vec![control_error_frame(
6016 &frame,
6017 "invalid_daemon_config",
6018 format!("supervisor rescan rejected daemon config: {err}"),
6019 )?]);
6020 }
6021 }
6022
6023 let configured_capabilities = modules
6024 .iter()
6025 .map(|module| (module.module_id.clone(), module.enabled))
6026 .collect::<Vec<_>>();
6027 let preview_capability_warnings = if preview {
6028 let (_, registrations) = self.runtime_capability_snapshot()?;
6029 let current_modules = self
6030 .supervisor
6031 .list()
6032 .into_iter()
6033 .map(|module| module.module_id().to_string())
6034 .collect::<BTreeSet<_>>();
6035 let resulting_modules = configured_capabilities.clone();
6036 let removed = current_modules
6037 .into_iter()
6038 .filter(|module_id| {
6039 !resulting_modules
6040 .iter()
6041 .any(|(configured_id, _)| configured_id == module_id)
6042 })
6043 .collect::<Vec<_>>();
6044 let disabled = match &mode {
6047 RescanMode::Candidate(_) => resulting_modules
6048 .iter()
6049 .filter(|(_, enabled)| !enabled)
6050 .map(|(module_id, _)| module_id.clone())
6051 .collect::<Vec<_>>(),
6052 _ => Vec::new(),
6053 };
6054 self.capability_evaluator.preview_warnings(
6055 resulting_modules,
6056 &removed,
6057 &disabled,
6058 ®istrations,
6059 )
6060 } else {
6061 Vec::new()
6062 };
6063 let reconciled = match &mode {
6064 RescanMode::Candidate(candidate) => self
6065 .preview_candidate(&context, modules, candidate)
6066 .map(|result| (result, Vec::new())),
6067 _ => {
6068 self.reconcile_supervised_modules(
6069 &context.supervisor,
6070 modules,
6071 preview,
6072 requester.clone(),
6073 )
6074 .await
6075 }
6076 };
6077 let (result, changed) = match reconciled {
6078 Ok(result) => result,
6079 Err(message) => {
6080 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6081 }
6082 };
6083 if !preview {
6084 if let Some(ledger) = &self.activity {
6085 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6086 ledger.config_changed(&result.added, &result.removed, &changed, requester);
6087 }
6088 if let Err(code) = self.configure_activity(activity_config).await {
6089 warn!(
6090 code,
6091 "activity retention could not be applied during rescan"
6092 );
6093 }
6094 self.capability_evaluator
6095 .configure(configured_capabilities, reserved_capabilities);
6096 self.capability_evaluator.wake_deadline_loop();
6097 self.refresh_capability_requirements();
6098 }
6099 let mut result = result;
6100 result.restart_required = restart_required;
6101 result.capability_warnings = preview_capability_warnings;
6102 let response = ClientControlResponse::SupervisorRescan { result };
6103 Ok(vec![control_response_body_frame(
6104 &frame,
6105 &response,
6106 "ClientControlResponse::SupervisorRescan",
6107 )?])
6108 }
6109
6110 async fn handle_supervisor_release_reserved(
6111 &self,
6112 frame: Frame,
6113 module_id: String,
6114 ) -> Result<Vec<Frame>, RouterError> {
6115 let Some(context) = self.rescan.clone() else {
6116 return Ok(vec![control_error_frame(
6117 &frame,
6118 "release_unavailable",
6119 "reserved-id release requires a daemon started with a reloadable config path",
6120 )?]);
6121 };
6122 let operation_lock = self.supervisor.operation_lock();
6123 let _operation_guard = operation_lock.lock().await;
6124 let loaded = match crate::daemon_config::load(&context.config_path) {
6125 Ok(Some(config)) => config,
6126 Ok(None) => {
6127 return Ok(vec![control_error_frame(
6128 &frame,
6129 "invalid_daemon_config",
6130 format!(
6131 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6132 context.config_path.display()
6133 ),
6134 )?])
6135 }
6136 Err(err) => {
6137 return Ok(vec![control_error_frame(
6138 &frame,
6139 "invalid_daemon_config",
6140 format!("unable to verify reserved-id release against daemon config: {err}"),
6141 )?])
6142 }
6143 };
6144 if loaded
6145 .modules
6146 .iter()
6147 .any(|configured| configured.module_id == module_id)
6148 {
6149 return Ok(vec![control_error_frame(
6150 &frame,
6151 "reserved_module_configured",
6152 format!(
6153 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6154 ),
6155 )?]);
6156 }
6157 if !self.supervisor.release_retained_reserved_gate(&module_id) {
6158 return Ok(vec![control_error_frame(
6159 &frame,
6160 "reserved_gate_not_retained",
6161 format!(
6162 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6163 ),
6164 )?]);
6165 }
6166
6167 let response = ClientControlResponse::SupervisorAck {
6168 module_id,
6169 applied: true,
6170 };
6171 Ok(vec![control_response_body_frame(
6172 &frame,
6173 &response,
6174 "ClientControlResponse::SupervisorAck",
6175 )?])
6176 }
6177
6178 fn plan_rescan(
6182 &self,
6183 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6184 ) -> Result<RescanPlan, String> {
6185 let mut current = BTreeMap::new();
6186 for module in self.supervisor.list() {
6187 let (spec, health) = module.configuration().map_err(|err| {
6188 format!(
6189 "failed to read configuration for module_id '{}': {err}",
6190 module.module_id()
6191 )
6192 })?;
6193 let enabled = module
6194 .status()
6195 .map_err(|err| {
6196 format!(
6197 "failed to read status for module_id '{}': {err}",
6198 module.module_id()
6199 )
6200 })?
6201 .enabled;
6202 current.insert(
6203 module.module_id().to_string(),
6204 (module, spec, health, enabled),
6205 );
6206 }
6207 let configured = configured_modules
6208 .into_iter()
6209 .map(|module| (module.module_id.clone(), module))
6210 .collect::<BTreeMap<_, _>>();
6211
6212 let added = configured
6213 .keys()
6214 .filter(|module_id| !current.contains_key(*module_id))
6215 .cloned()
6216 .collect::<Vec<_>>();
6217 let removed = current
6218 .keys()
6219 .filter(|module_id| !configured.contains_key(*module_id))
6220 .cloned()
6221 .collect::<Vec<_>>();
6222 let mut changed_pending_reload = Vec::new();
6223 let mut configuration_changes = BTreeSet::new();
6224 let mut enabled_changes = BTreeSet::new();
6225 let mut unchanged = 0_u32;
6226
6227 for (module_id, configured_module) in &configured {
6228 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6229 else {
6230 continue;
6231 };
6232 let launch_changed = *current_spec != configured_module.module_spec();
6236 let configuration_changed =
6237 launch_changed || *current_health != configured_module.health;
6238 let enabled_changed = *current_enabled != configured_module.enabled;
6239 if configuration_changed {
6240 configuration_changes.insert(module_id.clone());
6241 }
6242 if launch_changed {
6243 changed_pending_reload.push(module_id.clone());
6244 }
6245 if enabled_changed {
6246 enabled_changes.insert(module_id.clone());
6247 }
6248 if !configuration_changed && !enabled_changed {
6249 unchanged = unchanged.saturating_add(1);
6250 }
6251 }
6252 Ok(RescanPlan {
6253 current,
6254 configured,
6255 added,
6256 removed,
6257 changed_pending_reload,
6258 configuration_changes,
6259 enabled_changes,
6260 unchanged,
6261 })
6262 }
6263
6264 async fn reconcile_supervised_modules(
6273 &self,
6274 supervisor: &Supervisor,
6275 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6276 preview: bool,
6277 requester: Principal,
6278 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6279 let plan = self.plan_rescan(configured_modules)?;
6280
6281 if preview {
6287 return Ok((plan.preview_result(), Vec::new()));
6288 }
6289 let changed = plan
6290 .configuration_changes
6291 .union(&plan.enabled_changes)
6292 .cloned()
6293 .collect::<Vec<_>>();
6294 let RescanPlan {
6295 current,
6296 configured,
6297 added,
6298 removed,
6299 changed_pending_reload,
6300 configuration_changes,
6301 enabled_changes,
6302 unchanged,
6303 } = plan;
6304 if !(added.is_empty()
6309 && removed.is_empty()
6310 && configuration_changes.is_empty()
6311 && enabled_changes.is_empty())
6312 {
6313 if let Some(context) = &self.rescan {
6314 context
6315 .generation
6316 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6317 }
6318 }
6319
6320 for module_id in &removed {
6321 let module = ¤t
6322 .get(module_id)
6323 .expect("removed module came from current supervisor state")
6324 .0;
6325 module
6326 .retire_requested(Some(requester.clone()))
6327 .await
6328 .map_err(|err| {
6329 format!("failed to retire module_id '{module_id}' during rescan: {err}")
6330 })?;
6331 self.supervisor.record_rescan_removal(module_id);
6365 self.supervisor.retire(module_id);
6366 self.route_outages.forget(module_id);
6367 }
6368
6369 for module_id in configured.keys() {
6370 let Some((module, _, _, _)) = current.get(module_id) else {
6371 continue;
6372 };
6373 let configured_module = configured
6374 .get(module_id)
6375 .expect("configured module id came from configured map");
6376 if configuration_changes.contains(module_id) {
6377 module
6378 .update_configuration(
6379 configured_module.module_spec(),
6380 configured_module.health.clone(),
6381 configured_module.drain_timeout_ms,
6382 )
6383 .await
6384 .map_err(|err| {
6385 format!(
6386 "failed to update module_id '{module_id}' configuration during rescan: {err}"
6387 )
6388 })?;
6389 }
6390 if enabled_changes.contains(module_id) {
6391 self.route_outages.mark_operator_action(module_id);
6394 module
6395 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6396 .await
6397 .map_err(|err| {
6398 self.route_outages.operator_action_ended_unrefused(module_id);
6399 format!(
6400 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6401 configured_module.enabled
6402 )
6403 })?;
6404 }
6405 }
6406
6407 for module_id in &added {
6408 let configured_module = configured
6409 .get(module_id)
6410 .expect("added module id came from configured map");
6411 supervisor
6412 .supervise_configured_requested(
6413 configured_module.module_spec(),
6414 configured_module.enabled,
6415 configured_module.health.clone(),
6416 configured_module.drain_timeout_ms,
6417 configured_module.restart,
6418 configured_module.register_timeout,
6419 Some(requester.clone()),
6420 )
6421 .map_err(|err| {
6422 format!("failed to add module_id '{module_id}' during rescan: {err}")
6423 })?;
6424 }
6425
6426 Ok((
6427 SupervisorRescanResult {
6428 added,
6429 removed,
6430 changed_pending_reload,
6431 enabled_changes: enabled_changes.iter().cloned().collect(),
6432 unchanged,
6433 preview: false,
6434 restart_required: Vec::new(),
6437 capability_warnings: Vec::new(),
6438 module_actions: Vec::new(),
6439 earlier_pending: Vec::new(),
6440 expect: None,
6441 },
6442 changed,
6443 ))
6444 }
6445
6446 async fn handle_supervisor_set_enabled(
6447 &self,
6448 frame: Frame,
6449 module_id: String,
6450 enabled: bool,
6451 requester: Principal,
6452 ) -> Result<Vec<Frame>, RouterError> {
6453 let operation_lock = self.supervisor.operation_lock();
6454 let _operation_guard = operation_lock.lock().await;
6455 let Some(module) = self.supervisor.get(&module_id) else {
6456 return Ok(vec![control_error_frame(
6457 &frame,
6458 "unknown_module",
6459 format!("module_id '{module_id}' is not supervised"),
6460 )?]);
6461 };
6462
6463 self.route_outages.mark_operator_action(&module_id);
6466 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6467 Ok(applied) => applied,
6468 Err(err) => {
6469 self.route_outages
6470 .operator_action_ended_unrefused(&module_id);
6471 return Ok(vec![control_error_frame(
6472 &frame,
6473 "target_unavailable",
6474 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6475 )?]);
6476 }
6477 };
6478 if !applied {
6479 self.route_outages
6482 .operator_action_ended_unrefused(&module_id);
6483 }
6484
6485 self.capability_evaluator.wake_deadline_loop();
6486 self.refresh_capability_requirements();
6487 let response = ClientControlResponse::SupervisorAck { module_id, applied };
6488 Ok(vec![control_response_body_frame(
6489 &frame,
6490 &response,
6491 "ClientControlResponse::SupervisorAck",
6492 )?])
6493 }
6494
6495 async fn handle_supervisor_health_probe(
6496 &self,
6497 frame: Frame,
6498 module_id: String,
6499 ) -> Result<Vec<Frame>, RouterError> {
6500 self.refresh_capability_requirements();
6501 let Some(registration) = self
6502 .registry
6503 .get_module(&module_id)
6504 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6505 else {
6506 return Ok(vec![control_error_frame(
6507 &frame,
6508 "unknown_module",
6509 format!("module_id '{module_id}' is not registered"),
6510 )?]);
6511 };
6512
6513 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6526 {
6527 return Ok(vec![control_error_frame(
6528 &frame,
6529 "health_not_advertised",
6530 format!("module_id '{module_id}' did not advertise health.check"),
6531 )?]);
6532 }
6533
6534 let deadline = Instant::now() + self.health_probe_timeout;
6535 let pending = match self.forwarding.begin_module_control_rpc_for(
6536 &module_id,
6537 MODULE_CONTROL_OP_HEALTH_CHECK,
6538 deadline,
6539 ) {
6540 Ok(pending) => pending,
6541 Err(err) => {
6542 return Ok(vec![control_error_frame(
6543 &frame,
6544 forwarding_error_code(&err),
6545 err.to_string(),
6546 )?])
6547 }
6548 };
6549
6550 let PendingModuleControlRpc {
6551 endpoint,
6552 module_sink,
6553 negotiated_ver,
6554 corr: probe_corr,
6555 receiver,
6556 } = pending;
6557 let mut guard =
6558 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6559 let probe_body =
6560 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6561 RouterError::backend(
6562 0,
6563 frame.header.corr,
6564 format!("failed to encode health.check request: {err}"),
6565 )
6566 })?;
6567 let probe_frame = Frame::build_with_version(
6568 negotiated_ver,
6569 FrameType::Request,
6570 control_flags(),
6571 0,
6572 0,
6573 probe_corr,
6574 probe_body,
6575 )
6576 .map_err(RouterError::FrameBuild)?;
6577
6578 match timeout_at(deadline, module_sink.send(probe_frame)).await {
6581 Ok(Ok(())) => {}
6582 Ok(Err(err)) => {
6583 return Ok(vec![control_error_frame(
6584 &frame,
6585 "target_unavailable",
6586 err.to_string(),
6587 )?]);
6588 }
6589 Err(_) => {
6590 return Ok(vec![control_error_frame(
6591 &frame,
6592 "module_timeout",
6593 format!(
6594 "module_id '{module_id}' did not answer health.check within {:?}",
6595 self.health_probe_timeout
6596 ),
6597 )?]);
6598 }
6599 }
6600
6601 match timeout_at(deadline, receiver).await {
6602 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6603 guard.disarm();
6604 let Some(report) = response.health_report() else {
6605 return Ok(vec![control_error_frame(
6606 &frame,
6607 "invalid_control_body",
6608 "health.check RPC returned a non-health response",
6609 )?]);
6610 };
6611 let HealthReport {
6616 status,
6617 detail,
6618 metrics,
6619 } = report;
6620 let capability_detail = self
6621 .capability_evaluator
6622 .required_problem_detail(&module_id);
6623 let response = ClientControlResponse::SupervisorHealthProbe {
6624 module_id,
6625 status,
6626 detail: append_capability_problem_detail(detail, capability_detail),
6627 metrics,
6628 };
6629 Ok(vec![control_response_body_frame(
6630 &frame,
6631 &response,
6632 "ClientControlResponse::SupervisorHealthProbe",
6633 )?])
6634 }
6635 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6636 guard.disarm();
6637 Ok(vec![control_error_body_frame(&frame, body)?])
6638 }
6639 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6640 guard.disarm();
6641 Ok(vec![control_error_frame(
6642 &frame,
6643 "target_unavailable",
6644 message,
6645 )?])
6646 }
6647 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6648 guard.disarm();
6649 Ok(vec![control_error_frame(
6650 &frame,
6651 "invalid_control_body",
6652 message,
6653 )?])
6654 }
6655 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6656 guard.disarm();
6657 Ok(vec![control_error_frame(
6658 &frame,
6659 "invalid_control_body",
6660 format!("expected module-control op '{expected}', got '{actual}'"),
6661 )?])
6662 }
6663 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6664 guard.disarm();
6665 Ok(vec![control_error_frame(
6666 &frame,
6667 "module_timeout",
6668 format!(
6669 "module_id '{module_id}' answered health.check after {:?}",
6670 self.health_probe_timeout
6671 ),
6672 )?])
6673 }
6674 Ok(Err(_)) => Ok(vec![control_error_frame(
6675 &frame,
6676 "target_unavailable",
6677 "health.check waiter was canceled before the module responded",
6678 )?]),
6679 Err(_) => Ok(vec![control_error_frame(
6680 &frame,
6681 "module_timeout",
6682 format!(
6683 "module_id '{module_id}' did not answer health.check within {:?}",
6684 self.health_probe_timeout
6685 ),
6686 )?]),
6687 }
6688 }
6689
6690 fn supervisor_status(
6691 &self,
6692 module_id: &str,
6693 corr: u64,
6694 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6695 self.supervisor
6696 .get(module_id)
6697 .map(|module| {
6698 let warming = module.is_warming_for_control("status").map_err(|err| {
6699 RouterError::backend(
6700 0,
6701 corr,
6702 format!(
6703 "failed to read supervisor warming state for module_id '{module_id}': {err}"
6704 ),
6705 )
6706 })?;
6707 module.status_for_control("status").map_err(|err| {
6708 RouterError::backend(
6709 0,
6710 corr,
6711 format!(
6712 "failed to read supervisor status for module_id '{module_id}': {err}"
6713 ),
6714 )
6715 }).map(|status| (status, warming))
6716 })
6717 .transpose()
6718 }
6719
6720 fn guard_module_control_op(
6721 &self,
6722 frame: &Frame,
6723 module_id: &str,
6724 op: &str,
6725 ) -> Result<Option<Frame>, RouterError> {
6726 if self.module_grants_op(module_id, op, frame.header.corr)? {
6727 return Ok(None);
6728 }
6729
6730 Ok(Some(control_error_frame(
6731 frame,
6732 "op_not_allowed",
6733 format!("module_id '{module_id}' did not grant control op '{op}'"),
6734 )?))
6735 }
6736
6737 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6738 let Some(registration) = self
6739 .registry
6740 .get_module(module_id)
6741 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6742 else {
6743 return Ok(false);
6744 };
6745 Ok(module_registration_grants_op(®istration.control_ops, op))
6746 }
6747
6748 fn handle_status_update(
6749 &self,
6750 endpoint: ModuleEndpointId,
6751 frame: Frame,
6752 ) -> Result<Vec<Frame>, RouterError> {
6753 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6754 Ok(update) => update,
6755 Err(err) => {
6756 if is_known_module_push_op(&frame.body) {
6761 return Ok(vec![control_error_frame(
6762 &frame,
6763 "invalid_control_body",
6764 format!("malformed module control push body: {err}"),
6765 )?]);
6766 }
6767 return Ok(Vec::new());
6768 }
6769 };
6770
6771 match update {
6772 ModuleControlPush::RouteStatus {
6773 route_channel,
6774 route_epoch,
6775 status,
6776 } => {
6777 self.forwarding
6778 .cache_status(endpoint, route_channel, route_epoch, status)
6779 .map_err(RouterError::Forwarding)?;
6780 }
6781 }
6782 Ok(Vec::new())
6783 }
6784
6785 fn handle_route_poll(
6786 &self,
6787 ctx: &RouteCtx,
6788 frame: Frame,
6789 route_channel: u16,
6790 route_epoch: u32,
6791 kind: PollKind,
6792 ) -> Result<Vec<Frame>, RouterError> {
6793 let snapshot = self
6794 .forwarding
6795 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6796 .map_err(RouterError::Forwarding)?;
6797 let response = match (kind, snapshot) {
6798 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6799 ClientControlResponse::RoutePoll {
6800 route_channel,
6801 route_epoch,
6802 status,
6803 live: None,
6804 }
6805 }
6806 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6807 route_channel,
6808 route_epoch,
6809 status: None,
6810 live: None,
6811 },
6812 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6813 let live = self
6839 .process_liveness
6840 .as_ref()
6841 .and_then(|source| source.process_live(&module_id))
6842 .unwrap_or(true);
6843 ClientControlResponse::RoutePoll {
6844 route_channel,
6845 route_epoch,
6846 status: None,
6847 live: Some(live),
6848 }
6849 }
6850 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6851 route_channel,
6852 route_epoch,
6853 status: None,
6854 live: Some(false),
6855 },
6856 };
6857
6858 Ok(vec![control_response_body_frame(
6859 &frame,
6860 &response,
6861 "ClientControlResponse::RoutePoll",
6862 )?])
6863 }
6864
6865 pub(crate) fn observe_module_control_completion(
6866 &self,
6867 completion: ModuleControlRpcCompletion,
6868 ) -> bool {
6869 match completion {
6870 ModuleControlRpcCompletion::Unknown => false,
6871 ModuleControlRpcCompletion::Settled => true,
6872 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6873 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6874 info!(
6875 module_id = %module_id,
6876 latency_ms,
6877 "late health.check answer proves the module is alive"
6878 );
6879 match self
6880 .supervisor
6881 .record_late_health_answer(&module_id, latency_ms)
6882 {
6883 Ok(true) => {}
6884 Ok(false) => debug!(
6885 module_id = %module_id,
6886 latency_ms,
6887 "late health.check answer has no active supervisor snapshot"
6888 ),
6889 Err(err) => warn!(
6890 module_id = %module_id,
6891 latency_ms,
6892 error = %err,
6893 "failed to record late health.check answer"
6894 ),
6895 }
6896 true
6897 }
6898 }
6899 }
6900
6901 fn refuse_to_end_module_connection_for_a_client(
6922 &self,
6923 module_connection_id: ConnectionId,
6924 corr: u64,
6925 err: ForwardingError,
6926 ) -> Result<(), RouterError> {
6927 if let ForwardingError::ConnectionClosing { connection_id } = err {
6928 if connection_id != module_connection_id {
6929 warn!(
6930 module_connection_id = module_connection_id.get(),
6931 client_connection_id = connection_id.get(),
6932 corr,
6933 "dropping a route.bind response for a closing client; the module connection keeps serving"
6934 );
6935 return Ok(());
6936 }
6937 }
6938 Err(RouterError::Forwarding(err))
6939 }
6940
6941 fn handle_module_relay_response(
6942 &self,
6943 connection_id: ConnectionId,
6944 frame: Frame,
6945 ) -> Result<Vec<Frame>, RouterError> {
6946 let mut secondary_error = None;
6947 let outcome = match frame.header.ty {
6948 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6949 Ok(probe) if probe.op == "route.bind" => {
6950 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6951 Ok(ModuleControlResponse::RouteBindAck {}) => {
6952 RouteBindRelayOutcome::Accepted
6953 }
6954 Ok(other) => {
6955 let message =
6956 format!("route.bind response carried unexpected body: {other:?}");
6957 secondary_error = Some(control_error_frame(
6958 &frame,
6959 "invalid_control_body",
6960 message.clone(),
6961 )?);
6962 RouteBindRelayOutcome::ModuleGone(message)
6963 }
6964 Err(err) => {
6965 let message = format!("malformed route.bind response body: {err}");
6966 secondary_error = Some(control_error_frame(
6967 &frame,
6968 "invalid_control_body",
6969 message.clone(),
6970 )?);
6971 RouteBindRelayOutcome::ModuleGone(message)
6972 }
6973 }
6974 }
6975 Ok(probe) => {
6976 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6977 {
6978 Ok(response) => ModuleControlRpcOutcome::Response(response),
6979 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6980 "malformed {} response body: {err}",
6981 probe.op
6982 )),
6983 };
6984 let completion = self
6985 .forwarding
6986 .complete_module_control_rpc(
6987 connection_id,
6988 frame.header.corr,
6989 Some(&probe.op),
6990 outcome,
6991 )
6992 .map_err(RouterError::Forwarding)?;
6993 if !self.observe_module_control_completion(completion) {
6994 debug!(
6995 connection_id = connection_id.get(),
6996 corr = frame.header.corr,
6997 op = %probe.op,
6998 "dropping late or unknown module-control RPC response"
6999 );
7000 }
7001 return Ok(Vec::new());
7002 }
7003 Err(err) => {
7004 if let Some(expected_op) = self
7005 .forwarding
7006 .pending_module_control_op(connection_id, frame.header.corr)
7007 .map_err(RouterError::Forwarding)?
7008 {
7009 let completion = self
7010 .forwarding
7011 .complete_module_control_rpc(
7012 connection_id,
7013 frame.header.corr,
7014 None,
7015 ModuleControlRpcOutcome::MalformedResponse(format!(
7016 "malformed {expected_op} response body: {err}"
7017 )),
7018 )
7019 .map_err(RouterError::Forwarding)?;
7020 if !self.observe_module_control_completion(completion) {
7021 debug!(
7022 connection_id = connection_id.get(),
7023 corr = frame.header.corr,
7024 "dropping late malformed module-control RPC response"
7025 );
7026 }
7027 return Ok(Vec::new());
7028 }
7029 let message = format!("malformed route.bind response body: {err}");
7030 secondary_error = Some(control_error_frame(
7031 &frame,
7032 "invalid_control_body",
7033 message.clone(),
7034 )?);
7035 RouteBindRelayOutcome::ModuleGone(message)
7036 }
7037 },
7038 FrameType::Error => {
7039 if self
7040 .forwarding
7041 .pending_module_control_op(connection_id, frame.header.corr)
7042 .map_err(RouterError::Forwarding)?
7043 .is_some()
7044 {
7045 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7046 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7047 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7048 "malformed module-control ERROR body: {err}"
7049 )),
7050 };
7051 let completion = self
7052 .forwarding
7053 .complete_module_control_rpc(
7054 connection_id,
7055 frame.header.corr,
7056 None,
7057 outcome,
7058 )
7059 .map_err(RouterError::Forwarding)?;
7060 if !self.observe_module_control_completion(completion) {
7061 debug!(
7062 connection_id = connection_id.get(),
7063 corr = frame.header.corr,
7064 "dropping late or unknown module-control RPC error"
7065 );
7066 }
7067 return Ok(Vec::new());
7068 }
7069 match serde_json::from_slice::<ErrorBody>(&frame.body) {
7070 Ok(body) => RouteBindRelayOutcome::Rejected(body),
7071 Err(err) => {
7072 let message = format!("malformed route.bind ERROR body: {err}");
7073 secondary_error = Some(control_error_frame(
7074 &frame,
7075 "invalid_control_body",
7076 message.clone(),
7077 )?);
7078 RouteBindRelayOutcome::ModuleGone(message)
7079 }
7080 }
7081 }
7082 ty => {
7083 return Ok(vec![control_error_frame(
7084 &frame,
7085 "unsupported_control_frame",
7086 format!("unsupported module channel-0 frame {ty:?}"),
7087 )?])
7088 }
7089 };
7090
7091 let settled =
7092 self.forwarding
7093 .complete_pending_relay(connection_id, frame.header.corr, outcome);
7094 let completion = match settled {
7095 Ok(completion) => completion,
7096 Err(err) => {
7097 self.refuse_to_end_module_connection_for_a_client(
7098 connection_id,
7099 frame.header.corr,
7100 err,
7101 )?;
7102 return Ok(secondary_error.into_iter().collect());
7103 }
7104 };
7105 if let Some(target) = completion.abandoned.as_ref() {
7106 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7107 }
7108 if !completion.settled {
7109 debug!(
7110 connection_id = connection_id.get(),
7111 corr = frame.header.corr,
7112 frame_type = ?frame.header.ty,
7113 "dropping late or unknown route.bind relay response"
7114 );
7115 }
7116 Ok(secondary_error.into_iter().collect())
7117 }
7118
7119 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7120 debug!(connection_id = connection_id.get(), "handling GOODBYE");
7121 self.cleanup_connection_with_end_reason(
7125 connection_id,
7126 RegistrationEndReason::ExplicitGoodbye,
7127 )
7128 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7129 Ok(Vec::new())
7130 }
7131}
7132
7133impl Default for ControlHandler {
7134 fn default() -> Self {
7135 Self::new(Arc::new(Registry::default()))
7136 }
7137}
7138
7139impl crate::supervise::SwapPromotionObserver for ControlHandler {
7140 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7141 self.apply_registration_capabilities(registration);
7142 }
7143}
7144
7145fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7146 CapabilityRequirementStatus {
7147 consumer: status.consumer,
7148 capability: status.capability,
7149 need: match status.need {
7150 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7151 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7152 },
7153 verdict: status.verdict.as_str().to_string(),
7154 episode_seq: status.episode_seq,
7155 config_satisfiable: status.config_satisfiable,
7156 runtime_available: status.runtime_available,
7157 detail: status.detail,
7158 }
7159}
7160
7161fn append_capability_problem_detail(
7162 detail: Option<String>,
7163 capability_detail: Option<String>,
7164) -> Option<String> {
7165 match (detail, capability_detail) {
7166 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7167 (Some(detail), None) => Some(detail),
7168 (None, Some(capability_detail)) => Some(capability_detail),
7169 (None, None) => None,
7170 }
7171}
7172
7173fn subc_ops() -> Vec<String> {
7174 SUBC_CONTROL_OPS
7175 .iter()
7176 .map(|op| (*op).to_string())
7177 .collect()
7178}
7179
7180fn module_subc_ops() -> Vec<String> {
7181 SUBC_CONTROL_OPS
7182 .iter()
7183 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7184 .map(|op| (*op).to_string())
7185 .collect()
7186}
7187
7188#[cfg(test)]
7189fn module_baseline_control_ops() -> Vec<String> {
7190 MODULE_BASELINE_CONTROL_OPS
7191 .iter()
7192 .map(|op| (*op).to_string())
7193 .collect()
7194}
7195
7196fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7197 let mut seen = HashSet::new();
7198 let mut effective = Vec::new();
7199 for op in MODULE_BASELINE_CONTROL_OPS {
7200 if seen.insert((*op).to_string()) {
7201 effective.push((*op).to_string());
7202 }
7203 }
7204 for op in declared.unwrap_or_default() {
7205 if seen.insert(op.clone()) {
7206 effective.push(op);
7207 }
7208 }
7209 effective
7210}
7211
7212fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7213 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7214}
7215
7216fn target_module_id(target: &RouteTarget) -> &str {
7217 match target {
7218 RouteTarget::ToolProvider { module_id }
7219 | RouteTarget::ManagementSurface { module_id }
7220 | RouteTarget::InternalService { module_id, .. } => module_id,
7221 }
7222}
7223
7224fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7225 roles.iter().any(|role| match (target, role) {
7226 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7227 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7228 (
7229 RouteTarget::InternalService { service_id, .. },
7230 ProviderRole::InternalService {
7231 service_id: provided,
7232 ..
7233 },
7234 ) => service_id == provided,
7235 _ => false,
7236 })
7237}
7238
7239fn is_routable_role(role: &ProviderRole) -> bool {
7240 matches!(
7241 role,
7242 ProviderRole::ToolProvider { .. }
7243 | ProviderRole::ManagementSurface { .. }
7244 | ProviderRole::InternalService { .. }
7245 )
7246}
7247
7248#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7249enum ControlRequestBodyError {
7250 UnknownOp,
7251 InvalidBody,
7252}
7253
7254#[derive(Debug, Deserialize)]
7255struct ControlOpProbe {
7256 op: String,
7257}
7258
7259const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7262
7263fn is_known_module_push_op(body: &[u8]) -> bool {
7264 serde_json::from_slice::<ControlOpProbe>(body)
7265 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7266 .unwrap_or(false)
7267}
7268
7269fn is_known_module_request_op(body: &[u8]) -> bool {
7270 serde_json::from_slice::<ControlOpProbe>(body)
7271 .map(|probe| is_module_to_subc_op(&probe.op))
7272 .unwrap_or(false)
7273}
7274
7275fn is_module_to_subc_op(op: &str) -> bool {
7276 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7277}
7278
7279fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7280 debug!(
7281 op = %op,
7282 connection_id = connection_id.get(),
7283 corr,
7284 "control dispatch"
7285 );
7286}
7287
7288fn log_slow_control_dispatch(
7289 dispatch_started_at: Option<StdInstant>,
7290 op: &'static str,
7291 connection_id: ConnectionId,
7292 corr: u64,
7293 route_open_timings: Option<&RouteOpenTimings>,
7294) {
7295 let Some(dispatch_started_at) = dispatch_started_at else {
7296 return;
7297 };
7298 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7300 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7301 if let Some(timings) = route_open_timings {
7302 warn!(
7303 op = %op,
7304 connection_id = connection_id.get(),
7305 corr,
7306 elapsed_ms = elapsed.as_millis() as u64,
7307 module_id = ?timings.module_id,
7308 principal = %timings.principal,
7309 admission_ms = timings.admission.as_millis() as u64,
7310 project_root_ms = timings.project_root.as_millis() as u64,
7311 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7312 bind_send_ms = timings.bind_send.as_millis() as u64,
7313 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7314 "slow control dispatch"
7315 );
7316 return;
7317 }
7318 warn!(
7319 op = %op,
7320 connection_id = connection_id.get(),
7321 corr,
7322 elapsed_ms = elapsed.as_millis() as u64,
7323 "slow control dispatch"
7324 );
7325 }
7326}
7327
7328#[cfg(test)]
7330type TestCallerLookup =
7331 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7332
7333fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7335 match request {
7336 ClientControlRequest::SupervisorRestart { module_id, .. } => {
7337 Some(PrivilegedOperation::Restart { module_id })
7338 }
7339 ClientControlRequest::SupervisorSwap { module_id, .. } => {
7340 Some(PrivilegedOperation::Swap { module_id })
7341 }
7342 ClientControlRequest::SupervisorReload { module_id } => {
7343 Some(PrivilegedOperation::Reload { module_id })
7344 }
7345 ClientControlRequest::SupervisorRescan { preview } => {
7346 if *preview {
7347 None
7348 } else {
7349 Some(PrivilegedOperation::Rescan)
7350 }
7351 }
7352 ClientControlRequest::SupervisorRescanChecked {
7357 preview,
7358 candidate,
7359 expect,
7360 } => (!*preview && candidate.is_none() && expect.is_some())
7361 .then_some(PrivilegedOperation::Rescan),
7362 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7363 Some(PrivilegedOperation::SetEnabled {
7364 module_id,
7365 enabled: *enabled,
7366 })
7367 }
7368 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7369 Some(PrivilegedOperation::ReleaseReserved { module_id })
7370 }
7371 ClientControlRequest::ServerDescribe {}
7372 | ClientControlRequest::CatalogList { .. }
7373 | ClientControlRequest::RouteOpen { .. }
7374 | ClientControlRequest::RoutePoll { .. }
7375 | ClientControlRequest::SupervisorList {}
7376 | ClientControlRequest::SupervisorHealth {}
7377 | ClientControlRequest::SupervisorHealthProbe { .. }
7378 | ClientControlRequest::SupervisorRoutes { .. }
7379 | ClientControlRequest::SupervisorProvenance { .. }
7380 | ClientControlRequest::SupervisorStderrTail { .. }
7381 | ClientControlRequest::SupervisorTerminals { .. }
7382 | ClientControlRequest::SupervisorSpawnSnapshot {}
7383 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7384 }
7385}
7386
7387fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7388 match request {
7389 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7390 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7391 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7392 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7393 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7394 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7395 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7396 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7397 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7398 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7399 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7400 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7401 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7402 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7403 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7404 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7405 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7406 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7407 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7408 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7409 }
7410}
7411
7412struct RouteOpenCaller {
7420 principal: String,
7424 scope: String,
7427 harness: String,
7430}
7431
7432impl RouteOpenCaller {
7433 fn new(
7434 principal: Option<&Principal>,
7435 scope: Option<&ScopeSelector>,
7436 identity: &BindIdentity,
7437 ) -> Self {
7438 Self {
7439 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
7440 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
7441 harness: bounded_log_text(&identity.harness),
7442 }
7443 }
7444
7445 fn unknown() -> Self {
7448 Self {
7449 principal: "unknown".to_string(),
7450 scope: "unknown".to_string(),
7451 harness: "unknown".to_string(),
7452 }
7453 }
7454}
7455
7456const MAX_LOGGED_TEXT_CHARS: usize = 64;
7459
7460fn bounded_log_text(value: &str) -> String {
7464 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
7465 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
7466 bounded.push('…');
7467 }
7468 format!("{bounded:?}")
7469}
7470
7471fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7477 use subc_protocol::scope::ScopeKind;
7478 let kind = match stamp.kind {
7479 ScopeKind::Head => "head",
7480 ScopeKind::Worker => "worker",
7481 ScopeKind::Ephemeral => "ephemeral",
7482 };
7483 format!(
7484 "{kind}/{}/{}@{}",
7485 principal_label(&stamp.owner),
7486 bounded_log_text(&stamp.scope_ref),
7487 stamp.scope_epoch
7488 )
7489}
7490
7491fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
7498 let epoch = selector
7499 .scope_epoch
7500 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
7501 format!(
7502 "requested/{}/{}@{epoch}",
7503 bounded_log_text(&principal_label(&selector.owner)),
7504 bounded_log_text(&selector.scope_ref),
7505 )
7506}
7507
7508fn principal_label(principal: &Principal) -> String {
7509 match principal {
7510 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7511 Principal::Direct => "direct".to_string(),
7512 other => format!("{other:?}"),
7513 }
7514}
7515
7516fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7517 match request {
7518 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7519 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7520 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7521 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7522 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7523 }
7524}
7525
7526fn parse_client_control_request(
7527 body: &[u8],
7528) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7529 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7530 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7531 Ok(probe)
7532 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7533 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7534 {
7535 ControlRequestBodyError::InvalidBody
7536 }
7537 Ok(_) => ControlRequestBodyError::UnknownOp,
7538 Err(_) => ControlRequestBodyError::InvalidBody,
7539 };
7540 (err, classification)
7541 })
7542}
7543
7544fn parse_module_control_request_from_module(
7545 body: &[u8],
7546) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7547 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7548 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7549 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7550 Ok(_) => ControlRequestBodyError::UnknownOp,
7551 Err(_) => ControlRequestBodyError::InvalidBody,
7552 };
7553 (err, classification)
7554 })
7555}
7556
7557#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7558enum ProviderRoleKind {
7559 ToolProvider,
7560 PipelineStage,
7561 ManagementSurface,
7562 InternalService,
7563}
7564
7565fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7566 match role {
7567 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7568 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7569 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7570 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7571 }
7572}
7573
7574fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7575 roles.iter().map(provider_role_kind).collect()
7576}
7577
7578const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7581
7582#[derive(Debug, Default, PartialEq, Eq)]
7584struct ScopeOutcomeCounts {
7585 created: usize,
7586 replaced: usize,
7587 updated: usize,
7588 unchanged: usize,
7589 refused: usize,
7590}
7591
7592impl ScopeOutcomeCounts {
7593 fn of(results: &[ScopeRecordResult]) -> Self {
7594 let mut counts = Self::default();
7595 for result in results {
7596 let slot = match result.outcome {
7597 ScopeRecordOutcome::Created => &mut counts.created,
7598 ScopeRecordOutcome::Replaced => &mut counts.replaced,
7599 ScopeRecordOutcome::Updated => &mut counts.updated,
7600 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7601 ScopeRecordOutcome::Refused => &mut counts.refused,
7602 };
7603 *slot += 1;
7604 }
7605 counts
7606 }
7607}
7608
7609#[cfg(test)]
7610mod scope_outcome_count_tests {
7611 use super::*;
7612
7613 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7614 ScopeRecordResult {
7615 scope_ref: "r".to_string(),
7616 scope_epoch: 1,
7617 outcome,
7618 code: None,
7619 message: None,
7620 version: None,
7621 parent_state: None,
7622 }
7623 }
7624
7625 #[test]
7628 fn every_outcome_is_counted_in_its_own_field() {
7629 let results = [
7630 result(ScopeRecordOutcome::Created),
7631 result(ScopeRecordOutcome::Created),
7632 result(ScopeRecordOutcome::Replaced),
7633 result(ScopeRecordOutcome::Updated),
7634 result(ScopeRecordOutcome::Unchanged),
7635 result(ScopeRecordOutcome::Refused),
7636 result(ScopeRecordOutcome::Refused),
7637 result(ScopeRecordOutcome::Refused),
7638 ];
7639 assert_eq!(
7640 ScopeOutcomeCounts::of(&results),
7641 ScopeOutcomeCounts {
7642 created: 2,
7643 replaced: 1,
7644 updated: 1,
7645 unchanged: 1,
7646 refused: 3,
7647 }
7648 );
7649 }
7650}
7651
7652fn capability_census_trigger(
7656 old: Option<&CapabilityDeclarations>,
7657 new: Option<&CapabilityDeclarations>,
7658) -> bool {
7659 let old_provides = old
7660 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7661 .unwrap_or_default();
7662 let old_denies = old
7663 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7664 .unwrap_or_default();
7665 let new = new.cloned().unwrap_or(CapabilityDeclarations {
7666 provides: Vec::new(),
7667 requires: Vec::new(),
7668 must_never_reach: Vec::new(),
7669 });
7670
7671 new.provides
7672 .iter()
7673 .any(|capability| !old_provides.contains(capability))
7674 || new
7675 .must_never_reach
7676 .iter()
7677 .any(|capability| !old_denies.contains(capability))
7678}
7679
7680fn denied_capability<'a>(
7683 opening_manifest: &'a ModuleManifest,
7684 target_manifest: &ModuleManifest,
7685) -> Option<&'a str> {
7686 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7687 let target_capabilities = target_manifest.capabilities.as_ref()?;
7688 opening_capabilities
7689 .must_never_reach
7690 .iter()
7691 .find(|denied| {
7692 target_capabilities
7693 .provides
7694 .iter()
7695 .any(|provided| provided == *denied)
7696 })
7697 .map(String::as_str)
7698}
7699
7700fn catalog_update_frozen_field_message(
7701 registered: &ModuleManifest,
7702 provides: &[ProviderRole],
7703) -> Option<String> {
7704 let old_has_provides = !registered.provides.is_empty();
7705 let new_has_provides = !provides.is_empty();
7706 if old_has_provides != new_has_provides {
7707 return Some(format!(
7708 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7709 registered.module_id
7710 ));
7711 }
7712
7713 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
7714 return Some(format!(
7715 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7716 registered.module_id
7717 ));
7718 }
7719
7720 let registered_concurrency = manifest_concurrency(registered);
7721 let mut candidate = registered.clone();
7722 candidate.provides = provides.to_vec();
7723 let candidate_concurrency = manifest_concurrency(&candidate);
7724 if candidate_concurrency != registered_concurrency {
7725 return Some(format!(
7726 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7727 registered.module_id, registered_concurrency, candidate_concurrency
7728 ));
7729 }
7730
7731 None
7734}
7735
7736fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7737 manifest.provides.iter().any(is_routable_role)
7738}
7739
7740fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7746 manifest
7747 .provides
7748 .iter()
7749 .find_map(|provider| match provider {
7750 ProviderRole::ToolProvider { concurrency, .. }
7751 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7752 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7753 })
7754 .unwrap_or(Concurrency::ModuleManaged)
7755}
7756
7757fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7764 let has_management_surface = manifest
7765 .provides
7766 .iter()
7767 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7768 if !has_management_surface {
7769 return false;
7770 }
7771 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7772 return false;
7773 };
7774 let Some(provides) = raw
7775 .get("manifest")
7776 .and_then(|manifest| manifest.get("provides"))
7777 .and_then(serde_json::Value::as_array)
7778 else {
7779 return false;
7780 };
7781 provides.iter().any(|role| {
7786 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7787 && role.get("concurrency").is_none()
7788 })
7789}
7790
7791fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7792 if peer_version != PROTOCOL_VERSION {
7793 return Err(format!(
7794 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7795 ));
7796 }
7797 Ok(PROTOCOL_VERSION)
7798}
7799
7800fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7801 Frame::build_with_version(
7802 response_version(frame),
7803 FrameType::Pong,
7804 frame.header.flags,
7805 0,
7806 0,
7807 frame.header.corr,
7808 Vec::new(),
7809 )
7810 .map_err(RouterError::FrameBuild)
7811}
7812
7813pub(crate) fn control_error_frame(
7814 frame: &Frame,
7815 code: &'static str,
7816 message: impl Into<String>,
7817) -> Result<Frame, RouterError> {
7818 control_error_body_frame(
7819 frame,
7820 ErrorBody {
7821 code: code.to_string(),
7822 message: message.into(),
7823 detail: None,
7824 },
7825 )
7826}
7827
7828fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7829 let body = serde_json::to_vec(&error).map_err(|err| {
7830 RouterError::backend(
7831 0,
7832 frame.header.corr,
7833 format!("failed to encode control ERROR: {err}"),
7834 )
7835 })?;
7836
7837 Frame::build_with_version(
7838 response_version(frame),
7839 FrameType::Error,
7840 control_flags(),
7841 0,
7842 0,
7843 frame.header.corr,
7844 body,
7845 )
7846 .map_err(RouterError::FrameBuild)
7847}
7848
7849fn control_response_body_frame<T: Serialize>(
7850 frame: &Frame,
7851 reply: &T,
7852 label: &'static str,
7853) -> Result<Frame, RouterError> {
7854 let body = serde_json::to_vec(reply).map_err(|err| {
7855 RouterError::backend(
7856 0,
7857 frame.header.corr,
7858 format!("failed to encode {label}: {err}"),
7859 )
7860 })?;
7861
7862 Frame::build_with_version(
7863 response_version(frame),
7864 FrameType::Response,
7865 control_flags(),
7866 0,
7867 0,
7868 frame.header.corr,
7869 body,
7870 )
7871 .map_err(RouterError::FrameBuild)
7872}
7873
7874fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7904 match err {
7905 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7906 ForwardingError::NoModuleConnection => "target_unavailable",
7907 ForwardingError::ModuleReloading { .. } => "module_reloading",
7908 ForwardingError::ClientRouteChannelExhausted { .. }
7909 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7910 ForwardingError::StaleModuleEndpoint
7911 | ForwardingError::UnknownReservation { .. }
7912 | ForwardingError::ConnectionClosing { .. }
7913 | ForwardingError::ClientEgressClosed { .. }
7914 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7915 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7918 ForwardingError::RelayCorrelationExhausted
7919 | ForwardingError::RouteOpenBuild(_)
7920 | ForwardingError::Poisoned => "forwarding_error",
7921 }
7922}
7923
7924fn response_version(frame: &Frame) -> u8 {
7925 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7926 frame.header.ver
7927 } else {
7928 PROTOCOL_VERSION
7929 }
7930}
7931
7932fn control_flags() -> Flags {
7933 Flags::new(false, Priority::Passive, false)
7934}
7935
7936fn send_goodbye_target_best_effort(
7941 counters: &DaemonCounters,
7942 target: &GoodbyeTarget,
7943 context: &'static str,
7944) {
7945 let Ok(frame) = Frame::build_with_version(
7946 target.negotiated_ver,
7947 FrameType::Goodbye,
7948 control_flags(),
7949 target.channel,
7950 target.epoch,
7951 0,
7952 Vec::new(),
7953 ) else {
7954 return;
7955 };
7956 crate::forwarding::send_module_route_goodbye(
7957 counters,
7958 &target.sink,
7959 frame,
7960 target.module_id.as_deref(),
7961 context,
7962 );
7963}
7964
7965pub(crate) fn send_route_control_pushes(
7966 forwarding: &ForwardingTable,
7967 routes: Vec<EndpointRoute>,
7968 push: ClientControlPush,
7969) {
7970 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7971 for route in routes {
7972 let target = route.goodbye_target;
7973 if let Some((existing, channels)) = targets
7974 .iter_mut()
7975 .find(|(existing, _)| existing.connection_id == target.connection_id)
7976 {
7977 debug_assert_eq!(
7978 existing.negotiated_ver, target.negotiated_ver,
7979 "one connection cannot negotiate multiple frame versions"
7980 );
7981 if !channels.contains(&target.channel) {
7982 channels.push(target.channel);
7983 }
7984 continue;
7985 }
7986 let channel = target.channel;
7987 targets.push((target, vec![channel]));
7988 }
7989 for (target, mut channels) in targets {
7990 channels.sort_unstable();
7991 let mut push = push.clone();
7992 match &mut push {
7993 ClientControlPush::RouteClosing {
7994 channels: covered, ..
7995 }
7996 | ClientControlPush::RouteClosed {
7997 channels: covered, ..
7998 } => *covered = channels,
7999 }
8000 let body = match serde_json::to_vec(&push) {
8001 Ok(body) => body,
8002 Err(err) => {
8003 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8004 continue;
8005 }
8006 };
8007 let frame = match Frame::build_with_version(
8008 target.negotiated_ver,
8009 FrameType::Push,
8010 control_flags(),
8011 0,
8012 0,
8013 0,
8014 body.clone(),
8015 ) {
8016 Ok(frame) => frame,
8017 Err(err) => {
8018 warn!(
8019 route_channel = target.channel,
8020 error = %err,
8021 "failed to build route lifecycle control PUSH frame"
8022 );
8023 continue;
8024 }
8025 };
8026 if let Err(err) = target.sink.try_send(frame) {
8027 if target.close_on_delivery_failure() {
8028 warn!(
8029 target_connection_id = target.connection_id.get(),
8030 route_channel = target.channel,
8031 error = %err,
8032 "route lifecycle control PUSH was not delivered to client; closing target connection"
8033 );
8034 let _ = forwarding.escalate_client_delivery_failure(
8035 target.connection_id,
8036 target.channel,
8037 target.epoch,
8038 CloseReason::new(
8039 "route_lifecycle_push_delivery_failed",
8040 format!(
8041 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8042 target.channel
8043 ),
8044 ),
8045 crate::forwarding::UndeliveredFrame {
8046 module_id: target.module_id.as_deref(),
8047 sink: &target.sink,
8048 },
8049 );
8050 }
8051 }
8052 }
8053}
8054
8055#[cfg(test)]
8056mod tests {
8057 mod activity_read {
8058 include!("activity_read_tests.rs");
8059 }
8060
8061 mod lifecycle_activity {
8062 include!("control_lifecycle_tests.rs");
8063 }
8064 #[cfg(unix)]
8065 #[tokio::test]
8066 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8067 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8068 let path = dir.join("subc.jsonc");
8069 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8070 "program":"/bin/sleep","args":["60"],"protocol":"none",
8071 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8072 }}}).to_string()).unwrap();
8073 let mut configured = crate::daemon_config::load(&path)
8074 .unwrap()
8075 .unwrap()
8076 .modules
8077 .pop()
8078 .unwrap();
8079 let registry = std::sync::Arc::new(crate::Registry::default());
8080 let handle = crate::SupervisorHandle::new();
8081 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8082 .with_handle(handle.clone());
8083 let module = supervisor
8084 .supervise_configured_with_health(
8085 configured.module_spec(),
8086 true,
8087 configured.health.clone(),
8088 None,
8089 configured.restart,
8090 )
8091 .unwrap();
8092 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8093 let before = module.status().unwrap().pid;
8094 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8095 configured.health.cadence = std::time::Duration::from_secs(3600);
8096 let health_only = handler
8097 .reconcile_supervised_modules(
8098 &supervisor,
8099 vec![configured.clone()],
8100 false,
8101 Principal::Direct,
8102 )
8103 .await
8104 .unwrap();
8105 assert!(
8106 health_only.0.changed_pending_reload.is_empty(),
8107 "health policy is already applied live"
8108 );
8109 assert_eq!(module.status().unwrap().pid, before);
8110 assert_eq!(
8111 module.configuration().unwrap().1.http,
8112 configured.health.http
8113 );
8114 configured.args = vec!["61".into()];
8115 let launch = handler
8116 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8117 .await
8118 .unwrap();
8119 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8120 assert_eq!(
8121 module.status().unwrap().pid,
8122 before,
8123 "a launch edit is stored until reload"
8124 );
8125 module.drain().await.unwrap();
8126 }
8127 use cortexkit_test_support::ScratchDir;
8128 use std::{
8129 collections::BTreeMap,
8130 fmt,
8131 path::PathBuf,
8132 sync::{Arc, Mutex},
8133 time::Duration,
8134 };
8135
8136 use serde_json::{json, Value};
8137 use subc_protocol::{
8138 manifest::{
8139 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8140 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8141 },
8142 session::HealthStatus,
8143 FrameType,
8144 };
8145
8146 use super::*;
8147 use crate::{
8148 forwarding::{DataRoute, DataRouteState},
8149 registry::ChannelState,
8150 router::FrameSink,
8151 stderr_tail::DEFAULT_MAX_LINE_BYTES,
8152 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8153 RouteCtx, Router,
8154 };
8155 use tokio::{
8156 sync::mpsc,
8157 time::{sleep, Instant},
8158 };
8159 use tracing::{
8160 field::{Field, Visit},
8161 Event, Subscriber,
8162 };
8163 use tracing_subscriber::{layer::Context, prelude::*, Layer};
8164
8165 fn fake_aft_stub_path() -> PathBuf {
8184 let mut path = std::env::current_exe().expect("current_exe available in tests");
8185 path.pop(); path.pop(); path.push(if cfg!(windows) {
8188 "fake-aft-stub.exe"
8189 } else {
8190 "fake-aft-stub"
8191 });
8192 assert!(
8193 path.exists(),
8194 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8195 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8196 path.display()
8197 );
8198 path
8199 }
8200
8201 fn client_retries(code: &str) -> bool {
8206 subc_protocol::error_codes::is_retryable_route_open(code)
8207 }
8208
8209 #[test]
8220 fn retryability_of_forwarding_codes_matches_the_failure() {
8221 let transient = [
8224 ForwardingError::NoModuleConnection,
8225 ForwardingError::ModuleReloading {
8226 module_id: "m".into(),
8227 },
8228 ForwardingError::StaleModuleEndpoint,
8229 ForwardingError::UnknownReservation {
8230 client_channel: 1,
8231 module_channel: 1,
8232 },
8233 ForwardingError::ConnectionClosing {
8234 connection_id: ConnectionId::new(1),
8235 },
8236 ForwardingError::ClientEgressClosed {
8237 connection_id: ConnectionId::new(1),
8238 },
8239 ForwardingError::ModuleEgressUnavailable {
8240 connection_id: ConnectionId::new(1),
8241 },
8242 ];
8243 for err in transient {
8244 let code = forwarding_error_code(&err);
8245 assert!(
8246 client_retries(code),
8247 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8248 );
8249 }
8250
8251 let permanent = [
8256 ForwardingError::ConnectionRoleConflict {
8257 connection_id: ConnectionId::new(1),
8258 },
8259 ForwardingError::ClientRouteChannelExhausted {
8260 connection_id: ConnectionId::new(1),
8261 },
8262 ForwardingError::ModuleRouteChannelExhausted {
8263 endpoint: ModuleEndpointId {
8264 connection_id: ConnectionId::new(1),
8265 generation: 1,
8266 },
8267 },
8268 ForwardingError::RelayCorrelationExhausted,
8269 ForwardingError::RouteOpenBuild("x".into()),
8270 ForwardingError::Poisoned,
8271 ];
8272 for err in permanent {
8273 let code = forwarding_error_code(&err);
8274 assert!(
8275 !client_retries(code),
8276 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8277 );
8278 }
8279 }
8280
8281 #[tokio::test]
8309 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8310 let handler = ControlHandler::default();
8311 let frame =
8312 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8313
8314 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8318 assert_eq!(
8319 stamped,
8320 Principal::Direct,
8321 "a caller that proved nothing must not be stamped as a supervised module"
8322 );
8323
8324 let forged = handler
8329 .route_open_principal(
8330 &frame,
8331 Some(ConsumerIdentity {
8332 module_id: "aft".to_string(),
8333 launch_nonce: "not-a-real-nonce".to_string(),
8334 }),
8335 )
8336 .unwrap();
8337 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8338 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8339 }
8340
8341 #[test]
8358 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8359 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8360 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8361 let ClientControlRequest::RouteOpen {
8362 consumer_identity, ..
8363 } = parsed
8364 else {
8365 panic!("route.open body must parse as RouteOpen");
8366 };
8367 assert_eq!(
8368 consumer_identity,
8369 Some(ConsumerIdentity {
8370 module_id: "aft".to_string(),
8371 launch_nonce: "n".to_string(),
8372 }),
8373 "the wire field name must reach the value route_open_principal reads"
8374 );
8375 }
8376
8377 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8378 ModuleManifest::builder(module_id, "0.1.0")
8379 .protocol_ver(protocol_ver)
8380 .provides(vec![ProviderRole::ToolProvider {
8381 tools: vec![Tool {
8382 name: "read".to_string(),
8383 description: None,
8384 execution_mode: ExecutionMode::Pure,
8385 schema: json!({"type": "object"}),
8386 }],
8387 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8388 concurrency: Concurrency::ModuleManaged,
8389 emits_push: true,
8390 sub_supervises: true,
8391 }])
8392 .build()
8393 }
8394
8395 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8396 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8397 }
8398
8399 fn hello_frame_with_control_ops(
8400 module_id: &str,
8401 protocol_ver: u8,
8402 corr: u64,
8403 control_ops: Option<Vec<String>>,
8404 ) -> Frame {
8405 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8406 }
8407
8408 fn hello_frame_with_nonce(
8409 module_id: &str,
8410 protocol_ver: u8,
8411 corr: u64,
8412 launch_nonce: Option<&str>,
8413 ) -> Frame {
8414 hello_frame_full(
8415 module_id,
8416 protocol_ver,
8417 corr,
8418 None,
8419 launch_nonce.map(ToOwned::to_owned),
8420 )
8421 }
8422
8423 fn hello_frame_full(
8424 module_id: &str,
8425 protocol_ver: u8,
8426 corr: u64,
8427 control_ops: Option<Vec<String>>,
8428 launch_nonce: Option<String>,
8429 ) -> Frame {
8430 let body = serde_json::to_vec(&ModuleHelloBody {
8431 manifest: manifest(module_id, protocol_ver),
8432 protocol_ver,
8433 control_ops,
8434 launch_nonce,
8435 })
8436 .unwrap();
8437 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8438 }
8439
8440 fn hello_frame_with_rich_ui(
8441 module_id: &str,
8442 protocol_ver: u8,
8443 corr: u64,
8444 launch_nonce: Option<&str>,
8445 rich_ui: Value,
8446 ) -> Frame {
8447 let mut body = serde_json::to_value(ModuleHelloBody {
8448 manifest: manifest(module_id, protocol_ver),
8449 protocol_ver,
8450 control_ops: None,
8451 launch_nonce: launch_nonce.map(str::to_string),
8452 })
8453 .unwrap();
8454 body["manifest"]["rich_ui"] = rich_ui;
8455 Frame::build(
8456 FrameType::Hello,
8457 control_flags(),
8458 0,
8459 0,
8460 corr,
8461 serde_json::to_vec(&body).unwrap(),
8462 )
8463 .unwrap()
8464 }
8465
8466 fn non_routable_hello_frame_with_control_ops(
8467 module_id: &str,
8468 corr: u64,
8469 control_ops: Option<Vec<String>>,
8470 ) -> Frame {
8471 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8472 manifest.provides.clear();
8473 let body = serde_json::to_vec(&ModuleHelloBody {
8474 manifest,
8475 protocol_ver: PROTOCOL_VERSION,
8476 control_ops,
8477 launch_nonce: None,
8478 })
8479 .unwrap();
8480 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8481 }
8482
8483 fn capability_grammar_hello_frame(
8484 capabilities: Value,
8485 runtime_computed: Option<Value>,
8486 corr: u64,
8487 ) -> Frame {
8488 let mut body = serde_json::to_value(ModuleHelloBody {
8489 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8490 protocol_ver: PROTOCOL_VERSION,
8491 control_ops: None,
8492 launch_nonce: None,
8493 })
8494 .expect("HELLO body serializes");
8495 body["manifest"]["capabilities"] = capabilities;
8496 if let Some(runtime_computed) = runtime_computed {
8497 body["runtime_computed"] = runtime_computed;
8498 }
8499 Frame::build(
8500 FrameType::Hello,
8501 control_flags(),
8502 0,
8503 0,
8504 corr,
8505 serde_json::to_vec(&body).expect("HELLO body reserializes"),
8506 )
8507 .expect("HELLO frame builds")
8508 }
8509
8510 fn channel_request(channel: u16, corr: u64) -> Frame {
8511 Frame::build(
8512 FrameType::Request,
8513 Flags::new(true, Priority::Interactive, false),
8514 channel,
8515 0,
8516 corr,
8517 b"opaque".to_vec(),
8518 )
8519 .unwrap()
8520 }
8521
8522 fn route_ctx(
8523 connection_id: ConnectionId,
8524 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8525 let (tx, rx) = mpsc::channel(8);
8526 (
8527 RouteCtx {
8528 connection_id,
8529 egress: FrameSink::new(tx),
8530 },
8531 rx,
8532 )
8533 }
8534
8535 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8536 serde_json::from_slice(&frame.body).unwrap()
8537 }
8538
8539 async fn hello_via_sink(
8544 handler: &ControlHandler,
8545 ctx: &RouteCtx,
8546 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8547 hello: Frame,
8548 ) -> Frame {
8549 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8550 assert!(
8551 replies.is_empty(),
8552 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8553 );
8554 let ack = rx
8555 .try_recv()
8556 .expect("HELLO_ACK is queued on the module sink")
8557 .frame;
8558 assert_eq!(ack.header.ty, FrameType::HelloAck);
8559 ack
8560 }
8561
8562 fn parse_error(frame: &Frame) -> Value {
8563 serde_json::from_slice(&frame.body).unwrap()
8564 }
8565
8566 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8567 serde_json::from_slice(&frame.body).unwrap()
8568 }
8569
8570 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8571 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8572 route_channel,
8573 route_epoch: 0,
8574 kind,
8575 })
8576 .unwrap();
8577 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8578 }
8579
8580 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8581 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8582 module_id: module_id.to_string(),
8583 })
8584 .unwrap();
8585 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8586 }
8587
8588 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8589 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8590 }
8591
8592 fn route_open_frame_with_consumer_capabilities(
8593 corr: u64,
8594 module_id: &str,
8595 project_root: ScratchDir,
8596 consumer_capabilities: Option<Vec<String>>,
8597 ) -> Frame {
8598 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8599 target: RouteTarget::ToolProvider {
8600 module_id: module_id.to_string(),
8601 },
8602 identity: BindIdentity::new(
8603 project_root.path().to_path_buf(),
8604 "unit".to_string(),
8605 "session".to_string(),
8606 ),
8607 consumer_identity: None,
8608 consumer_capabilities,
8609 role_versions: None,
8610 admission_facts: None,
8611 scope: None,
8612 })
8613 .unwrap();
8614 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8615 }
8616
8617 fn route_open_frame_with_role_versions(
8618 corr: u64,
8619 module_id: &str,
8620 project_root: ScratchDir,
8621 role_versions: Option<BTreeMap<String, String>>,
8622 ) -> Frame {
8623 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8624 target: RouteTarget::ToolProvider {
8625 module_id: module_id.to_string(),
8626 },
8627 identity: BindIdentity::new(
8628 project_root.path().to_path_buf(),
8629 "unit".to_string(),
8630 format!("session-{corr}"),
8631 ),
8632 consumer_identity: None,
8633 consumer_capabilities: None,
8634 role_versions,
8635 admission_facts: None,
8636 scope: None,
8637 })
8638 .unwrap();
8639 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8640 }
8641
8642 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8643 entries
8644 .iter()
8645 .map(|(role, version)| (role.to_string(), version.to_string()))
8646 .collect()
8647 }
8648
8649 fn route_open_frame_with_admission_facts(
8650 corr: u64,
8651 module_id: &str,
8652 project_root: ScratchDir,
8653 consumer_identity: Option<subc_control::ConsumerIdentity>,
8654 facts: Option<Value>,
8655 ) -> Frame {
8656 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8657 target: RouteTarget::ToolProvider {
8658 module_id: module_id.to_string(),
8659 },
8660 identity: BindIdentity::new(
8661 project_root.path().to_path_buf(),
8662 "unit".to_string(),
8663 format!("session-{corr}"),
8664 ),
8665 consumer_identity,
8666 consumer_capabilities: None,
8667 role_versions: None,
8668 admission_facts: facts,
8669 scope: None,
8670 })
8671 .unwrap();
8672 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8673 }
8674
8675 #[derive(Clone, Default)]
8676 struct EventCapture {
8677 events: Arc<Mutex<Vec<CapturedEvent>>>,
8678 }
8679
8680 #[derive(Clone, Debug)]
8681 struct CapturedEvent {
8682 target: String,
8683 level: tracing::Level,
8684 fields: BTreeMap<String, String>,
8685 }
8686
8687 impl EventCapture {
8688 fn events(&self) -> Vec<CapturedEvent> {
8689 self.events.lock().unwrap().clone()
8690 }
8691 }
8692
8693 impl<S> Layer<S> for EventCapture
8694 where
8695 S: Subscriber,
8696 {
8697 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8698 let mut visitor = EventFieldVisitor::default();
8699 event.record(&mut visitor);
8700 self.events.lock().unwrap().push(CapturedEvent {
8701 target: event.metadata().target().to_string(),
8702 level: *event.metadata().level(),
8703 fields: visitor.fields,
8704 });
8705 }
8706 }
8707
8708 #[derive(Default)]
8709 struct EventFieldVisitor {
8710 fields: BTreeMap<String, String>,
8711 }
8712
8713 impl Visit for EventFieldVisitor {
8714 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8715 self.fields
8716 .insert(field.name().to_string(), format!("{value:?}"));
8717 }
8718 }
8719
8720 fn health_response(corr: u64, status: HealthStatus) -> Frame {
8721 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8722 status,
8723 detail: Some("warming".to_string()),
8724 metrics: Some(json!({"queue_depth": 3})),
8725 })
8726 .unwrap();
8727 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8728 }
8729
8730 fn route_bind_ack(corr: u64) -> Frame {
8731 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8732 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8733 }
8734
8735 fn unique_project_root(label: &str) -> ScratchDir {
8736 ScratchDir::new(label)
8737 }
8738
8739 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8740 match parse_route_poll(frame) {
8741 ClientControlResponse::RoutePoll {
8742 status: None,
8743 live: Some(live),
8744 ..
8745 } => assert_eq!(live, expected_live),
8746 other => panic!("unexpected route.poll response: {other:?}"),
8747 }
8748 }
8749
8750 fn bind_liveness_route(
8751 registry: &Registry,
8752 forwarding: &ForwardingTable,
8753 module_id: &str,
8754 ) -> (RouteCtx, u16, u32) {
8755 let module_connection = ConnectionId::new(101);
8756 let client_connection = ConnectionId::new(202);
8757 let registration = registry
8758 .register_with_control_ops(
8759 manifest(module_id, PROTOCOL_VERSION),
8760 PROTOCOL_VERSION,
8761 module_connection,
8762 module_baseline_control_ops(),
8763 )
8764 .unwrap();
8765 let (module_tx, _module_rx) = mpsc::channel(8);
8766 let endpoint = forwarding
8767 .register_module_connection(
8768 module_connection,
8769 module_id.to_string(),
8770 PROTOCOL_VERSION,
8771 manifest_concurrency(®istration.manifest),
8772 FrameSink::new(module_tx),
8773 )
8774 .unwrap();
8775 let (client_ctx, _client_rx) = route_ctx(client_connection);
8776 let pending = forwarding
8777 .begin_route_bind_relay_for_test(
8778 client_connection,
8779 client_ctx.egress.clone(),
8780 1,
8781 module_id,
8782 )
8783 .unwrap();
8784 assert_eq!(pending.endpoint, endpoint);
8785 let route_channel = pending.client_channel;
8786 let route_epoch = pending.client_epoch;
8787 forwarding
8788 .complete_pending_relay(
8789 module_connection,
8790 pending.corr,
8791 RouteBindRelayOutcome::Accepted,
8792 )
8793 .unwrap();
8794 (client_ctx, route_channel, route_epoch)
8795 }
8796
8797 struct FakeProcessLiveness {
8798 live: Option<bool>,
8799 }
8800
8801 impl ModuleProcessLiveness for FakeProcessLiveness {
8802 fn process_live(&self, _module_id: &str) -> Option<bool> {
8803 self.live
8804 }
8805 }
8806
8807 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8808 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8809 {
8810 let registry = Arc::new(Registry::default());
8811 let supervisor_handle = SupervisorHandle::new();
8812 let supervisor = Supervisor::new_for_test(
8813 Arc::clone(®istry),
8814 RestartPolicy::new(1, Duration::from_millis(10)),
8815 )
8816 .with_handle(supervisor_handle.clone());
8817 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8818 let module = supervisor
8819 .spawn(ModuleSpec {
8820 module_id: "stderr-tail-wire".to_string(),
8821 program: fake_aft_stub_path(),
8822 args: Vec::new(),
8823 env: vec![
8824 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8825 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8826 ],
8827 reserved: false,
8828 reserved_prefixes: Vec::new(),
8829 protocol: ModuleProtocol::Subc,
8830 overlap: Default::default(),
8831 })
8832 .unwrap();
8833
8834 let deadline = Instant::now() + Duration::from_secs(5);
8835 loop {
8836 let tail = module.stderr_tail(None, None);
8837 if tail
8838 .entries
8839 .iter()
8840 .any(|entry| matches!(entry, TailEntry::ProcessStart))
8841 && tail.entries.iter().any(|entry| {
8842 matches!(
8843 entry,
8844 TailEntry::Line {
8845 truncated: true,
8846 ..
8847 }
8848 )
8849 })
8850 {
8851 break;
8852 }
8853 assert!(
8854 Instant::now() < deadline,
8855 "module did not produce a truncated line and restart boundary: {tail:?}"
8856 );
8857 sleep(Duration::from_millis(10)).await;
8858 }
8859
8860 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8861 let request = ClientControlRequest::SupervisorStderrTail {
8862 module_id: "stderr-tail-wire".to_string(),
8863 max_lines: None,
8864 max_bytes: None,
8865 };
8866 let frame = Frame::build(
8867 FrameType::Request,
8868 control_flags(),
8869 0,
8870 0,
8871 1,
8872 serde_json::to_vec(&request).unwrap(),
8873 )
8874 .unwrap();
8875 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8876 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8877 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8878 serde_json::from_slice(&responses[0].body).unwrap()
8879 else {
8880 panic!("expected supervisor.stderr_tail response");
8881 };
8882
8883 assert!(
8884 tail.entries
8885 .iter()
8886 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8887 "the control response lost the restart boundary"
8888 );
8889 let Some(StderrTailEntry::Line {
8890 text,
8891 truncated,
8892 at_ms,
8893 }) = tail.entries.iter().find(|entry| {
8894 matches!(
8895 entry,
8896 StderrTailEntry::Line {
8897 truncated: true,
8898 ..
8899 }
8900 )
8901 })
8902 else {
8903 panic!("the control response lost the truncated line");
8904 };
8905 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8906 assert!(*truncated);
8907 assert!(
8908 at_ms.is_some(),
8909 "the control response lost the line's capture time"
8910 );
8911 }
8912
8913 #[tokio::test(flavor = "current_thread")]
8918 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8919 let dir = ScratchDir::new("terminals-off-worker");
8920 let journal_path = dir.join("terminals.jsonl");
8921 let registry = Arc::new(Registry::default());
8922 let supervisor_handle = SupervisorHandle::new();
8923 let supervisor =
8924 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8925 .with_handle(supervisor_handle.clone())
8926 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8927 let module = supervisor
8928 .spawn(ModuleSpec {
8929 module_id: "terminal-off-worker".to_string(),
8930 program: fake_aft_stub_path(),
8931 args: Vec::new(),
8932 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8933 reserved: false,
8934 reserved_prefixes: Vec::new(),
8935 protocol: ModuleProtocol::Subc,
8936 overlap: Default::default(),
8937 })
8938 .unwrap();
8939 let deadline = Instant::now() + Duration::from_secs(5);
8940 while module.terminal_history().entries.len() != 2 {
8941 assert!(Instant::now() < deadline, "module did not record two exits");
8942 sleep(Duration::from_millis(10)).await;
8943 }
8944
8945 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8946 let handler =
8947 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
8948 let frame = Frame::build(
8949 FrameType::Request,
8950 control_flags(),
8951 0,
8952 0,
8953 1,
8954 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8955 module_id: "terminal-off-worker".to_string(),
8956 })
8957 .unwrap(),
8958 )
8959 .unwrap();
8960 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8961 let spawned_at = std::time::Instant::now();
8962 let read = tokio::spawn({
8963 let handler = Arc::clone(&handler);
8964 async move { handler.handle_control_frame(&ctx, frame).await }
8965 });
8966 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8969 .await
8970 .unwrap()
8971 .expect("the history read reached its pause");
8972 let elapsed = spawned_at.elapsed();
8973 assert!(
8974 elapsed < Duration::from_secs(2) && !read.is_finished(),
8975 "this task could not run while the history read was paused \
8976 (resumed after {elapsed:?}, read finished: {})",
8977 read.is_finished()
8978 );
8979
8980 drop(release);
8981 let responses = read.await.unwrap().unwrap();
8982 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8983 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8984 panic!("expected supervisor.terminals response");
8985 };
8986 assert_eq!(terminals.entries.len(), 2);
8987 assert_eq!(terminals.journal_skipped_lines, 0);
8988 assert_eq!(terminals.journal_read_errors, 0);
8989 }
8990
8991 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8992 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8993 let registry = Arc::new(Registry::default());
8994 let supervisor_handle = SupervisorHandle::new();
8995 let supervisor =
8996 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8997 .with_handle(supervisor_handle.clone());
8998 let module = supervisor
8999 .spawn(ModuleSpec {
9000 module_id: "terminal-golden".to_string(),
9001 program: fake_aft_stub_path(),
9002 args: Vec::new(),
9003 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9004 reserved: false,
9005 reserved_prefixes: Vec::new(),
9006 protocol: ModuleProtocol::Subc,
9007 overlap: Default::default(),
9008 })
9009 .unwrap();
9010
9011 let deadline = Instant::now() + Duration::from_secs(5);
9012 while module.terminal_history().entries.len() != 2 {
9013 assert!(
9014 Instant::now() < deadline,
9015 "module did not retain two terminal exits: {:?}",
9016 module.terminal_history()
9017 );
9018 sleep(Duration::from_millis(10)).await;
9019 }
9020
9021 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9022 let request = ClientControlRequest::SupervisorTerminals {
9023 module_id: "terminal-golden".to_string(),
9024 };
9025 let frame = Frame::build(
9026 FrameType::Request,
9027 control_flags(),
9028 0,
9029 0,
9030 1,
9031 serde_json::to_vec(&request).unwrap(),
9032 )
9033 .unwrap();
9034 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9035 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9036 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9037 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9038 panic!("expected supervisor.terminals response");
9039 };
9040 assert_eq!(terminals.entries.len(), 2);
9041 assert_eq!(terminals.dropped, 0);
9042
9043 let mut rendered = serde_json::to_value(response).unwrap();
9044 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9047 for (index, entry) in rendered["entries"]
9048 .as_array_mut()
9049 .expect("terminal response entries array")
9050 .iter_mut()
9051 .enumerate()
9052 {
9053 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9054 }
9055
9056 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9057 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9058 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9059 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9060 std::fs::write(&golden_path, &serialized).unwrap();
9061 }
9062 let expected: Value =
9063 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9064 assert_eq!(rendered, expected);
9065 }
9066
9067 #[test]
9068 fn hello_registers_manifest_and_returns_ack() {
9069 let registry = Arc::new(Registry::default());
9070 let handler = ControlHandler::new(Arc::clone(®istry));
9071 let conn = ConnectionId::new(1);
9072
9073 let responses = handler
9074 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9075 .unwrap();
9076
9077 assert_eq!(responses.len(), 1);
9078 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9079 assert_eq!(responses[0].header.channel, 0);
9080 assert_eq!(responses[0].header.corr, 7);
9081 let ack = parse_ack(&responses[0]);
9082 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9083 assert!(ack
9084 .subc_capabilities
9085 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9086 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9087 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9088 assert!(ack
9089 .subc_ops
9090 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9091 assert!(ack
9092 .subc_ops
9093 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9094
9095 let registration = registry.get_module("aft").unwrap().unwrap();
9096 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9097 assert_eq!(registration.state, ChannelState::Active);
9098 assert_eq!(registration.connection_id, conn);
9099 assert_eq!(registration.control_ops, module_baseline_control_ops());
9100 }
9101
9102 #[test]
9103 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9104 let invalid_identifiers = [
9105 ("case_change", "credentials-Provider/v1"),
9106 ("leading_zero", "credentials-provider/v01"),
9107 ("trailing_hyphen", "credentials-provider-/v1"),
9108 ("consecutive_hyphens", "credentials--provider/v1"),
9109 ("uppercase", "Credentials-provider/v1"),
9110 ("missing_v", "credentials-provider/1"),
9111 ("whitespace", "credentials provider/v1"),
9112 ("zero_version", "credentials-provider/v0"),
9113 ("out_of_range_version", "credentials-provider/v4294967296"),
9114 (
9115 "overlength_name",
9116 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9117 ),
9118 ];
9119 let mut cases = invalid_identifiers
9120 .into_iter()
9121 .map(|(name, identifier)| {
9122 (
9123 format!("identifier_{name}"),
9124 "capabilities.provides[0]".to_string(),
9125 identifier.to_string(),
9126 json!({ "provides": [identifier] }),
9127 None,
9128 )
9129 })
9130 .collect::<Vec<_>>();
9131 cases.extend([
9132 (
9133 "unknown_need".to_string(),
9134 "capabilities.requires[0].need".to_string(),
9135 "deferred".to_string(),
9136 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9137 None,
9138 ),
9139 (
9140 "duplicate_provides".to_string(),
9141 "capabilities.provides[1]".to_string(),
9142 "credentials-provider/v1".to_string(),
9143 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9144 None,
9145 ),
9146 (
9147 "duplicate_must_never_reach".to_string(),
9148 "capabilities.must_never_reach[1]".to_string(),
9149 "credentials-provider/v1".to_string(),
9150 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9151 None,
9152 ),
9153 (
9154 "duplicate_requires_same_need".to_string(),
9155 "capabilities.requires[1]".to_string(),
9156 "credentials-provider/v1".to_string(),
9157 json!({ "requires": [
9158 { "capability": "credentials-provider/v1", "need": "required" },
9159 { "capability": "credentials-provider/v1", "need": "required" }
9160 ] }),
9161 None,
9162 ),
9163 (
9164 "duplicate_requires_conflicting_need".to_string(),
9165 "capabilities.requires[1]".to_string(),
9166 "credentials-provider/v1".to_string(),
9167 json!({ "requires": [
9168 { "capability": "credentials-provider/v1", "need": "required" },
9169 { "capability": "credentials-provider/v1", "need": "optional" }
9170 ] }),
9171 None,
9172 ),
9173 (
9174 "capabilities_root_pointer".to_string(),
9175 "runtime_computed[0]".to_string(),
9176 "/capabilities".to_string(),
9177 json!({}),
9178 Some(json!(["/capabilities"])),
9179 ),
9180 (
9181 "capabilities_descendant_pointer".to_string(),
9182 "runtime_computed[0]".to_string(),
9183 "/capabilities/provides".to_string(),
9184 json!({}),
9185 Some(json!(["/capabilities/provides"])),
9186 ),
9187 (
9188 "malformed_pointer_without_leading_slash".to_string(),
9189 "runtime_computed[0]".to_string(),
9190 "capabilities".to_string(),
9191 json!({}),
9192 Some(json!(["capabilities"])),
9193 ),
9194 (
9195 "malformed_pointer_escape".to_string(),
9196 "runtime_computed[0]".to_string(),
9197 "/roles/~2/tools".to_string(),
9198 json!({}),
9199 Some(json!(["/roles/~2/tools"])),
9200 ),
9201 (
9202 "unknown_capabilities_field".to_string(),
9203 "capabilities.future".to_string(),
9204 "<array>".to_string(),
9205 json!({ "future": [] }),
9206 None,
9207 ),
9208 ]);
9209
9210 for (index, (name, field, value, capabilities, runtime_computed)) in
9211 cases.into_iter().enumerate()
9212 {
9213 let registry = Arc::new(Registry::default());
9214 let handler = ControlHandler::new(Arc::clone(®istry));
9215 let response = handler
9216 .handle_control(
9217 ConnectionId::new((index + 1) as u64),
9218 capability_grammar_hello_frame(
9219 capabilities,
9220 runtime_computed,
9221 index as u64 + 1,
9222 ),
9223 )
9224 .expect("invalid HELLO returns a refusal");
9225
9226 assert_eq!(response.len(), 1, "{name} must emit one refusal");
9227 let error = parse_error(&response[0]);
9228 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9229 let message = error["message"]
9230 .as_str()
9231 .expect("error message is a string");
9232 assert!(
9233 message.contains(&field),
9234 "{name}: field missing from {message}"
9235 );
9236 assert!(
9237 message.contains(&value),
9238 "{name}: value missing from {message}"
9239 );
9240 assert_eq!(
9241 registry
9242 .active_registration_count()
9243 .expect("registry reads"),
9244 0,
9245 "{name}: refused HELLO must not create a catalog entry"
9246 );
9247 }
9248 }
9249
9250 #[test]
9251 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9252 let registry = Arc::new(Registry::default());
9253 let handler = ControlHandler::new(Arc::clone(®istry));
9254 let capabilities = json!({
9255 "provides": ["credentials-provider/v1"],
9256 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9257 "must_never_reach": ["federation-transport/v1"]
9258 });
9259 let response = handler
9260 .handle_control(
9261 ConnectionId::new(99),
9262 capability_grammar_hello_frame(
9263 capabilities.clone(),
9264 Some(json!(["/roles/0/tools"])),
9265 99,
9266 ),
9267 )
9268 .expect("valid HELLO registers");
9269 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9270
9271 let request = Frame::build(
9272 FrameType::Request,
9273 control_flags(),
9274 0,
9275 0,
9276 100,
9277 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9278 .expect("catalog request serializes"),
9279 )
9280 .expect("catalog request frame builds");
9281 let response = handler
9282 .handle_catalog_list(request, None)
9283 .expect("catalog list succeeds");
9284 let ClientControlResponse::CatalogList { modules, .. } =
9285 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9286 else {
9287 panic!("catalog request must return catalog.list");
9288 };
9289 assert_eq!(modules.len(), 1);
9290 assert_eq!(
9291 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9292 capabilities
9293 );
9294 }
9295
9296 #[test]
9297 fn catalog_list_mirrors_management_operation_description() {
9298 let registry = Arc::new(Registry::default());
9299 let handler = ControlHandler::new(Arc::clone(®istry));
9300 let description = "List managed records and return their identifiers and metadata.";
9301 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9302 manifest.provides = vec![ProviderRole::ManagementSurface {
9303 operations: vec![ManagementOperation {
9304 name: "records.list".to_string(),
9305 kind: ManagementOperationKind::Query,
9306 description: Some(description.to_string()),
9307 }],
9308 config_schema: json!({"type": "object"}),
9309 observability: vec![ObservabilitySurface {
9310 name: "records.stats".to_string(),
9311 kind: ObservabilityKind::Snapshot,
9312 }],
9313 identity_scope: vec![IdentityScope::Project],
9314 concurrency: Concurrency::ModuleManaged,
9315 }];
9316 registry
9317 .register_with_control_ops(
9318 manifest,
9319 PROTOCOL_VERSION,
9320 ConnectionId::new(99),
9321 Vec::new(),
9322 )
9323 .expect("described management manifest registers");
9324
9325 let request = Frame::build(
9326 FrameType::Request,
9327 control_flags(),
9328 0,
9329 0,
9330 100,
9331 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9332 .expect("catalog request serializes"),
9333 )
9334 .expect("catalog request frame builds");
9335 let response = handler
9336 .handle_catalog_list(request, None)
9337 .expect("catalog list succeeds");
9338 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9339 assert_eq!(
9340 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9341 "catalog.list must preserve the declared operation description verbatim"
9342 );
9343 }
9344
9345 #[test]
9346 fn catalog_list_mirrors_registered_operation_delivery() {
9347 let registry = Arc::new(Registry::default());
9348 let handler = ControlHandler::new(Arc::clone(®istry));
9349 let delivery =
9350 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9351 let mut body = serde_json::to_value(ModuleHelloBody {
9352 manifest: manifest("watch-management", PROTOCOL_VERSION),
9353 protocol_ver: PROTOCOL_VERSION,
9354 control_ops: None,
9355 launch_nonce: None,
9356 })
9357 .unwrap();
9358 body["manifest"]["provides"] = json!([{
9359 "role": "management_surface",
9360 "operations": [
9361 {"name": "status", "kind": "query"},
9362 {"name": "future", "kind": "query"},
9363 {"name": "reset", "kind": "mutate"}
9364 ],
9365 "config_schema": {}, "observability": [], "identity_scope": []
9366 }]);
9367 body["manifest"]["operation_delivery"] = delivery.clone();
9368 let hello = Frame::build(
9369 FrameType::Hello,
9370 control_flags(),
9371 0,
9372 0,
9373 99,
9374 serde_json::to_vec(&body).unwrap(),
9375 )
9376 .unwrap();
9377 let response = handler
9378 .handle_control(ConnectionId::new(99), hello)
9379 .unwrap();
9380 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9381
9382 let (_, registrations) = registry.list_modules().unwrap();
9383 let registered = ®istrations[0].manifest;
9384 assert!(registered.is_watch_operation("status"));
9385 assert!(!registered.is_watch_operation("future"));
9386 assert!(!registered.is_watch_operation("reset"));
9387 assert!(!registered.is_watch_operation("missing"));
9388
9389 let request = Frame::build(
9390 FrameType::Request,
9391 control_flags(),
9392 0,
9393 0,
9394 100,
9395 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9396 )
9397 .unwrap();
9398 let response = handler.handle_catalog_list(request, None).unwrap();
9399 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9400 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9401 let ClientControlResponse::CatalogList { modules, .. } =
9402 serde_json::from_slice(&response[0].body).unwrap()
9403 else {
9404 panic!("expected catalog.list");
9405 };
9406 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9407 }
9408
9409 #[test]
9410 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
9411 let registry = Arc::new(Registry::default());
9412 let supervisor = SupervisorHandle::new();
9413 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9414 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9415 let descriptor = json!({
9416 "schema": "ckios.richui.manifest/1",
9417 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9418 "bytes": 4096,
9419 "chunks": 4
9420 });
9421 let connection = ConnectionId::new(201);
9422 let hello = handler
9423 .handle_control(
9424 connection,
9425 hello_frame_with_rich_ui(
9426 "vault",
9427 PROTOCOL_VERSION,
9428 201,
9429 Some("the-real-nonce"),
9430 descriptor.clone(),
9431 ),
9432 )
9433 .unwrap();
9434 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9435
9436 let request =
9437 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
9438 let provides = registry
9439 .get_module_by_connection(connection)
9440 .unwrap()
9441 .unwrap()
9442 .manifest
9443 .provides;
9444 handler
9445 .handle_catalog_update(connection, request, provides, None, Some(false))
9446 .unwrap();
9447
9448 let request = Frame::build(
9449 FrameType::Request,
9450 control_flags(),
9451 0,
9452 0,
9453 203,
9454 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9455 )
9456 .unwrap();
9457 let response = handler.handle_catalog_list(request, None).unwrap();
9458 let ClientControlResponse::CatalogList { modules, .. } =
9459 serde_json::from_slice(&response[0].body).unwrap()
9460 else {
9461 panic!("expected catalog.list");
9462 };
9463 assert_eq!(
9464 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
9465 descriptor
9466 );
9467 assert!(modules[0].nonce_verified);
9468 }
9469
9470 fn hello_with_role_declarations(declarations: Value) -> Frame {
9471 let mut body = serde_json::to_value(ModuleHelloBody {
9472 manifest: manifest("role-test", PROTOCOL_VERSION),
9473 protocol_ver: PROTOCOL_VERSION,
9474 control_ops: None,
9475 launch_nonce: None,
9476 })
9477 .unwrap();
9478 body["manifest"]["role_declarations"] = declarations;
9479 Frame::build(
9480 FrameType::Hello,
9481 control_flags(),
9482 0,
9483 0,
9484 301,
9485 serde_json::to_vec(&body).unwrap(),
9486 )
9487 .unwrap()
9488 }
9489
9490 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
9491 let request = Frame::build(
9492 FrameType::Request,
9493 control_flags(),
9494 0,
9495 0,
9496 302,
9497 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9498 )
9499 .unwrap();
9500 let response = handler.handle_catalog_list(request, None).unwrap();
9501 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
9502 let ClientControlResponse::CatalogList { mut modules, .. } =
9503 serde_json::from_slice(&response[0].body).unwrap()
9504 else {
9505 panic!("expected catalog.list");
9506 };
9507 (raw["modules"][0].clone(), modules.remove(0))
9508 }
9509
9510 #[test]
9511 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
9512 let registry = Arc::new(Registry::default());
9513 let handler = ControlHandler::new(Arc::clone(®istry));
9514 let declarations = json!({
9515 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
9516 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
9517 });
9518 let connection = ConnectionId::new(301);
9519 let response = handler
9520 .handle_control(
9521 connection,
9522 hello_with_role_declarations(declarations.clone()),
9523 )
9524 .unwrap();
9525 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9526 let registered = registry
9527 .get_module_by_connection(connection)
9528 .unwrap()
9529 .unwrap()
9530 .manifest;
9531 assert_eq!(
9535 serde_json::to_value(®istered.role_declarations).unwrap(),
9536 declarations
9537 );
9538 let (raw, entry) = role_declarations_catalog(&handler);
9539 assert_eq!(raw["role_declarations"], declarations);
9540 assert_eq!(entry.role_declarations, registered.role_declarations);
9541
9542 let request =
9543 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
9544 handler
9545 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
9546 .unwrap();
9547 let (raw, entry) = role_declarations_catalog(&handler);
9548 assert_eq!(raw["role_declarations"], declarations);
9549 assert_eq!(entry.role_declarations, registered.role_declarations);
9550 }
9551
9552 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
9553 let registry = Arc::new(Registry::default());
9554 let handler = ControlHandler::new(Arc::clone(®istry));
9555 let capture = EventCapture::default();
9556 let _subscriber =
9557 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9558 let response = handler
9559 .handle_control(
9560 ConnectionId::new(301),
9561 hello_with_role_declarations(declarations),
9562 )
9563 .unwrap();
9564 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9565 assert!(registry
9566 .get_module_by_connection(ConnectionId::new(301))
9567 .unwrap()
9568 .unwrap()
9569 .manifest
9570 .role_declarations
9571 .is_empty());
9572 let (raw, entry) = role_declarations_catalog(&handler);
9573 assert!(raw.get("role_declarations").is_none());
9574 assert!(entry.role_declarations.is_empty());
9575 let warnings: Vec<_> = capture
9576 .events()
9577 .into_iter()
9578 .filter(|event| {
9579 event.fields.get("message").map(String::as_str)
9580 == Some("dropping whole role_declarations map from HELLO")
9581 })
9582 .collect();
9583 assert_eq!(warnings.len(), 1);
9584 assert_eq!(warnings[0].level, tracing::Level::WARN);
9585 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
9586 assert_eq!(warnings[0].fields["reason"], reason);
9587 }
9588
9589 #[test]
9590 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
9591 for value in [
9592 json!(null),
9593 json!(false),
9594 json!(0),
9595 json!("opaque"),
9596 json!([]),
9597 ] {
9598 assert_hello_drops_role_declarations_with_warning(
9599 json!({"routing/v1": {}, "broken/v1": value}),
9600 "role_declarations values must be objects",
9601 );
9602 }
9603 assert_hello_drops_role_declarations_with_warning(
9604 json!([]),
9605 "role_declarations must be an object",
9606 );
9607 }
9608
9609 #[test]
9610 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
9611 assert_hello_drops_role_declarations_with_warning(
9612 json!({"routing/v1": {}, "bad key": {}}),
9613 "role_declarations key must use capability-name grammar",
9614 );
9615 }
9616
9617 #[test]
9618 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
9619 assert_hello_drops_role_declarations_with_warning(
9620 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
9621 "role_declarations exceeds 16384 compact JSON bytes",
9622 );
9623 }
9624
9625 #[test]
9626 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
9627 let registry = Arc::new(Registry::default());
9628 let handler = ControlHandler::new(registry.clone());
9629 let descriptor = json!({
9630 "schema": "ckios.richui.manifest/1",
9631 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9632 "bytes": 128,
9633 "chunks": 2
9634 });
9635 let malformed = [
9636 ("schema", json!("")),
9637 ("schema", json!("a".repeat(65))),
9638 ("schema", json!("Bad schema")),
9639 (
9640 "manifest_digest",
9641 json!(format!("sha256:{}", "A".repeat(64))),
9642 ),
9643 ("manifest_digest", json!("sha256:abc")),
9644 ("bytes", json!(0)),
9645 ("bytes", json!(16_777_217)),
9646 ("bytes", json!("128")),
9647 ("chunks", json!(0)),
9648 ("chunks", json!(1025)),
9649 ("chunks", json!("2")),
9650 ];
9651 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
9652 let mut invalid = descriptor.clone();
9653 invalid[field] = invalid_value;
9654 let response = handler
9655 .handle_control(
9656 ConnectionId::new(210 + index as u64),
9657 hello_frame_with_rich_ui(
9658 &format!("bad-rich-ui-{index}"),
9659 PROTOCOL_VERSION,
9660 210 + index as u64,
9661 None,
9662 invalid,
9663 ),
9664 )
9665 .unwrap();
9666 assert_eq!(response[0].header.ty, FrameType::Error);
9667 let error = parse_error(&response[0]);
9668 assert_eq!(error["code"], "invalid_manifest");
9669 assert!(error["message"]
9670 .as_str()
9671 .unwrap()
9672 .contains(&format!("rich_ui.{field}")));
9673 assert!(registry
9674 .get_module(&format!("bad-rich-ui-{index}"))
9675 .unwrap()
9676 .is_none());
9677 }
9678 }
9679
9680 #[test]
9681 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
9682 let registry = Arc::new(Registry::default());
9683 let handler = ControlHandler::new(Arc::clone(®istry));
9684 let connection = ConnectionId::new(219);
9685 registry
9686 .register_with_control_ops(
9687 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
9688 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
9689 "bad schema",
9690 format!("sha256:{}", "a".repeat(64)),
9691 128,
9692 2,
9693 )))
9694 .build(),
9695 PROTOCOL_VERSION,
9696 connection,
9697 Vec::new(),
9698 )
9699 .unwrap();
9700 let request =
9701 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
9702 let response = handler
9703 .handle_catalog_update(connection, request, Vec::new(), None, None)
9704 .unwrap();
9705 assert_eq!(response[0].header.ty, FrameType::Error);
9706 let error = parse_error(&response[0]);
9707 assert_eq!(error["code"], "invalid_manifest");
9708 assert!(error["message"]
9709 .as_str()
9710 .unwrap()
9711 .contains("rich_ui.schema"));
9712 }
9713
9714 #[test]
9715 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
9716 let registry = Arc::new(Registry::default());
9717 let supervisor = SupervisorHandle::new();
9718 supervisor.open_swap("provider", "candidate-nonce".to_string());
9719 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9720 let connection = ConnectionId::new(218);
9721 let hello = handler
9722 .handle_control(
9723 connection,
9724 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
9725 )
9726 .unwrap();
9727 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9728 assert!(registry.list_modules().unwrap().1.is_empty());
9729
9730 registry.promote_candidate("provider").unwrap().unwrap();
9731 let request = Frame::build(
9732 FrameType::Request,
9733 control_flags(),
9734 0,
9735 0,
9736 219,
9737 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9738 )
9739 .unwrap();
9740 let response = handler.handle_catalog_list(request, None).unwrap();
9741 let ClientControlResponse::CatalogList { modules, .. } =
9742 serde_json::from_slice(&response[0].body).unwrap()
9743 else {
9744 panic!("expected catalog.list");
9745 };
9746 assert_eq!(modules.len(), 1);
9747 assert!(modules[0].nonce_verified);
9748 }
9749
9750 #[test]
9751 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
9752 let registry = Arc::new(Registry::default());
9753 let handler = ControlHandler::new(Arc::clone(®istry));
9754 let hello = Frame::build(
9755 FrameType::Hello,
9756 control_flags(),
9757 0,
9758 0,
9759 220,
9760 serde_json::to_vec(&ModuleHelloBody {
9761 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
9762 protocol_ver: PROTOCOL_VERSION,
9763 control_ops: None,
9764 launch_nonce: Some("unverified".to_string()),
9765 })
9766 .unwrap(),
9767 )
9768 .unwrap();
9769 assert_eq!(
9770 handler
9771 .handle_control(ConnectionId::new(220), hello)
9772 .unwrap()[0]
9773 .header
9774 .ty,
9775 FrameType::HelloAck
9776 );
9777 let request = Frame::build(
9778 FrameType::Request,
9779 control_flags(),
9780 0,
9781 0,
9782 221,
9783 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9784 )
9785 .unwrap();
9786 let response = handler.handle_catalog_list(request, None).unwrap();
9787 let ClientControlResponse::CatalogList { modules, .. } =
9788 serde_json::from_slice(&response[0].body).unwrap()
9789 else {
9790 panic!("expected catalog.list");
9791 };
9792 assert_eq!(
9793 serde_json::to_vec(&modules[0]).unwrap(),
9794 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
9795 );
9796 assert!(!modules[0].nonce_verified);
9797 }
9798
9799 #[test]
9800 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
9801 let registry = Arc::new(Registry::default());
9802 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9803 [("vault".to_string(), true), ("squatter".to_string(), true)],
9804 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9805 );
9806 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
9807 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9808 provides: vec!["credentials-provider/v1".to_string()],
9809 requires: Vec::new(),
9810 must_never_reach: Vec::new(),
9811 });
9812 let frame = Frame::build(
9813 FrameType::Hello,
9814 control_flags(),
9815 0,
9816 0,
9817 77,
9818 serde_json::to_vec(&ModuleHelloBody {
9819 manifest: squatter,
9820 protocol_ver: PROTOCOL_VERSION,
9821 control_ops: None,
9822 launch_nonce: None,
9823 })
9824 .expect("HELLO serializes"),
9825 )
9826 .expect("HELLO frame builds");
9827 let response = handler
9828 .handle_control(ConnectionId::new(77), frame)
9829 .expect("reserved claim receives a typed refusal");
9830 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9831 assert_eq!(
9832 registry
9833 .active_registration_count()
9834 .expect("registry reads"),
9835 0,
9836 "a reserved capability refusal must not leave a catalog entry"
9837 );
9838 }
9839
9840 #[test]
9841 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
9842 for settlement in ["timeout", "inconclusive", "drop"] {
9843 let breakers = RouteBindBreakers::default();
9844 let RouteBindAdmission::Admitted {
9845 guard: mut old,
9846 probe: false,
9847 } = breakers.admit("prov")
9848 else {
9849 panic!("ordinary relay admitted")
9850 };
9851 let RouteBindAdmission::Admitted {
9852 guard: mut opener, ..
9853 } = breakers.admit("prov")
9854 else {
9855 panic!("second relay admitted")
9856 };
9857 assert!(
9858 !opener
9859 .record_timeout(1, Duration::ZERO)
9860 .unwrap()
9861 .reopened_after_probe
9862 );
9863 let RouteBindAdmission::Admitted {
9864 guard: mut probe,
9865 probe: true,
9866 } = breakers.admit("prov")
9867 else {
9868 panic!("one cooldown probe admitted")
9869 };
9870 match settlement {
9871 "timeout" => assert!(
9872 !old.record_timeout(1, Duration::ZERO)
9873 .unwrap()
9874 .reopened_after_probe
9875 ),
9876 "inconclusive" => old.record_inconclusive(),
9877 "drop" => drop(old),
9878 _ => unreachable!(),
9879 }
9880 assert!(
9881 matches!(
9882 breakers.admit("prov"),
9883 RouteBindAdmission::Refused {
9884 probe_in_flight: true,
9885 ..
9886 }
9887 ),
9888 "{settlement} of a pre-open relay cannot release the real probe"
9889 );
9890 assert!(
9891 probe
9892 .record_timeout(1, Duration::ZERO)
9893 .unwrap()
9894 .reopened_after_probe
9895 );
9896 assert!(matches!(
9897 breakers.admit("prov"),
9898 RouteBindAdmission::Admitted { probe: true, .. }
9899 ));
9900 }
9901 let breakers = RouteBindBreakers::default();
9902 let admit = || match breakers.admit("prov") {
9903 RouteBindAdmission::Admitted { guard, .. } => guard,
9904 _ => panic!("relay admitted"),
9905 };
9906 admit().record_timeout(1, Duration::ZERO);
9907 let mut old_probe = admit();
9908 breakers.reset_for_new_module_connection("prov");
9909 admit().record_timeout(1, Duration::ZERO);
9910 let _new_probe = admit();
9911 old_probe.record_inconclusive();
9912 assert!(matches!(
9913 breakers.admit("prov"),
9914 RouteBindAdmission::Refused {
9915 probe_in_flight: true,
9916 ..
9917 }
9918 ));
9919 }
9920
9921 #[tokio::test]
9922 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
9923 for candidate in [false, true] {
9924 let registry = Arc::new(Registry::default());
9925 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9926 [("vault".to_string(), true), ("squatter".to_string(), true)],
9927 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9928 );
9929 let conn = ConnectionId::new(77);
9930 let (ctx, mut rx) = route_ctx(conn);
9931 let initial = capability_manifest("squatter", &[], &[]);
9932 if candidate {
9933 registry
9934 .register_candidate_with_control_ops(
9935 initial.clone(),
9936 PROTOCOL_VERSION,
9937 conn,
9938 module_baseline_control_ops(),
9939 )
9940 .unwrap();
9941 handler
9942 .forwarding
9943 .register_candidate_module_connection(
9944 conn,
9945 "squatter".to_string(),
9946 PROTOCOL_VERSION,
9947 manifest_concurrency(&initial),
9948 ctx.egress.clone(),
9949 )
9950 .unwrap();
9951 } else {
9952 hello_via_sink(
9953 &handler,
9954 &ctx,
9955 &mut rx,
9956 hello_frame_with_manifest(initial.clone(), 1),
9957 )
9958 .await;
9959 }
9960 let response = handler
9961 .handle_control_frame(
9962 &ctx,
9963 catalog_update_with_capabilities_frame(
9964 2,
9965 capability_manifest("squatter", &["credentials-provider/v1"], &[])
9966 .capabilities
9967 .unwrap(),
9968 ),
9969 )
9970 .await
9971 .unwrap();
9972 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9973 assert_eq!(
9974 registry
9975 .get_module_by_connection(conn)
9976 .unwrap()
9977 .unwrap()
9978 .manifest,
9979 initial
9980 );
9981 }
9982 }
9983
9984 #[test]
9985 fn server_describe_surfaces_required_capability_verdict_fields() {
9986 let registry = Arc::new(Registry::default());
9987 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9988 [
9989 ("consumer".to_string(), true),
9990 ("provider".to_string(), false),
9991 ],
9992 BTreeMap::new(),
9993 );
9994 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9995 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9996 provides: Vec::new(),
9997 requires: vec![subc_protocol::manifest::CapabilityRequirement {
9998 capability: "credentials-provider/v1".to_string(),
9999 need: subc_protocol::manifest::CapabilityNeed::Required,
10000 }],
10001 must_never_reach: Vec::new(),
10002 });
10003 let hello = Frame::build(
10004 FrameType::Hello,
10005 control_flags(),
10006 0,
10007 0,
10008 78,
10009 serde_json::to_vec(&ModuleHelloBody {
10010 manifest: consumer,
10011 protocol_ver: PROTOCOL_VERSION,
10012 control_ops: None,
10013 launch_nonce: None,
10014 })
10015 .expect("HELLO serializes"),
10016 )
10017 .expect("HELLO frame builds");
10018 handler
10019 .handle_control(ConnectionId::new(78), hello)
10020 .expect("consumer registers");
10021 let describe = Frame::build(
10022 FrameType::Request,
10023 control_flags(),
10024 0,
10025 0,
10026 79,
10027 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
10028 .expect("request serializes"),
10029 )
10030 .expect("describe frame builds");
10031 let response = handler
10032 .handle_server_describe(describe)
10033 .expect("server.describe succeeds");
10034 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
10035 let requirement = &rendered["capability_requirements"][0];
10036 assert_eq!(requirement["consumer"], "consumer");
10037 assert_eq!(requirement["verdict"], "never_provided");
10038 assert_eq!(requirement["episode_seq"], 1);
10039 assert_eq!(requirement["config_satisfiable"], false);
10040 assert_eq!(requirement["runtime_available"], false);
10041 assert!(requirement["detail"]
10042 .as_str()
10043 .expect("detail string")
10044 .contains("credentials-provider/v1"));
10045 }
10046
10047 fn deregister_required_provider(
10053 daemon_shutdown_begun: bool,
10054 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
10055 const CAPABILITY: &str = "project-identity/v1";
10056 let registry = Arc::new(Registry::default());
10057 let daemon_shutdown = DaemonShutdownFlag::default();
10058 let handler = ControlHandler::new(Arc::clone(®istry))
10059 .with_capability_config(
10060 [
10061 ("consumer".to_string(), true),
10062 ("provider".to_string(), true),
10063 ],
10064 BTreeMap::new(),
10065 )
10066 .with_daemon_shutdown_flag(daemon_shutdown.clone());
10067 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10068 consumer.capabilities = Some(CapabilityDeclarations {
10069 provides: Vec::new(),
10070 requires: vec![subc_protocol::manifest::CapabilityRequirement {
10071 capability: CAPABILITY.to_string(),
10072 need: subc_protocol::manifest::CapabilityNeed::Required,
10073 }],
10074 must_never_reach: Vec::new(),
10075 });
10076 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
10077 let provider_conn = ConnectionId::new(182);
10078 for (conn, manifest, corr) in [
10079 (ConnectionId::new(181), consumer, 181),
10080 (provider_conn, provider, 182),
10081 ] {
10082 let hello = handler
10083 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
10084 .expect("module registers");
10085 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10086 }
10087 assert_eq!(
10088 handler.capability_evaluator.verdict("consumer", CAPABILITY),
10089 Some(CapabilityVerdict::Provided),
10090 "the registered provider must satisfy the requirement first"
10091 );
10092 if daemon_shutdown_begun {
10093 daemon_shutdown.set_for_test();
10094 }
10095
10096 let capture = EventCapture::default();
10097 let _subscriber =
10098 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10099 handler
10100 .cleanup_connection(provider_conn)
10101 .expect("provider deregisters");
10102 let events = capture
10103 .events()
10104 .into_iter()
10105 .filter(|event| {
10106 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
10107 })
10108 .collect();
10109 (
10110 events,
10111 handler.capability_evaluator.verdict("consumer", CAPABILITY),
10112 )
10113 }
10114
10115 #[test]
10116 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
10117 let (events, verdict) = deregister_required_provider(true);
10118 assert_eq!(
10119 verdict,
10120 Some(CapabilityVerdict::NeverProvided),
10121 "shutdown must not change the verdict route.open refusals read"
10122 );
10123 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10124 let event = &events[0];
10125 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
10126 assert_eq!(
10127 event.fields.get("cause").map(String::as_str),
10128 Some("\"daemon_shutdown\""),
10129 "{event:?}"
10130 );
10131 assert_eq!(
10132 event.fields.get("verdict").map(String::as_str),
10133 Some("\"never_provided\""),
10134 "{event:?}"
10135 );
10136 }
10137
10138 #[test]
10139 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
10140 let (events, verdict) = deregister_required_provider(false);
10141 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
10142 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10143 let event = &events[0];
10144 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
10145 assert_eq!(event.fields.get("cause"), None, "{event:?}");
10146 assert_eq!(
10147 event.fields.get("verdict").map(String::as_str),
10148 Some("\"never_provided\""),
10149 "{event:?}"
10150 );
10151 }
10152
10153 #[test]
10154 fn catalog_list_omits_capabilities_for_legacy_manifest() {
10155 let registry = Arc::new(Registry::default());
10156 let handler = ControlHandler::new(Arc::clone(®istry));
10157 let hello = handler
10158 .handle_control(
10159 ConnectionId::new(101),
10160 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
10161 )
10162 .expect("legacy HELLO registers");
10163 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10164
10165 let request = Frame::build(
10166 FrameType::Request,
10167 control_flags(),
10168 0,
10169 0,
10170 102,
10171 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10172 .expect("catalog request serializes"),
10173 )
10174 .expect("catalog request frame builds");
10175 let response = handler
10176 .handle_catalog_list(request, None)
10177 .expect("catalog list succeeds");
10178 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10179 assert!(
10180 body["modules"][0].get("capabilities").is_none(),
10181 "legacy manifest must retain an absent capabilities field on catalog.list"
10182 );
10183 }
10184
10185 #[test]
10186 fn hello_ack_omits_storage_when_no_storage_config() {
10187 let registry = Arc::new(Registry::default());
10188 let handler = ControlHandler::new(Arc::clone(®istry));
10189 let responses = handler
10190 .handle_control(
10191 ConnectionId::new(1),
10192 hello_frame("aft", PROTOCOL_VERSION, 7),
10193 )
10194 .unwrap();
10195 let ack = parse_ack(&responses[0]);
10196 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
10197 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
10198 }
10199
10200 #[tokio::test]
10201 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
10202 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
10203 let registry = Arc::new(Registry::default());
10204 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
10205 let responses = handler
10206 .handle_control(
10207 ConnectionId::new(1),
10208 hello_frame("aft", PROTOCOL_VERSION, 7),
10209 )
10210 .unwrap();
10211 let ack = parse_ack(&responses[0]);
10212 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
10213
10214 let described = handler
10215 .handle_control_frame(
10216 &route_ctx(ConnectionId::new(2)).0,
10217 Frame::build(
10218 FrameType::Request,
10219 control_flags(),
10220 0,
10221 0,
10222 9,
10223 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10224 )
10225 .unwrap(),
10226 )
10227 .await
10228 .unwrap();
10229 let ClientControlResponse::ServerDescribe { machine_id, .. } =
10230 serde_json::from_slice(&described[0].body).unwrap()
10231 else {
10232 panic!("server.describe answered with another shape");
10233 };
10234 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
10235 }
10236
10237 #[test]
10238 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
10239 let registry = Arc::new(Registry::default());
10242 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
10243 crate::daemon_config::StorageConfig::Sqlite {
10244 data_home: std::path::PathBuf::from("/data"),
10245 },
10246 ));
10247
10248 let responses = handler
10249 .handle_control(
10250 ConnectionId::new(1),
10251 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
10252 )
10253 .unwrap();
10254 let ack = parse_ack(&responses[0]);
10255 assert_eq!(
10256 ack.storage,
10257 Some(serde_json::json!({
10258 "module_id": "alfonso-routing",
10259 "storage_namespace": "default",
10260 "isolation": { "kind": "module" },
10261 "backend": {
10262 "backend": "sqlite",
10263 "path": "/data/cortexkit/alfonso-routing/store.db"
10264 }
10265 })),
10266 "the delivered descriptor is the module's own sqlite store path"
10267 );
10268 }
10269
10270 #[test]
10271 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
10272 let registry = Arc::new(Registry::default());
10273 let handler = ControlHandler::new(Arc::clone(®istry));
10274 let conn = ConnectionId::new(1);
10275 let responses = handler
10276 .handle_control(
10277 conn,
10278 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10279 )
10280 .unwrap();
10281 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
10282 let registration = registry.get_module("aft").unwrap().unwrap();
10283 assert_eq!(registration.control_ops, module_baseline_control_ops());
10284
10285 let frame =
10286 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
10287 assert!(handler
10288 .guard_module_control_op(&frame, "aft", "route.bind")
10289 .unwrap()
10290 .is_none());
10291 let error = handler
10292 .guard_module_control_op(&frame, "aft", "test.synthetic")
10293 .unwrap()
10294 .expect("synthetic ungranted op should be rejected");
10295 assert_eq!(error.header.ty, FrameType::Error);
10296 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
10297 }
10298
10299 #[test]
10300 fn hello_control_ops_some_adds_optional_grants() {
10301 let registry = Arc::new(Registry::default());
10302 let handler = ControlHandler::new(Arc::clone(®istry));
10303 handler
10304 .handle_control(
10305 ConnectionId::new(1),
10306 hello_frame_with_control_ops(
10307 "aft",
10308 PROTOCOL_VERSION,
10309 7,
10310 Some(vec![
10311 "future.synthetic".to_string(),
10312 "route.bind".to_string(),
10313 ]),
10314 ),
10315 )
10316 .unwrap();
10317 let registration = registry.get_module("aft").unwrap().unwrap();
10318 assert_eq!(
10319 registration.control_ops,
10320 vec![
10321 "route.bind".to_string(),
10322 "route.status".to_string(),
10323 "future.synthetic".to_string(),
10324 ]
10325 );
10326 let frame =
10327 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
10328 assert!(handler
10329 .guard_module_control_op(&frame, "aft", "future.synthetic")
10330 .unwrap()
10331 .is_none());
10332 }
10333
10334 #[tokio::test]
10335 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
10336 let registry = Arc::new(Registry::default());
10337 let forwarding = Arc::new(ForwardingTable::default());
10338 let handler =
10339 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10340 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
10341 hello_via_sink(
10342 &handler,
10343 &module_ctx,
10344 &mut module_rx,
10345 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10346 )
10347 .await;
10348
10349 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
10350 let responses = handler
10351 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
10352 .await
10353 .unwrap();
10354 assert_eq!(responses.len(), 1);
10355 assert_eq!(responses[0].header.ty, FrameType::Error);
10356 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
10357 assert!(module_rx.try_recv().is_err());
10358 }
10359
10360 #[tokio::test]
10361 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
10362 let registry = Arc::new(Registry::default());
10363 let forwarding = Arc::new(ForwardingTable::default());
10364 let handler =
10365 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10366 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
10367 hello_via_sink(
10368 &handler,
10369 &module_ctx,
10370 &mut module_rx,
10371 hello_frame_with_control_ops(
10372 "aft",
10373 PROTOCOL_VERSION,
10374 7,
10375 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10376 ),
10377 )
10378 .await;
10379
10380 let project_root = unique_project_root("demux");
10381 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
10382 let route_handler = handler.clone();
10383 let route_task = tokio::spawn(async move {
10384 route_handler
10385 .handle_control_frame(
10386 &route_client_ctx,
10387 route_open_frame(100, "aft", project_root),
10388 )
10389 .await
10390 .unwrap()
10391 });
10392 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10393 .await
10394 .unwrap()
10395 .unwrap();
10396 assert!(matches!(
10397 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
10398 ModuleControlRequest::RouteBind { .. }
10399 ));
10400
10401 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
10402 let health_handler = handler.clone();
10403 let health_task = tokio::spawn(async move {
10404 health_handler
10405 .handle_control_frame(
10406 &health_client_ctx,
10407 supervisor_health_probe_frame(101, "aft"),
10408 )
10409 .await
10410 .unwrap()
10411 });
10412 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10413 .await
10414 .unwrap()
10415 .unwrap();
10416 assert_eq!(
10417 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10418 ModuleControlRequest::HealthCheck {}
10419 );
10420
10421 handler
10422 .handle_control_frame(
10423 &module_ctx,
10424 health_response(health_frame.header.corr, HealthStatus::Degraded),
10425 )
10426 .await
10427 .unwrap();
10428 let health_response = health_task.await.unwrap();
10429 assert_eq!(health_response.len(), 1);
10430 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10431 ClientControlResponse::SupervisorHealthProbe {
10432 module_id,
10433 status,
10434 detail,
10435 metrics,
10436 } => {
10437 assert_eq!(module_id, "aft");
10438 assert_eq!(status, HealthStatus::Degraded);
10439 assert_eq!(detail.as_deref(), Some("warming"));
10440 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
10441 }
10442 other => panic!("unexpected health response: {other:?}"),
10443 }
10444
10445 handler
10446 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10447 .await
10448 .unwrap();
10449 let route_response = route_task.await.unwrap();
10450 assert!(route_response.is_empty());
10451 let published = route_client_rx.recv().await.unwrap();
10452 assert!(matches!(
10453 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10454 ClientControlResponse::RouteOpen { .. }
10455 ));
10456 }
10457
10458 async fn relay_route_open(
10463 handler: &ControlHandler,
10464 client_connection: ConnectionId,
10465 client_egress: &FrameSink,
10466 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10467 corr: u64,
10468 module_id: &str,
10469 project_root_label: &str,
10470 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
10471 let ctx = RouteCtx {
10472 connection_id: client_connection,
10473 egress: client_egress.clone(),
10474 };
10475 let handler = handler.clone();
10476 let project_root = unique_project_root(project_root_label);
10477 let module_id = module_id.to_string();
10478 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
10479 let task = tokio::spawn(async move {
10480 let _guard = tracing::dispatcher::set_default(&dispatch);
10481 handler
10482 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
10483 .await
10484 .unwrap()
10485 });
10486 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
10487 .await
10488 .expect("module receives the relayed route.bind")
10489 .expect("module egress is open");
10490 (task, bind.frame)
10491 }
10492
10493 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
10494 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
10495 ModuleControlRequest::RouteBind {
10496 route_channel,
10497 epoch,
10498 ..
10499 } => (route_channel, epoch),
10500 other => panic!("expected a route.bind request, got {other:?}"),
10501 }
10502 }
10503
10504 fn published_route(frame: &Frame) -> (u16, u32) {
10505 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
10506 ClientControlResponse::RouteOpen {
10507 route_channel,
10508 route_epoch,
10509 } => (route_channel, route_epoch),
10510 other => panic!("expected a route.open response, got {other:?}"),
10511 }
10512 }
10513
10514 #[tokio::test]
10534 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
10535 let registry = Arc::new(Registry::default());
10536 let forwarding = Arc::new(ForwardingTable::default());
10537 let handler =
10538 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10539
10540 let module_connection = ConnectionId::new(30);
10541 let (module_ctx, mut module_rx) = route_ctx(module_connection);
10542 hello_via_sink(
10543 &handler,
10544 &module_ctx,
10545 &mut module_rx,
10546 hello_frame("aft", PROTOCOL_VERSION, 7),
10547 )
10548 .await;
10549
10550 let dying_client = ConnectionId::new(31);
10551 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
10552
10553 let (first_task, first_bind) = relay_route_open(
10556 &handler,
10557 dying_client,
10558 &dying_ctx.egress,
10559 &mut module_rx,
10560 100,
10561 "aft",
10562 "closing-first",
10563 )
10564 .await;
10565 handler
10566 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
10567 .await
10568 .unwrap();
10569 assert!(first_task.await.unwrap().is_empty());
10570 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
10571
10572 let (second_task, second_bind) = relay_route_open(
10574 &handler,
10575 dying_client,
10576 &dying_ctx.egress,
10577 &mut module_rx,
10578 101,
10579 "aft",
10580 "closing-second",
10581 )
10582 .await;
10583 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
10584
10585 assert!(forwarding
10588 .escalate_client_delivery_failure(
10589 dying_client,
10590 first_channel,
10591 first_epoch,
10592 CloseReason::new(
10593 "module_to_client_delivery_failed",
10594 "client egress refused a module frame",
10595 ),
10596 crate::forwarding::UndeliveredFrame {
10597 module_id: None,
10598 sink: &dying_ctx.egress,
10599 },
10600 )
10601 .unwrap());
10602 assert!(!dying_ctx.egress.is_closed());
10603
10604 let ack = handler
10606 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
10607 .await;
10608 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10609 if module_loop_error.is_some() {
10610 handler.cleanup_connection(module_connection).unwrap();
10614 }
10615 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10619 .await
10620 .ok()
10621 .flatten();
10622
10623 assert!(
10625 registry
10626 .get_module_by_connection(module_connection)
10627 .unwrap()
10628 .is_some(),
10629 "one client's closing connection ended the shared module connection: \
10630 {module_loop_error:?}"
10631 );
10632 let cotenant = ConnectionId::new(32);
10634 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10635 let (cotenant_task, cotenant_bind) = relay_route_open(
10636 &handler,
10637 cotenant,
10638 &cotenant_ctx.egress,
10639 &mut module_rx,
10640 102,
10641 "aft",
10642 "closing-cotenant",
10643 )
10644 .await;
10645 handler
10646 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
10647 .await
10648 .unwrap();
10649 assert!(cotenant_task.await.unwrap().is_empty());
10650 let (cotenant_channel, cotenant_epoch) =
10651 published_route(&cotenant_rx.recv().await.unwrap());
10652 assert!(matches!(
10653 forwarding
10654 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10655 .unwrap(),
10656 DataRoute::Client(DataRouteState::Bound(_))
10657 ));
10658
10659 let goodbye = post_ack_module_frame
10662 .expect("module receives a GOODBYE for the abandoned route channel");
10663 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10664 assert_eq!(goodbye.header.channel, abandoned_channel);
10665 assert_eq!(goodbye.header.epoch, abandoned_epoch);
10666
10667 assert!(dying_rx.try_recv().is_err());
10671 let second_response = second_task.await.unwrap();
10672 assert_eq!(second_response.len(), 1);
10673 assert_eq!(
10674 parse_error(&second_response[0])["code"],
10675 "target_unavailable"
10676 );
10677 }
10678
10679 async fn catalog_snapshot(
10682 handler: &ControlHandler,
10683 client_ctx: &RouteCtx,
10684 corr: u64,
10685 ) -> (u64, Vec<(String, bool)>) {
10686 let body =
10687 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
10688 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
10689 let reply = handler
10690 .handle_control_frame(client_ctx, request)
10691 .await
10692 .unwrap()
10693 .pop()
10694 .unwrap();
10695 let ClientControlResponse::CatalogList {
10696 generation,
10697 modules,
10698 ..
10699 } = serde_json::from_slice(&reply.body).unwrap()
10700 else {
10701 panic!("not a catalog.list reply");
10702 };
10703 (
10704 generation,
10705 modules
10706 .into_iter()
10707 .map(|entry| (entry.module_id, entry.ready))
10708 .collect(),
10709 )
10710 }
10711
10712 fn hello_held_before_forwarding() -> (
10718 ControlHandler,
10719 Arc<std::sync::Barrier>,
10720 std::thread::JoinHandle<Vec<Frame>>,
10721 RouteCtx,
10722 mpsc::Receiver<crate::router::OutboundFrame>,
10723 ) {
10724 let hold = Arc::new(std::sync::Barrier::new(2));
10725 let handler = ControlHandler::with_forwarding(
10726 Arc::new(Registry::default()),
10727 Arc::new(ForwardingTable::default()),
10728 )
10729 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
10730 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10731 let hello = {
10732 let handler = handler.clone();
10733 let module_ctx = module_ctx.clone();
10734 std::thread::spawn(move || {
10735 handler
10736 .handle_hello(
10737 module_ctx.connection_id,
10738 Some(module_ctx.egress.clone()),
10739 hello_frame("aft", PROTOCOL_VERSION, 7),
10740 )
10741 .unwrap()
10742 })
10743 };
10744 (handler, hold, hello, module_ctx, module_rx)
10745 }
10746
10747 #[tokio::test]
10756 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
10757 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
10758 let client = ConnectionId::new(41);
10759 let (client_ctx, mut client_rx) = route_ctx(client);
10760
10761 hold.wait();
10762 assert!(handler.registry.get_module("aft").unwrap().is_some());
10764 assert!(!handler
10765 .forwarding
10766 .has_live_module_connection("aft")
10767 .unwrap());
10768 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
10769 assert!(
10770 listed.is_empty(),
10771 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
10772 );
10773 let refused = handler
10774 .handle_control_frame(
10775 &client_ctx,
10776 route_open_frame(301, "aft", unique_project_root("listed-window")),
10777 )
10778 .await
10779 .unwrap();
10780 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
10781 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
10782 let duplicate = handler
10783 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
10784 .await
10785 .unwrap();
10786 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10787
10788 hold.wait();
10789 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10790 let ack = module_rx.try_recv().unwrap().frame;
10791 assert_eq!(ack.header.ty, FrameType::HelloAck);
10792 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10793 assert_eq!(listed, vec![("aft".to_string(), true)]);
10794
10795 let (open_task, bind) = relay_route_open(
10796 &handler,
10797 client,
10798 &client_ctx.egress,
10799 &mut module_rx,
10800 303,
10801 "aft",
10802 "listed-after",
10803 )
10804 .await;
10805 assert!(handler
10806 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10807 .await
10808 .unwrap()
10809 .is_empty());
10810 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
10811 published_route(&client_rx.try_recv().unwrap().frame);
10812 }
10813
10814 #[tokio::test]
10821 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
10822 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
10823 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10824 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10825
10826 hold.wait();
10827 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
10828 assert!(listed.is_empty());
10829 assert_eq!(
10830 during, before,
10831 "the generation moved while the module was not listed"
10832 );
10833
10834 hold.wait();
10835 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10836 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10837 assert_eq!(listed, vec![("aft".to_string(), true)]);
10838 assert_eq!(
10839 after,
10840 before + 1,
10841 "the module appeared without exactly one generation bump"
10842 );
10843 }
10844
10845 #[tokio::test]
10850 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
10851 let registry = Arc::new(Registry::default());
10852 let handler = ControlHandler::with_forwarding(
10853 Arc::clone(®istry),
10854 Arc::new(ForwardingTable::default()),
10855 );
10856 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10857 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10858 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10859 drop(module_rx);
10860
10861 let replies = handler
10862 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
10863 .await
10864 .unwrap();
10865 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
10866 assert!(registry.get_module("aft").unwrap().is_none());
10867 assert!(registry
10868 .get_module_by_connection(module_ctx.connection_id)
10869 .unwrap()
10870 .is_none());
10871 assert_eq!(
10872 catalog_snapshot(&handler, &client_ctx, 301).await,
10873 (before, Vec::new())
10874 );
10875 }
10876
10877 async fn route_open_held_after_bind_send() -> (
10881 ControlHandler,
10882 Arc<tokio::sync::Barrier>,
10883 RouteCtx,
10884 mpsc::Receiver<crate::router::OutboundFrame>,
10885 ) {
10886 let hold = Arc::new(tokio::sync::Barrier::new(2));
10887 let handler = ControlHandler::with_forwarding(
10888 Arc::new(Registry::default()),
10889 Arc::new(ForwardingTable::default()),
10890 )
10891 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
10892 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10893 hello_via_sink(
10894 &handler,
10895 &module_ctx,
10896 &mut module_rx,
10897 hello_frame("aft", PROTOCOL_VERSION, 7),
10898 )
10899 .await;
10900 (handler, hold, module_ctx, module_rx)
10901 }
10902
10903 #[tokio::test]
10910 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
10911 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10912 let client = ConnectionId::new(41);
10913 let (client_ctx, mut client_rx) = route_ctx(client);
10914
10915 let (open_task, bind) = relay_route_open(
10916 &handler,
10917 client,
10918 &client_ctx.egress,
10919 &mut module_rx,
10920 200,
10921 "aft",
10922 "acked-before-mark",
10923 )
10924 .await;
10925 let (module_channel, module_epoch) = route_bind_channel(&bind);
10926 hold.wait().await;
10928
10929 assert!(handler
10931 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10932 .await
10933 .unwrap()
10934 .is_empty());
10935 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
10936
10937 hold.wait().await;
10938 assert!(
10939 open_task.await.unwrap().is_empty(),
10940 "route.open answers accepted"
10941 );
10942
10943 match module_rx.try_recv() {
10946 Err(mpsc::error::TryRecvError::Empty) => {}
10947 Ok(outbound) => panic!(
10948 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
10949 outbound.frame.header.ty,
10950 outbound.frame.header.channel,
10951 outbound.frame.header.epoch
10952 ),
10953 Err(err) => panic!("module egress closed: {err}"),
10954 }
10955
10956 let router = Router::with_control_handler(Arc::new(handler.clone()));
10959 let request = Frame::build(
10960 FrameType::Request,
10961 Flags::new(true, Priority::Interactive, false),
10962 client_channel,
10963 client_epoch,
10964 201,
10965 b"first request".to_vec(),
10966 )
10967 .unwrap();
10968 router
10969 .route_for_connection(&client_ctx, request)
10970 .await
10971 .unwrap();
10972 let delivered = module_rx.try_recv().unwrap().frame;
10973 assert_eq!(delivered.header.ty, FrameType::Request);
10974 assert_eq!(
10975 (delivered.header.channel, delivered.header.epoch),
10976 (module_channel, module_epoch)
10977 );
10978 assert_eq!(delivered.body, b"first request");
10979 }
10980
10981 #[tokio::test]
10987 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
10988 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10989 let client = ConnectionId::new(42);
10990 let (client_ctx, _client_rx) = route_ctx(client);
10991
10992 let (open_task, bind) = relay_route_open(
10993 &handler,
10994 client,
10995 &client_ctx.egress,
10996 &mut module_rx,
10997 300,
10998 "aft",
10999 "released-before-mark",
11000 )
11001 .await;
11002 let (module_channel, module_epoch) = route_bind_channel(&bind);
11003 hold.wait().await;
11004
11005 handler.cleanup_connection(client).unwrap();
11006 assert!(
11007 module_rx.try_recv().is_err(),
11008 "client teardown must not GOODBYE a bind route.open is still queueing"
11009 );
11010
11011 hold.wait().await;
11012 let response = open_task.await.unwrap();
11013 assert_eq!(response.len(), 1);
11014 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11015
11016 let goodbye = module_rx
11017 .try_recv()
11018 .expect("module is told to drop the binding the daemon abandoned")
11019 .frame;
11020 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11021 assert_eq!(
11022 (goodbye.header.channel, goodbye.header.epoch),
11023 (module_channel, module_epoch)
11024 );
11025 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11026 }
11027
11028 #[tokio::test]
11033 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
11034 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11035 let client = ConnectionId::new(43);
11036 let (client_ctx, client_rx) = route_ctx(client);
11037
11038 let (open_task, bind) = relay_route_open(
11039 &handler,
11040 client,
11041 &client_ctx.egress,
11042 &mut module_rx,
11043 400,
11044 "aft",
11045 "gone-client-before-mark",
11046 )
11047 .await;
11048 let (module_channel, module_epoch) = route_bind_channel(&bind);
11049 hold.wait().await;
11050
11051 drop(client_rx);
11052 assert!(client_ctx.egress.is_closed());
11053 handler
11054 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11055 .await
11056 .unwrap();
11057
11058 hold.wait().await;
11059 let response = open_task.await.unwrap();
11060 assert_eq!(response.len(), 1);
11061 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11062
11063 let goodbye = module_rx
11064 .try_recv()
11065 .expect("module is told to drop the binding it acked")
11066 .frame;
11067 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11068 assert_eq!(
11069 (goodbye.header.channel, goodbye.header.epoch),
11070 (module_channel, module_epoch)
11071 );
11072 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11073 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11074 }
11075
11076 #[test]
11083 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
11084 let handler = ControlHandler::default();
11085 let module_connection = ConnectionId::new(30);
11086 let client_connection = ConnectionId::new(31);
11087
11088 handler
11089 .refuse_to_end_module_connection_for_a_client(
11090 module_connection,
11091 77,
11092 ForwardingError::ConnectionClosing {
11093 connection_id: client_connection,
11094 },
11095 )
11096 .expect("a closing client must never end the module connection");
11097
11098 assert!(matches!(
11099 handler.refuse_to_end_module_connection_for_a_client(
11100 module_connection,
11101 78,
11102 ForwardingError::ConnectionClosing {
11103 connection_id: module_connection,
11104 },
11105 ),
11106 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
11107 connection_id
11108 })) if connection_id == module_connection
11109 ));
11110 assert!(matches!(
11111 handler.refuse_to_end_module_connection_for_a_client(
11112 module_connection,
11113 79,
11114 ForwardingError::Poisoned,
11115 ),
11116 Err(RouterError::Forwarding(ForwardingError::Poisoned))
11117 ));
11118 assert!(matches!(
11119 handler.refuse_to_end_module_connection_for_a_client(
11120 module_connection,
11121 80,
11122 ForwardingError::StaleModuleEndpoint,
11123 ),
11124 Err(RouterError::Forwarding(
11125 ForwardingError::StaleModuleEndpoint
11126 ))
11127 ));
11128 }
11129
11130 #[tokio::test]
11143 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
11144 let registry = Arc::new(Registry::default());
11145 let forwarding = Arc::new(ForwardingTable::default());
11146 let supervisor = SupervisorHandle::new();
11147 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11148 let handler =
11149 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11150 .with_supervisor(supervisor);
11151
11152 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11153 hello_via_sink(
11154 &handler,
11155 &target_ctx,
11156 &mut target_rx,
11157 hello_frame("target", PROTOCOL_VERSION, 1),
11158 )
11159 .await;
11160
11161 let wrong_nonce = handler
11165 .handle_control_frame(
11166 &route_ctx(ConnectionId::new(91)).0,
11167 route_open_frame_with_admission_facts(
11168 20,
11169 "target",
11170 unique_project_root("admission-facts"),
11171 Some(subc_control::ConsumerIdentity {
11172 module_id: "fed".to_string(),
11173 launch_nonce: "not-the-real-nonce".to_string(),
11174 }),
11175 None,
11176 ),
11177 )
11178 .await
11179 .unwrap();
11180 assert_eq!(
11181 parse_error(&wrong_nonce[0])["code"],
11182 "bad_consumer_identity",
11183 "a mismatched launch nonce must be refused, not stamped Reserved"
11184 );
11185
11186 let never_spawned = handler
11190 .handle_control_frame(
11191 &route_ctx(ConnectionId::new(92)).0,
11192 route_open_frame_with_admission_facts(
11193 21,
11194 "target",
11195 unique_project_root("admission-facts"),
11196 Some(subc_control::ConsumerIdentity {
11197 module_id: "never-spawned".to_string(),
11198 launch_nonce: "any-nonce".to_string(),
11199 }),
11200 None,
11201 ),
11202 )
11203 .await
11204 .unwrap();
11205 assert_eq!(
11206 parse_error(&never_spawned[0])["code"],
11207 "bad_consumer_identity",
11208 "an unspawned module_id must be refused rather than accepted for lack of a record"
11209 );
11210 }
11211
11212 #[tokio::test]
11235 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
11236 let registry = Arc::new(Registry::default());
11237 let forwarding = Arc::new(ForwardingTable::default());
11238 let supervisor = SupervisorHandle::new();
11239 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11240 let handler =
11241 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11242 .with_supervisor(supervisor);
11243
11244 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
11245 hello_via_sink(
11246 &handler,
11247 &target_ctx,
11248 &mut target_rx,
11249 hello_frame("target", PROTOCOL_VERSION, 1),
11250 )
11251 .await;
11252
11253 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
11254 let route_handler = handler.clone();
11255 let route_task = tokio::spawn(async move {
11256 route_handler
11257 .handle_control_frame(
11258 &client_ctx,
11259 route_open_frame_with_admission_facts(
11260 30,
11261 "target",
11262 unique_project_root("admission-facts"),
11263 Some(subc_control::ConsumerIdentity {
11264 module_id: "fed".to_string(),
11265 launch_nonce: "fed-nonce".to_string(),
11266 }),
11267 None,
11268 ),
11269 )
11270 .await
11271 .unwrap()
11272 });
11273
11274 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11281 .await
11282 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
11283 .expect("module control channel closed before route.bind");
11284 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11285 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11286 panic!("expected route.bind")
11287 };
11288 assert_eq!(
11289 principal,
11290 Some(Principal::Reserved {
11291 module_id: "fed".to_string()
11292 }),
11293 "a correctly attested consumer must be stamped Reserved for its own id"
11294 );
11295
11296 handler
11297 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11298 .await
11299 .unwrap();
11300 assert!(route_task.await.unwrap().is_empty());
11301 assert!(
11302 matches!(
11303 serde_json::from_slice::<ClientControlResponse>(
11304 &client_rx.recv().await.unwrap().body
11305 )
11306 .unwrap(),
11307 ClientControlResponse::RouteOpen { .. }
11308 ),
11309 "the route must actually open, not merely avoid an error"
11310 );
11311 }
11312
11313 #[tokio::test(start_paused = true)]
11314 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
11315 let registry = Arc::new(Registry::default());
11316 let forwarding = Arc::new(ForwardingTable::default());
11317 let supervisor = SupervisorHandle::new();
11318 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11319 let handler =
11320 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11321 .with_supervisor(supervisor);
11322
11323 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
11324 hello_via_sink(
11325 &handler,
11326 &target_ctx,
11327 &mut target_rx,
11328 hello_frame("target", PROTOCOL_VERSION, 1),
11329 )
11330 .await;
11331
11332 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
11333 let direct_handler = handler.clone();
11334 let direct_open = tokio::spawn(async move {
11335 direct_handler
11336 .handle_control_frame(
11337 &direct_ctx,
11338 route_open_frame(2, "target", unique_project_root("route-census-direct")),
11339 )
11340 .await
11341 .unwrap()
11342 });
11343 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11344 .await
11345 .expect("no direct route.bind within 5s")
11346 .expect("target control channel closed before direct route.bind");
11347 handler
11348 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
11349 .await
11350 .unwrap();
11351 assert!(direct_open.await.unwrap().is_empty());
11352 let _ = direct_rx.recv().await.unwrap();
11353
11354 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
11355 let reserved_handler = handler.clone();
11356 let reserved_open = tokio::spawn(async move {
11357 reserved_handler
11358 .handle_control_frame(
11359 &reserved_ctx,
11360 route_open_frame_with_admission_facts(
11361 3,
11362 "target",
11363 unique_project_root("admission-facts"),
11364 Some(ConsumerIdentity {
11365 module_id: "fed".to_string(),
11366 launch_nonce: "fed-nonce".to_string(),
11367 }),
11368 None,
11369 ),
11370 )
11371 .await
11372 .unwrap()
11373 });
11374 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11375 .await
11376 .expect("no reserved route.bind within 5s")
11377 .expect("target control channel closed before reserved route.bind");
11378 handler
11379 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
11380 .await
11381 .unwrap();
11382 assert!(reserved_open.await.unwrap().is_empty());
11383 let _ = reserved_rx.recv().await.unwrap();
11384
11385 forwarding
11386 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
11387 .unwrap();
11388 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
11389 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
11390 module_id: Some("target".to_string()),
11391 })
11392 .unwrap();
11393 let census_frame =
11394 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
11395 let response = handler
11396 .handle_control_frame(&census_ctx, census_frame)
11397 .await
11398 .unwrap()
11399 .pop()
11400 .unwrap();
11401 let actual: Value = serde_json::from_slice(&response.body).unwrap();
11402 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
11403 assert!(matches!(
11404 decoded,
11405 ClientControlResponse::SupervisorRoutes { .. }
11406 ));
11407 let routes = actual["modules"][0]["routes"].as_array().unwrap();
11408 assert_eq!(routes.len(), 2);
11409 assert!(routes.iter().all(|route| route["draining"] == true));
11410 assert!(
11413 routes.iter().all(|route| route["drain_reason"] == "reload"),
11414 "draining routes must name the drain's reason: {routes:?}"
11415 );
11416 assert!(routes.iter().any(|route| {
11417 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
11418 }));
11419 assert!(routes.iter().any(|route| {
11420 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
11421 }));
11422
11423 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11424 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
11425 if std::env::var_os("UPDATE_GOLDEN").is_some() {
11426 std::fs::write(
11427 &golden_path,
11428 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
11429 )
11430 .unwrap();
11431 }
11432 let expected: Value =
11433 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
11434 assert_eq!(actual, expected);
11435 }
11436
11437 async fn query_live_roots(
11438 handler: &ControlHandler,
11439 module_ctx: &RouteCtx,
11440 ) -> ModuleControlResponseToModule {
11441 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
11442 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
11443 let response = handler
11444 .handle_control_frame(module_ctx, frame)
11445 .await
11446 .unwrap()
11447 .pop()
11448 .unwrap();
11449 serde_json::from_slice(&response.body).unwrap()
11450 }
11451
11452 #[tokio::test(start_paused = true)]
11453 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
11454 let registry = Arc::new(Registry::default());
11455 let forwarding = Arc::new(ForwardingTable::default());
11456 let handler = ControlHandler::with_forwarding(registry, forwarding);
11457 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
11458 hello_via_sink(
11459 &handler,
11460 &target_ctx,
11461 &mut target_rx,
11462 hello_frame("target", PROTOCOL_VERSION, 1),
11463 )
11464 .await;
11465 let root = unique_project_root("live-roots-known");
11466 let path = ProjectRootId::from_path_allowing_missing(root.path())
11467 .unwrap()
11468 .as_path()
11469 .to_path_buf();
11470 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
11471 let open_handler = handler.clone();
11472 let opened = tokio::spawn(async move {
11473 open_handler
11474 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
11475 .await
11476 .unwrap()
11477 });
11478 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11479 .await
11480 .unwrap()
11481 .unwrap();
11482 handler
11483 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
11484 .await
11485 .unwrap();
11486 assert!(opened.await.unwrap().is_empty());
11487 let _ = client_rx.recv().await.unwrap();
11488
11489 let root = unique_project_root("live-roots-pending");
11490 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
11491 .unwrap()
11492 .as_path()
11493 .to_path_buf();
11494 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
11495 let open_handler = handler.clone();
11496 let pending = tokio::spawn(async move {
11497 open_handler
11498 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
11499 .await
11500 .unwrap()
11501 });
11502 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11503 .await
11504 .unwrap()
11505 .unwrap();
11506 let actual = query_live_roots(&handler, &target_ctx).await;
11507 let ModuleControlResponseToModule::LiveRoots {
11508 roots,
11509 unknown_root_bindings,
11510 total_bindings,
11511 } = actual
11512 else {
11513 panic!("expected live roots")
11514 };
11515 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
11516 assert_eq!(unknown_root_bindings, 0);
11517 assert_eq!(
11518 roots.len(),
11519 2,
11520 "root-known arm must retain each canonical root"
11521 );
11522 assert_eq!(
11523 total_bindings,
11524 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11525 );
11526 let counts = roots
11527 .iter()
11528 .map(|root| (root.project_root.clone(), root.bound, root.pending))
11529 .collect::<Vec<_>>();
11530 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
11531 expected.sort_by(|a, b| a.0.cmp(&b.0));
11532 assert_eq!(
11533 counts, expected,
11534 "roots must sort by path and count pending separately"
11535 );
11536 handler
11537 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
11538 .await
11539 .unwrap();
11540 assert!(pending.await.unwrap().is_empty());
11541 }
11542
11543 #[tokio::test(start_paused = true)]
11544 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
11545 let forwarding = Arc::new(ForwardingTable::default());
11546 let handler =
11547 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11548 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
11549 hello_via_sink(
11550 &handler,
11551 &target_ctx,
11552 &mut target_rx,
11553 hello_frame("target", PROTOCOL_VERSION, 1),
11554 )
11555 .await;
11556 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
11557 let pending = forwarding
11558 .begin_route_bind_relay_for_test(
11559 client_ctx.connection_id,
11560 client_ctx.egress.clone(),
11561 2,
11562 "target",
11563 )
11564 .unwrap();
11565 forwarding
11566 .complete_pending_relay(
11567 target_ctx.connection_id,
11568 pending.corr,
11569 RouteBindRelayOutcome::Accepted,
11570 )
11571 .unwrap();
11572 let actual = query_live_roots(&handler, &target_ctx).await;
11573 let ModuleControlResponseToModule::LiveRoots {
11574 roots,
11575 unknown_root_bindings,
11576 total_bindings,
11577 } = actual
11578 else {
11579 panic!("expected live roots")
11580 };
11581 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
11582 assert_eq!(
11583 unknown_root_bindings, 1,
11584 "unknown-root arm must not read as no bindings"
11585 );
11586 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
11587 assert_eq!(
11588 total_bindings,
11589 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11590 );
11591 }
11592
11593 #[tokio::test(start_paused = true)]
11601 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
11602 let forwarding = Arc::new(ForwardingTable::default());
11603 let handler =
11604 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11605 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
11606 let replies = handler
11607 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
11608 .await
11609 .unwrap();
11610 let queued_by_hello = module_rx.len();
11611
11612 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
11613 let open_handler = handler.clone();
11614 let open = tokio::spawn(async move {
11615 open_handler
11616 .handle_control_frame(
11617 &client_ctx,
11618 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
11619 )
11620 .await
11621 .unwrap()
11622 });
11623 let mut spins = 0;
11625 while module_rx.len() == queued_by_hello {
11626 spins += 1;
11627 assert!(spins < 10_000, "route.open never queued a route.bind");
11628 tokio::task::yield_now().await;
11629 }
11630
11631 for reply in replies {
11633 module_ctx.egress.send(reply).await.unwrap();
11634 }
11635
11636 let first = module_rx.recv().await.unwrap().frame;
11637 assert_eq!(
11638 first.header.ty,
11639 FrameType::HelloAck,
11640 "the first frame a registering module reads must be its HELLO_ACK"
11641 );
11642 assert_eq!(first.header.corr, 7);
11643 let second = module_rx.recv().await.unwrap().frame;
11644 assert_eq!(second.header.ty, FrameType::Request);
11645 assert!(
11646 matches!(
11647 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
11648 ModuleControlRequest::RouteBind { .. }
11649 ),
11650 "the route.bind follows the ack"
11651 );
11652 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
11653
11654 handler
11655 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
11656 .await
11657 .unwrap();
11658 assert!(open.await.unwrap().is_empty());
11659 let _ = client_rx.recv().await.unwrap();
11660 }
11661
11662 #[tokio::test(start_paused = true)]
11663 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
11664 let handler = ControlHandler::with_forwarding(
11665 Arc::new(Registry::default()),
11666 Arc::new(ForwardingTable::default()),
11667 );
11668 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
11669 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
11670 hello_via_sink(
11671 &handler,
11672 &first_ctx,
11673 &mut first_rx,
11674 hello_frame("first", PROTOCOL_VERSION, 1),
11675 )
11676 .await;
11677 hello_via_sink(
11678 &handler,
11679 &second_ctx,
11680 &mut second_rx,
11681 hello_frame("second", PROTOCOL_VERSION, 2),
11682 )
11683 .await;
11684 let root = unique_project_root("second-only");
11685 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
11686 let cloned = handler.clone();
11687 let open = tokio::spawn(async move {
11688 cloned
11689 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
11690 .await
11691 .unwrap()
11692 });
11693 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
11694 .await
11695 .unwrap()
11696 .unwrap();
11697 let first = query_live_roots(&handler, &first_ctx).await;
11698 let second = query_live_roots(&handler, &second_ctx).await;
11699 assert!(
11700 matches!(
11701 first,
11702 ModuleControlResponseToModule::LiveRoots {
11703 total_bindings: 0,
11704 ..
11705 }
11706 ),
11707 "cross-module scope must not expose another module's roots"
11708 );
11709 assert!(
11710 matches!(
11711 second,
11712 ModuleControlResponseToModule::LiveRoots {
11713 total_bindings: 1,
11714 ..
11715 }
11716 ),
11717 "second module must see its pending route"
11718 );
11719 handler
11720 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
11721 .await
11722 .unwrap();
11723 assert!(open.await.unwrap().is_empty());
11724 }
11725
11726 #[tokio::test(start_paused = true)]
11727 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
11728 let handler = ControlHandler::with_forwarding(
11729 Arc::new(Registry::default()),
11730 Arc::new(ForwardingTable::default()),
11731 );
11732 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
11733 hello_via_sink(
11734 &handler,
11735 &target_ctx,
11736 &mut target_rx,
11737 hello_frame("target", PROTOCOL_VERSION, 1),
11738 )
11739 .await;
11740 let actual = query_live_roots(&handler, &target_ctx).await;
11741 let ModuleControlResponseToModule::LiveRoots {
11742 roots,
11743 unknown_root_bindings,
11744 total_bindings,
11745 } = actual
11746 else {
11747 panic!("expected live roots")
11748 };
11749 assert!(roots.is_empty());
11750 assert_eq!(unknown_root_bindings, 0);
11751 assert_eq!(total_bindings, 0);
11752 assert_eq!(
11753 total_bindings,
11754 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11755 );
11756 }
11757
11758 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
11765 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11766 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
11767 let text = std::fs::read_to_string(&path)
11768 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
11769 let vectors: Vec<(String, Value)> = text
11770 .lines()
11771 .filter(|line| !line.trim().is_empty())
11772 .map(|line| {
11773 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
11774 let id = entry["corpus_id"]
11775 .as_str()
11776 .expect("every vector carries a corpus_id")
11777 .to_string();
11778 (id, entry["package"].clone())
11779 })
11780 .collect();
11781 assert_eq!(
11785 vectors.len(),
11786 3,
11787 "vendored fed corpus changed size; re-sync from subc-federation"
11788 );
11789
11790 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
11805 let richest = vectors
11806 .iter()
11807 .filter_map(|(_, package)| package.as_object())
11808 .map(|object| {
11809 object
11810 .keys()
11811 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
11812 .count()
11813 })
11814 .max()
11815 .unwrap_or(0);
11816 assert!(
11817 richest >= 2,
11818 "vendored corpus no longer carries a package with unmodelled fields, \
11819 so the relay test can no longer distinguish a verbatim relay from a lossy one"
11820 );
11821
11822 vectors
11823 }
11824
11825 #[tokio::test]
11833 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
11834 for (corpus_id, package) in fed_admission_facts_vectors() {
11835 let registry = Arc::new(Registry::default());
11836 let forwarding = Arc::new(ForwardingTable::default());
11837 let supervisor = SupervisorHandle::new();
11838 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11839 let handler =
11840 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11841 .with_supervisor(supervisor)
11842 .with_admission_facts_config(
11843 Some("fed".to_string()),
11844 Some(vec!["target".to_string()]),
11845 );
11846
11847 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11848 hello_via_sink(
11849 &handler,
11850 &target_ctx,
11851 &mut target_rx,
11852 hello_frame("target", PROTOCOL_VERSION, 1),
11853 )
11854 .await;
11855
11856 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
11857 let route_handler = handler.clone();
11858 let expected = package.clone();
11859 let route_task = tokio::spawn(async move {
11860 route_handler
11861 .handle_control_frame(
11862 &client_ctx,
11863 route_open_frame_with_admission_facts(
11864 20,
11865 "target",
11866 unique_project_root("admission-facts"),
11867 Some(subc_control::ConsumerIdentity {
11868 module_id: "fed".to_string(),
11869 launch_nonce: "fed-nonce".to_string(),
11870 }),
11871 Some(package),
11872 ),
11873 )
11874 .await
11875 .unwrap()
11876 });
11877
11878 let bind_frame = target_rx.recv().await.unwrap();
11879 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11880 let ModuleControlRequest::RouteBind {
11881 admission_facts, ..
11882 } = bind
11883 else {
11884 panic!("{corpus_id}: expected route.bind")
11885 };
11886 assert_eq!(
11887 admission_facts,
11888 Some(expected),
11889 "{corpus_id}: relay must not add, drop or reshape any field"
11890 );
11891
11892 handler
11893 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11894 .await
11895 .unwrap();
11896 route_task.await.unwrap();
11897 }
11898 }
11899
11900 #[tokio::test]
11901 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
11902 let registry = Arc::new(Registry::default());
11903 let forwarding = Arc::new(ForwardingTable::default());
11904 let supervisor = SupervisorHandle::new();
11905 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11906 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
11907 let handler =
11908 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11909 .with_supervisor(supervisor)
11910 .with_admission_facts_config(
11911 Some("fed".to_string()),
11912 Some(vec!["target".to_string()]),
11913 );
11914
11915 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
11916 hello_via_sink(
11917 &handler,
11918 &target_ctx,
11919 &mut target_rx,
11920 hello_frame("target", PROTOCOL_VERSION, 1),
11921 )
11922 .await;
11923 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
11924 hello_via_sink(
11925 &handler,
11926 &other_ctx,
11927 &mut other_rx,
11928 hello_frame("other", PROTOCOL_VERSION, 2),
11929 )
11930 .await;
11931
11932 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
11933 let expected_facts = facts.clone();
11934 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
11935 let route_handler = handler.clone();
11936 let route_task = tokio::spawn(async move {
11937 route_handler
11938 .handle_control_frame(
11939 &client_ctx,
11940 route_open_frame_with_admission_facts(
11941 10,
11942 "target",
11943 unique_project_root("admission-facts"),
11944 Some(subc_control::ConsumerIdentity {
11945 module_id: "fed".to_string(),
11946 launch_nonce: "fed-nonce".to_string(),
11947 }),
11948 Some(facts.clone()),
11949 ),
11950 )
11951 .await
11952 .unwrap()
11953 });
11954 let bind_frame = target_rx.recv().await.unwrap();
11955 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11956 let ModuleControlRequest::RouteBind {
11957 admission_facts, ..
11958 } = bind
11959 else {
11960 panic!("expected route.bind")
11961 };
11962 assert_eq!(admission_facts, Some(expected_facts));
11963 handler
11964 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11965 .await
11966 .unwrap();
11967 assert!(route_task.await.unwrap().is_empty());
11968 assert!(matches!(
11969 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
11970 .unwrap(),
11971 ClientControlResponse::RouteOpen { .. }
11972 ));
11973
11974 let direct = handler
11975 .handle_control_frame(
11976 &route_ctx(ConnectionId::new(73)).0,
11977 route_open_frame_with_admission_facts(
11978 11,
11979 "target",
11980 unique_project_root("admission-facts"),
11981 None,
11982 Some(json!({"x": 1})),
11983 ),
11984 )
11985 .await
11986 .unwrap();
11987 assert_eq!(
11988 parse_error(&direct[0])["code"],
11989 "admission_facts_not_permitted"
11990 );
11991
11992 let different_reserved = handler
11993 .handle_control_frame(
11994 &route_ctx(ConnectionId::new(77)).0,
11995 route_open_frame_with_admission_facts(
11996 15,
11997 "target",
11998 unique_project_root("admission-facts"),
11999 Some(subc_control::ConsumerIdentity {
12000 module_id: "other".to_string(),
12001 launch_nonce: "other-nonce".to_string(),
12002 }),
12003 Some(json!({"x": 1})),
12004 ),
12005 )
12006 .await
12007 .unwrap();
12008 assert_eq!(
12009 parse_error(&different_reserved[0])["code"],
12010 "admission_facts_not_permitted"
12011 );
12012
12013 let other_target = handler
12014 .handle_control_frame(
12015 &route_ctx(ConnectionId::new(74)).0,
12016 route_open_frame_with_admission_facts(
12017 12,
12018 "other",
12019 unique_project_root("admission-facts"),
12020 Some(subc_control::ConsumerIdentity {
12021 module_id: "fed".to_string(),
12022 launch_nonce: "fed-nonce".to_string(),
12023 }),
12024 Some(json!({"x": 1})),
12025 ),
12026 )
12027 .await
12028 .unwrap();
12029 assert_eq!(
12030 parse_error(&other_target[0])["code"],
12031 "admission_facts_target_not_allowed"
12032 );
12033
12034 let nonexistent = handler
12035 .handle_control_frame(
12036 &route_ctx(ConnectionId::new(75)).0,
12037 route_open_frame_with_admission_facts(
12038 13,
12039 "missing",
12040 unique_project_root("admission-facts"),
12041 None,
12042 Some(json!({"x": 1})),
12043 ),
12044 )
12045 .await
12046 .unwrap();
12047 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
12048
12049 let described = handler
12050 .handle_control_frame(
12051 &route_ctx(ConnectionId::new(76)).0,
12052 Frame::build(
12053 FrameType::Request,
12054 control_flags(),
12055 0,
12056 0,
12057 14,
12058 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12059 )
12060 .unwrap(),
12061 )
12062 .await
12063 .unwrap();
12064 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12065 serde_json::from_slice(&described[0].body).unwrap()
12066 else {
12067 panic!("expected server.describe response")
12068 };
12069 assert!(capabilities
12070 .iter()
12071 .any(|cap| cap == "admission_facts_relay_v1"));
12072 }
12073
12074 #[tokio::test]
12075 async fn admission_facts_without_configured_carrier_are_rejected() {
12076 let registry = Arc::new(Registry::default());
12077 let forwarding = Arc::new(ForwardingTable::default());
12078 let handler = ControlHandler::with_forwarding(registry, forwarding);
12079 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
12080 hello_via_sink(
12081 &handler,
12082 &target_ctx,
12083 &mut target_rx,
12084 hello_frame("target", PROTOCOL_VERSION, 1),
12085 )
12086 .await;
12087
12088 let responses = handler
12089 .handle_control_frame(
12090 &route_ctx(ConnectionId::new(79)).0,
12091 route_open_frame_with_admission_facts(
12092 16,
12093 "target",
12094 unique_project_root("admission-facts"),
12095 None,
12096 Some(json!({"x": 1})),
12097 ),
12098 )
12099 .await
12100 .unwrap();
12101 assert_eq!(
12102 parse_error(&responses[0])["code"],
12103 "admission_facts_not_permitted"
12104 );
12105 }
12106
12107 #[tokio::test]
12108 async fn route_open_relays_consumer_capabilities_verbatim() {
12109 let registry = Arc::new(Registry::default());
12110 let forwarding = Arc::new(ForwardingTable::default());
12111 let handler =
12112 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12113 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
12114 hello_via_sink(
12115 &handler,
12116 &module_ctx,
12117 &mut module_rx,
12118 hello_frame("aft", PROTOCOL_VERSION, 7),
12119 )
12120 .await;
12121
12122 let expected = vec!["elicitation".to_string(), "roots".to_string()];
12123 let expected_for_request = expected.clone();
12124 let project_root = unique_project_root("consumer-capabilities-present");
12125 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
12126 let route_handler = handler.clone();
12127 let route_task = tokio::spawn(async move {
12128 route_handler
12129 .handle_control_frame(
12130 &client_ctx,
12131 route_open_frame_with_consumer_capabilities(
12132 401,
12133 "aft",
12134 project_root,
12135 Some(expected_for_request),
12136 ),
12137 )
12138 .await
12139 .unwrap()
12140 });
12141 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12142 .await
12143 .unwrap()
12144 .unwrap();
12145 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12146 let ModuleControlRequest::RouteBind {
12147 consumer_capabilities,
12148 ..
12149 } = bind
12150 else {
12151 panic!("expected route.bind request, got {bind:?}");
12152 };
12153 assert_eq!(consumer_capabilities, Some(expected.clone()));
12154
12155 handler
12156 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12157 .await
12158 .unwrap();
12159 let route_response = route_task.await.unwrap();
12160 assert!(route_response.is_empty());
12161 let published = client_rx.recv().await.unwrap();
12162 assert!(matches!(
12163 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12164 ClientControlResponse::RouteOpen { .. }
12165 ));
12166 }
12167
12168 #[tokio::test]
12169 async fn route_open_without_consumer_capabilities_relays_none() {
12170 let registry = Arc::new(Registry::default());
12171 let forwarding = Arc::new(ForwardingTable::default());
12172 let handler =
12173 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12174 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
12175 hello_via_sink(
12176 &handler,
12177 &module_ctx,
12178 &mut module_rx,
12179 hello_frame("aft", PROTOCOL_VERSION, 7),
12180 )
12181 .await;
12182
12183 let project_root = unique_project_root("consumer-capabilities-absent");
12184 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
12185 let route_handler = handler.clone();
12186 let route_task = tokio::spawn(async move {
12187 route_handler
12188 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
12189 .await
12190 .unwrap()
12191 });
12192 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12193 .await
12194 .unwrap()
12195 .unwrap();
12196 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12197 let ModuleControlRequest::RouteBind {
12198 consumer_capabilities,
12199 ..
12200 } = bind
12201 else {
12202 panic!("expected route.bind request, got {bind:?}");
12203 };
12204 assert_eq!(consumer_capabilities, None);
12205
12206 handler
12207 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12208 .await
12209 .unwrap();
12210 let route_response = route_task.await.unwrap();
12211 assert!(route_response.is_empty());
12212 let published = client_rx.recv().await.unwrap();
12213 assert!(matches!(
12214 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12215 ClientControlResponse::RouteOpen { .. }
12216 ));
12217 }
12218
12219 async fn bind_role_versions_for(
12223 sent: Option<BTreeMap<String, String>>,
12224 connection: u64,
12225 ) -> Option<BTreeMap<String, String>> {
12226 let registry = Arc::new(Registry::default());
12227 let forwarding = Arc::new(ForwardingTable::default());
12228 let handler =
12229 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12230 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
12231 hello_via_sink(
12232 &handler,
12233 &module_ctx,
12234 &mut module_rx,
12235 hello_frame("aft", PROTOCOL_VERSION, 7),
12236 )
12237 .await;
12238 let project_root = unique_project_root("role-versions");
12239 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
12240 let route_handler = handler.clone();
12241 let route_task = tokio::spawn(async move {
12242 route_handler
12243 .handle_control_frame(
12244 &client_ctx,
12245 route_open_frame_with_role_versions(403, "aft", project_root, sent),
12246 )
12247 .await
12248 .unwrap()
12249 });
12250 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12251 .await
12252 .expect("a well-formed route.open reaches the module as a bind")
12253 .unwrap();
12254 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12255 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
12256 panic!("expected route.bind request, got {bind:?}");
12257 };
12258 handler
12259 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12260 .await
12261 .unwrap();
12262 assert!(route_task.await.unwrap().is_empty());
12263 let published = client_rx.recv().await.unwrap();
12264 assert!(matches!(
12265 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12266 ClientControlResponse::RouteOpen { .. }
12267 ));
12268 role_versions
12269 }
12270
12271 #[tokio::test]
12272 async fn route_open_relays_role_versions_verbatim() {
12273 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
12274 assert_eq!(
12275 bind_role_versions_for(Some(sent.clone()), 141).await,
12276 Some(sent)
12277 );
12278 }
12279
12280 #[tokio::test]
12283 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
12284 assert_eq!(bind_role_versions_for(None, 143).await, None);
12285 assert_eq!(
12286 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
12287 None
12288 );
12289 }
12290
12291 #[tokio::test]
12292 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
12293 let registry = Arc::new(Registry::default());
12294 let forwarding = Arc::new(ForwardingTable::default());
12295 let handler =
12296 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12297 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
12298 hello_via_sink(
12299 &handler,
12300 &module_ctx,
12301 &mut module_rx,
12302 hello_frame("aft", PROTOCOL_VERSION, 7),
12303 )
12304 .await;
12305
12306 let nine: BTreeMap<String, String> = (0..9)
12307 .map(|index| (format!("role-{index}"), "v1".to_string()))
12308 .collect();
12309 for (label, malformed) in [
12310 (
12311 "invalid role name",
12312 role_versions(&[("Tool_Provider", "v1")]),
12313 ),
12314 ("invalid version", role_versions(&[("tool-provider", "v0")])),
12315 ("nine entries", nine),
12316 ] {
12317 let responses = tokio::time::timeout(
12320 Duration::from_secs(1),
12321 handler.handle_control_frame(
12322 &route_ctx(ConnectionId::new(148)).0,
12323 route_open_frame_with_role_versions(
12324 404,
12325 "aft",
12326 unique_project_root("role-versions-malformed"),
12327 Some(malformed),
12328 ),
12329 ),
12330 )
12331 .await
12332 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
12333 .unwrap();
12334 assert_eq!(responses.len(), 1, "{label}");
12335 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
12336 let error = parse_error(&responses[0]);
12337 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
12338 assert_eq!(
12339 error["detail"]["field"], "role_versions",
12340 "{label}: {error}"
12341 );
12342 assert!(
12343 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
12344 "{label}: a malformed declaration is terminal"
12345 );
12346 assert!(
12347 module_rx.try_recv().is_err(),
12348 "{label}: the module must never see a bind"
12349 );
12350 }
12351 }
12352
12353 #[tokio::test]
12357 async fn route_role_versions_capability_is_advertised() {
12358 let handler = ControlHandler::new(Arc::new(Registry::default()));
12359 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
12360 let ack = hello_via_sink(
12361 &handler,
12362 &ctx,
12363 &mut rx,
12364 hello_frame("m", PROTOCOL_VERSION, 1),
12365 )
12366 .await;
12367 let ack = parse_ack(&ack);
12368 assert!(
12369 ack.subc_capabilities
12370 .iter()
12371 .any(|c| c == "route-role-versions/v1"),
12372 "{:?}",
12373 ack.subc_capabilities
12374 );
12375
12376 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12377 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12378 let reply = handler
12379 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
12380 .await
12381 .unwrap()
12382 .pop()
12383 .unwrap();
12384 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12385 serde_json::from_slice(&reply.body).unwrap()
12386 else {
12387 panic!("not a server.describe reply");
12388 };
12389 assert!(
12390 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
12391 "{capabilities:?}"
12392 );
12393 }
12394
12395 #[tokio::test]
12399 async fn catalog_events_capability_is_advertised() {
12400 let handler = ControlHandler::new(Arc::new(Registry::default()));
12401 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12402 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12403 let reply = handler
12404 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
12405 .await
12406 .unwrap()
12407 .pop()
12408 .unwrap();
12409 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12410 serde_json::from_slice(&reply.body).unwrap()
12411 else {
12412 panic!("not a server.describe reply");
12413 };
12414 assert!(
12415 capabilities
12416 .iter()
12417 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
12418 "{capabilities:?}"
12419 );
12420 }
12421
12422 #[tokio::test]
12423 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
12424 let handler = ControlHandler::new(Arc::new(Registry::default()));
12425 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
12426 let ack = hello_via_sink(
12427 &handler,
12428 &module,
12429 &mut module_rx,
12430 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
12431 )
12432 .await;
12433 let ack = parse_ack(&ack);
12434 assert!(
12435 ack.subc_capabilities
12436 .iter()
12437 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12438 "{:?}",
12439 ack.subc_capabilities
12440 );
12441 assert!(ack
12442 .subc_capabilities
12443 .iter()
12444 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12445
12446 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
12447 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12448 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12449 let reply = handler
12450 .handle_control_frame(&client, frame)
12451 .await
12452 .unwrap()
12453 .pop()
12454 .unwrap();
12455 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12456 serde_json::from_slice(&reply.body).unwrap()
12457 else {
12458 panic!("not a server.describe reply");
12459 };
12460 assert!(
12461 capabilities
12462 .iter()
12463 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12464 "{capabilities:?}"
12465 );
12466 assert!(capabilities
12467 .iter()
12468 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12469 }
12470
12471 #[tokio::test]
12472 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
12473 let handler = ControlHandler::new(Arc::new(Registry::default()));
12474 let (module, mut rx) = route_ctx(ConnectionId::new(156));
12475 let ack = hello_via_sink(
12476 &handler,
12477 &module,
12478 &mut rx,
12479 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
12480 )
12481 .await;
12482 assert!(parse_ack(&ack)
12483 .subc_capabilities
12484 .iter()
12485 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12486 assert!(parse_ack(&ack)
12487 .subc_capabilities
12488 .iter()
12489 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12490 let (client, _rx) = route_ctx(ConnectionId::new(157));
12491 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12492 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12493 let reply = handler
12494 .handle_control_frame(&client, frame)
12495 .await
12496 .unwrap()
12497 .pop()
12498 .unwrap();
12499 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12500 serde_json::from_slice(&reply.body).unwrap()
12501 else {
12502 panic!("server.describe reply")
12503 };
12504 assert!(capabilities
12505 .iter()
12506 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12507 assert!(capabilities
12508 .iter()
12509 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12510 }
12511
12512 #[tokio::test]
12513 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
12514 let handler = ControlHandler::new(Arc::new(Registry::default()));
12515 let (module, mut rx) = route_ctx(ConnectionId::new(158));
12516 let ack = hello_via_sink(
12517 &handler,
12518 &module,
12519 &mut rx,
12520 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
12521 )
12522 .await;
12523 assert!(parse_ack(&ack)
12524 .subc_capabilities
12525 .iter()
12526 .any(|cap| cap == "scope-workdir/v1"));
12527 let (client, _rx) = route_ctx(ConnectionId::new(159));
12528 let frame = Frame::build(
12529 FrameType::Request,
12530 control_flags(),
12531 0,
12532 0,
12533 5,
12534 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12535 )
12536 .unwrap();
12537 let reply = handler
12538 .handle_control_frame(&client, frame)
12539 .await
12540 .unwrap()
12541 .remove(0);
12542 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12543 serde_json::from_slice(&reply.body).unwrap()
12544 else {
12545 panic!("server.describe reply")
12546 };
12547 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
12548 }
12549
12550 #[tokio::test]
12551 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
12552 let handler = ControlHandler::new(Arc::new(Registry::default()));
12553 let (module, mut rx) = route_ctx(ConnectionId::new(154));
12554 let ack = hello_via_sink(
12555 &handler,
12556 &module,
12557 &mut rx,
12558 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
12559 )
12560 .await;
12561 assert!(parse_ack(&ack)
12562 .subc_capabilities
12563 .iter()
12564 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12565 let (client, _rx) = route_ctx(ConnectionId::new(155));
12566 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12567 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12568 let reply = handler
12569 .handle_control_frame(&client, frame)
12570 .await
12571 .unwrap()
12572 .pop()
12573 .unwrap();
12574 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12575 serde_json::from_slice(&reply.body).unwrap()
12576 else {
12577 panic!("not a server.describe reply")
12578 };
12579 assert!(capabilities
12580 .iter()
12581 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12582 }
12583
12584 #[tokio::test]
12585 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
12586 let registry = Arc::new(Registry::default());
12587 let forwarding = Arc::new(ForwardingTable::default());
12588 let handler =
12589 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12590 .with_health_probe_timeout(Duration::from_secs(5));
12591 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
12592 hello_via_sink(
12593 &handler,
12594 &module_ctx,
12595 &mut module_rx,
12596 non_routable_hello_frame_with_control_ops(
12597 "mcp",
12598 300,
12599 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12600 ),
12601 )
12602 .await;
12603 assert!(registry
12604 .get_module("mcp")
12605 .unwrap()
12606 .unwrap()
12607 .manifest
12608 .provides
12609 .is_empty());
12610
12611 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
12612 let route_response = handler
12613 .handle_control_frame(
12614 &route_client_ctx,
12615 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
12616 )
12617 .await
12618 .unwrap();
12619 assert_eq!(route_response[0].header.ty, FrameType::Error);
12620 assert_eq!(
12621 parse_error(&route_response[0])["code"],
12622 "target_unavailable"
12623 );
12624 assert!(parse_error(&route_response[0])["message"]
12625 .as_str()
12626 .unwrap()
12627 .contains("does not provide the requested target"));
12628 assert!(module_rx.try_recv().is_err());
12629
12630 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
12631 let health_handler = handler.clone();
12632 let health_task = tokio::spawn(async move {
12633 health_handler
12634 .handle_control_frame(
12635 &health_client_ctx,
12636 supervisor_health_probe_frame(302, "mcp"),
12637 )
12638 .await
12639 .unwrap()
12640 });
12641 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12642 .await
12643 .unwrap()
12644 .unwrap();
12645 assert_eq!(
12646 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
12647 ModuleControlRequest::HealthCheck {}
12648 );
12649 handler
12650 .handle_control_frame(
12651 &module_ctx,
12652 health_response(health_frame.header.corr, HealthStatus::Ok),
12653 )
12654 .await
12655 .unwrap();
12656 let health_response = health_task.await.unwrap();
12657 assert_eq!(health_response[0].header.ty, FrameType::Response);
12658 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
12659 ClientControlResponse::SupervisorHealthProbe {
12660 module_id, status, ..
12661 } => {
12662 assert_eq!(module_id, "mcp");
12663 assert_eq!(status, HealthStatus::Ok);
12664 }
12665 other => panic!("unexpected health response: {other:?}"),
12666 }
12667
12668 forwarding
12673 .cleanup_connection(module_ctx.connection_id)
12674 .unwrap();
12675 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
12676 let cleanup_response = tokio::time::timeout(
12677 Duration::from_millis(200),
12678 handler.handle_control_frame(
12679 &cleanup_probe_ctx,
12680 supervisor_health_probe_frame(303, "mcp"),
12681 ),
12682 )
12683 .await
12684 .expect("probe should fail immediately when the control lane is gone")
12685 .unwrap();
12686 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
12687 assert_eq!(
12688 parse_error(&cleanup_response[0])["code"],
12689 "target_unavailable"
12690 );
12691 assert!(parse_error(&cleanup_response[0])["message"]
12692 .as_str()
12693 .unwrap()
12694 .contains("no module connection"));
12695
12696 handler
12697 .cleanup_connection(module_ctx.connection_id)
12698 .unwrap();
12699 }
12700
12701 #[tokio::test]
12702 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
12703 let registry = Arc::new(Registry::default());
12704 let supervisor_handle = SupervisorHandle::new();
12705 let supervisor =
12706 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12707 .with_handle(supervisor_handle.clone())
12708 .with_connection_file_path(
12709 std::env::temp_dir()
12710 .join(format!("subc-route-open-warming-{}", std::process::id())),
12711 );
12712 let module = supervisor
12713 .supervise_configured(
12714 ModuleSpec {
12715 module_id: "warming".to_string(),
12716 program: fake_aft_stub_path(),
12717 args: Vec::new(),
12718 env: Vec::new(),
12719 reserved: false,
12720 reserved_prefixes: Vec::new(),
12721 protocol: ModuleProtocol::Subc,
12722 overlap: Default::default(),
12723 },
12724 true,
12725 )
12726 .unwrap();
12727 assert_eq!(module.state().unwrap(), ModuleState::Running);
12728
12729 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12730 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
12731 let response = handler
12732 .handle_control_frame(
12733 &ctx,
12734 route_open_frame(304, "warming", unique_project_root("warming")),
12735 )
12736 .await
12737 .unwrap();
12738 module.stop().await.unwrap();
12739
12740 assert_eq!(response[0].header.ty, FrameType::Error);
12741 let error = parse_error(&response[0]);
12742 assert_eq!(error["code"], "module_warming");
12743 assert!(error["message"]
12744 .as_str()
12745 .unwrap()
12746 .contains("state=running, enabled=true, live=false"));
12747 }
12748
12749 #[test]
12750 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
12751 let handler = ControlHandler::new(Arc::new(Registry::default()));
12752 let capture = EventCapture::default();
12753 let _subscriber =
12754 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12755 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
12756 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
12757 let pending = (0..limit).collect::<Vec<_>>();
12758 let response = handler
12759 .route_open_capacity_refusal(
12760 &ctx,
12761 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
12762 "busy",
12763 pending.len(),
12764 limit,
12765 )
12766 .unwrap();
12767 assert_eq!(parse_error(&response)["code"], "target_unavailable");
12768 let event = capture
12769 .events()
12770 .into_iter()
12771 .find(|event| {
12772 event.target == "control"
12773 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
12774 })
12775 .expect("connection admission refusal event");
12776 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12777 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12778 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
12781 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
12782 }
12783
12784 #[test]
12785 fn route_open_target_cap_logs_admission_reason_and_capacity() {
12786 let handler = ControlHandler::new(Arc::new(Registry::default()));
12787 let capture = EventCapture::default();
12788 let _subscriber =
12789 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12790 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
12791 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
12792 let guards = (0..limit)
12793 .map(|_| {
12794 handler
12795 .route_bind_concurrency
12796 .try_admit("busy", limit)
12797 .unwrap()
12798 })
12799 .collect::<Vec<_>>();
12800 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
12801 Err(in_flight) => in_flight,
12802 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
12803 };
12804 let response = handler
12805 .route_open_target_capacity_refusal(
12806 &ctx,
12807 &RouteOpenCaller::unknown(),
12808 &route_open_frame(397, "busy", unique_project_root("target-cap")),
12809 "busy",
12810 in_flight,
12811 )
12812 .unwrap();
12813 assert_eq!(parse_error(&response)["code"], "target_unavailable");
12814 let event = capture
12815 .events()
12816 .into_iter()
12817 .find(|event| {
12818 event.target == "control"
12819 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
12820 })
12821 .expect("target admission refusal event");
12822 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12823 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12824 drop(guards);
12825 }
12826
12827 #[cfg(unix)]
12828 #[tokio::test(start_paused = true)]
12829 async fn supervisor_list_shows_a_failed_modules_next_retry() {
12830 let registry = Arc::new(Registry::default());
12831 let handle = SupervisorHandle::new();
12832 let supervisor =
12833 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
12834 .with_handle(handle.clone());
12835 let module = supervisor
12836 .spawn(crate::ModuleSpec {
12837 module_id: "retry-list".into(),
12838 program: PathBuf::from("/bin/sh"),
12839 args: vec!["-c".into(), "exit 17".into()],
12840 env: vec![],
12841 reserved: false,
12842 reserved_prefixes: vec![],
12843 protocol: ModuleProtocol::None,
12844 overlap: Default::default(),
12845 })
12846 .unwrap();
12847 let deadline = std::time::Instant::now() + Duration::from_secs(10);
12848 while module.state().unwrap() != crate::ModuleState::Failed {
12849 assert!(
12850 std::time::Instant::now() < deadline,
12851 "waiting for budget failure"
12852 );
12853 tokio::task::yield_now().await;
12854 }
12855 let handler = ControlHandler::new(registry).with_supervisor(handle);
12856 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12857 let replies = handler.handle_supervisor_list(frame).await.unwrap();
12858 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12859 assert_eq!(body["modules"][0]["state"], "failed");
12860 let retry = body["modules"][0]["next_retry_at_ms"]
12861 .as_u64()
12862 .expect("scheduled retry on the wire");
12863 let now = std::time::SystemTime::now()
12864 .duration_since(std::time::UNIX_EPOCH)
12865 .unwrap()
12866 .as_millis() as u64;
12867 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
12868 module.set_enabled(false).await.unwrap();
12869 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
12870 let replies = handler.handle_supervisor_list(frame).await.unwrap();
12871 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12872 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
12873 }
12874
12875 #[tokio::test]
12876 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
12877 let registry = Arc::new(Registry::default());
12878 let handle = SupervisorHandle::new();
12879 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
12880 .with_handle(handle.clone());
12881 supervisor
12882 .supervise_configured(
12883 crate::ModuleSpec {
12884 module_id: "drift".into(),
12885 program: PathBuf::from("disabled-module"),
12886 args: Vec::new(),
12887 env: Vec::new(),
12888 reserved: false,
12889 reserved_prefixes: Vec::new(),
12890 protocol: ModuleProtocol::Subc,
12891 overlap: Default::default(),
12892 },
12893 false,
12894 )
12895 .unwrap();
12896 let handler = ControlHandler::new(registry).with_supervisor(handle);
12897 for configured in [None, Some(false), Some(true)] {
12898 handler.capability_evaluator.configure(
12899 configured.map(|enabled| ("drift".to_string(), enabled)),
12900 BTreeMap::new(),
12901 );
12902 let frame =
12903 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12904 let replies = handler
12905 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
12906 .await
12907 .unwrap();
12908 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12909 assert!(body.get("configured_enabled").is_none());
12910 assert_eq!(body["op"], "supervisor.ack");
12911 let frame =
12912 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12913 let replies = handler.handle_supervisor_list(frame).await.unwrap();
12914 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12915 assert_eq!(
12916 body["modules"][0].get("configured_enabled"),
12917 configured.map(Value::Bool).as_ref()
12918 );
12919 }
12920 }
12921
12922 #[tokio::test(flavor = "current_thread")]
12923 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
12924 let handler = ControlHandler::new(Arc::new(Registry::default()));
12925 let capture = EventCapture::default();
12926 let _subscriber =
12927 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12928 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
12929 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12930
12931 let _ = handler
12932 .handle_client_control_request(
12933 &ctx,
12934 frame,
12935 ClientControlRequest::SupervisorSetEnabled {
12936 module_id: "broca".to_string(),
12937 enabled: false,
12938 },
12939 &mut None,
12940 )
12941 .await
12942 .unwrap();
12943
12944 let events = capture
12945 .events()
12946 .into_iter()
12947 .filter(|event| {
12948 event.target == "control"
12949 && event.fields.get("message").map(String::as_str)
12950 == Some("supervisor request received")
12951 })
12952 .collect::<Vec<_>>();
12953 assert_eq!(events.len(), 1, "one supervisor request log line");
12954 let event = &events[0];
12955 assert_eq!(event.level, tracing::Level::INFO);
12956 assert_eq!(
12957 event.fields.get("op"),
12958 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
12959 );
12960 assert_eq!(
12961 event.fields.get("module_id"),
12962 Some(&"\"broca\"".to_string())
12963 );
12964 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
12965 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
12966 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
12967 }
12968
12969 #[tokio::test(flavor = "current_thread")]
12970 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
12971 let registry = Arc::new(Registry::default());
12972 let module_connection = ConnectionId::new(303);
12973 registry
12974 .register_with_control_ops(
12975 manifest("aft", PROTOCOL_VERSION),
12976 PROTOCOL_VERSION,
12977 module_connection,
12978 module_baseline_control_ops(),
12979 )
12980 .unwrap();
12981 let handler = ControlHandler::new(registry);
12982 let capture = EventCapture::default();
12983 let _subscriber =
12984 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12985 let (ctx, _rx) = route_ctx(module_connection);
12986 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12987
12988 let _ = handler
12989 .handle_client_control_request(
12990 &ctx,
12991 frame,
12992 ClientControlRequest::SupervisorSetEnabled {
12993 module_id: "broca".to_string(),
12994 enabled: false,
12995 },
12996 &mut None,
12997 )
12998 .await
12999 .unwrap();
13000
13001 let callers = capture
13002 .events()
13003 .into_iter()
13004 .filter(|event| {
13005 event.fields.get("message").map(String::as_str)
13006 == Some("supervisor request received")
13007 })
13008 .map(|event| event.fields.get("caller").cloned())
13009 .collect::<Vec<_>>();
13010 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
13011 }
13012
13013 #[tokio::test(flavor = "current_thread")]
13014 async fn supervisor_list_does_not_log_request_received() {
13015 let handler = ControlHandler::new(Arc::new(Registry::default()));
13016 let capture = EventCapture::default();
13017 let _subscriber =
13018 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13019 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
13020 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13021
13022 handler
13023 .handle_client_control_request(
13024 &ctx,
13025 frame,
13026 ClientControlRequest::SupervisorList {},
13027 &mut None,
13028 )
13029 .await
13030 .unwrap();
13031
13032 assert!(capture.events().into_iter().all(|event| {
13033 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13034 }));
13035 }
13036
13037 #[tokio::test(flavor = "current_thread")]
13038 async fn supervisor_rescan_preview_does_not_log_request_received() {
13039 let handler = ControlHandler::new(Arc::new(Registry::default()));
13040 let capture = EventCapture::default();
13041 let _subscriber =
13042 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13043 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
13044 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13045
13046 handler
13047 .handle_client_control_request(
13048 &ctx,
13049 frame,
13050 ClientControlRequest::SupervisorRescan { preview: true },
13051 &mut None,
13052 )
13053 .await
13054 .unwrap();
13055
13056 assert!(capture.events().into_iter().all(|event| {
13057 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13058 }));
13059 }
13060
13061 struct ApproveOperator;
13062 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
13063 fn prompt(
13064 &self,
13065 _: &str,
13066 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
13067 ) -> crate::operator_confirm::ProviderResult {
13068 crate::operator_confirm::ProviderResult::Approved
13069 }
13070 }
13071
13072 #[tokio::test]
13073 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
13074 assert_eq!(
13075 ControlHandler::default().privileged_operator_prompt,
13076 PrivilegedOperatorPrompt::Log
13077 );
13078 for (initial, changed, running) in [
13079 ("log", "enforce", PrivilegedOperatorPrompt::Log),
13080 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
13081 ] {
13082 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
13083 let path = dir.join("subc.jsonc");
13084 std::fs::write(
13085 &path,
13086 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
13087 )
13088 .unwrap();
13089 let config = crate::daemon_config::load(&path).unwrap().unwrap();
13090 let registry = Arc::new(Registry::default());
13091 let handle = SupervisorHandle::new();
13092 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
13093 .with_handle(handle.clone());
13094 let handler = ControlHandler::new(registry)
13095 .with_supervisor(handle)
13096 .with_privileged_operator_prompt(config.privileged_operator_prompt)
13097 .with_supervisor_rescan(supervisor, &path, config.port);
13098 handler.forwarding.operator_confirms().configure(
13099 Arc::new(ApproveOperator),
13100 crate::operator_confirm::OperatorLimits::default(),
13101 );
13102 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
13103
13104 for (disk_mode, needs_restart) in [
13105 (Some(initial), false),
13106 (Some(changed), true),
13107 (Some(changed), true),
13108 (None, running == PrivilegedOperatorPrompt::Enforce),
13109 (Some(initial), false),
13110 ] {
13111 let mut doc = json!({"version":1});
13112 if let Some(mode) = disk_mode {
13113 doc["privileged_operator_prompt"] = json!(mode);
13114 }
13115 std::fs::write(&path, doc.to_string()).unwrap();
13116 for preview in [true, false] {
13117 let request = ClientControlRequest::SupervisorRescan { preview };
13118 let frame = Frame::build(
13119 FrameType::Request,
13120 control_flags(),
13121 0,
13122 0,
13123 1,
13124 serde_json::to_vec(&request).unwrap(),
13125 )
13126 .unwrap();
13127 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13128 assert_eq!(frames.len(), 1);
13129 let ClientControlResponse::SupervisorRescan { result } =
13130 serde_json::from_slice(&frames[0].body).unwrap()
13131 else {
13132 panic!("expected a rescan response");
13133 };
13134 let expected = if needs_restart {
13135 vec!["privileged_operator_prompt"]
13136 } else {
13137 Vec::new()
13138 };
13139 assert_eq!(result.restart_required, expected);
13140 assert_eq!(handler.privileged_operator_prompt, running);
13141 assert_eq!(
13142 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
13143 running
13144 );
13145 }
13146 }
13147 }
13148 }
13149
13150 #[tokio::test]
13151 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
13152 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
13153 let path = dir.join("subc.jsonc");
13154 let registry = Arc::new(Registry::default());
13155 let handle = SupervisorHandle::new();
13156 let supervisor =
13157 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
13158 let handler = ControlHandler::new(registry)
13159 .with_supervisor(handle)
13160 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
13161 .with_supervisor_rescan(supervisor, &path, None);
13162 handler.forwarding.operator_confirms().configure(
13163 Arc::new(ApproveOperator),
13164 crate::operator_confirm::OperatorLimits::default(),
13165 );
13166 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
13167 for preview in [true, false] {
13168 std::fs::write(
13169 &path,
13170 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
13171 )
13172 .unwrap();
13173 let request = ClientControlRequest::SupervisorRescan { preview };
13174 let frame = Frame::build(
13175 FrameType::Request,
13176 control_flags(),
13177 0,
13178 0,
13179 1,
13180 serde_json::to_vec(&request).unwrap(),
13181 )
13182 .unwrap();
13183 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13184 assert_eq!(frames.len(), 1);
13185 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
13186 assert_eq!(
13187 handler.privileged_operator_prompt,
13188 PrivilegedOperatorPrompt::Enforce
13189 );
13190 }
13191 }
13192
13193 #[tokio::test]
13198 async fn route_open_refusal_names_the_check_that_refused() {
13199 let handler = ControlHandler::new(Arc::new(Registry::default()));
13200 let capture = EventCapture::default();
13201 let _subscriber =
13202 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13203 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13204 let response = handler
13205 .handle_control_frame(
13206 &ctx,
13207 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
13208 )
13209 .await
13210 .unwrap();
13211
13212 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13213 let event = capture
13214 .events()
13215 .into_iter()
13216 .find(|event| {
13217 event.target == "control"
13218 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13219 })
13220 .expect("route.open refusal event");
13221 assert_eq!(
13222 event.fields.get("reason"),
13223 Some(&"\"not_registered\"".to_string())
13224 );
13225 }
13226
13227 #[tokio::test]
13228 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
13229 let registry = Arc::new(Registry::default());
13230 let supervisor_handle = SupervisorHandle::new();
13231 let supervisor =
13232 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13233 .with_handle(supervisor_handle.clone())
13234 .with_connection_file_path(std::env::temp_dir().join(format!(
13235 "subc-route-open-refusal-info-{}",
13236 std::process::id()
13237 )));
13238 let module = supervisor
13239 .supervise_configured(
13240 ModuleSpec {
13241 module_id: "warming".to_string(),
13242 program: fake_aft_stub_path(),
13243 args: Vec::new(),
13244 env: Vec::new(),
13245 reserved: false,
13246 reserved_prefixes: Vec::new(),
13247 protocol: ModuleProtocol::Subc,
13248 overlap: Default::default(),
13249 },
13250 true,
13251 )
13252 .unwrap();
13253 assert_eq!(module.state().unwrap(), ModuleState::Running);
13254
13255 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13256 assert!(handler
13257 .counters()
13258 .snapshot()
13259 .get("route_open_refused_by_code")
13260 .is_none());
13261 let capture = EventCapture::default();
13262 let _subscriber =
13263 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13264 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
13265 let response = handler
13266 .handle_control_frame(
13267 &ctx,
13268 route_open_frame(394, "warming", unique_project_root("refusal-info")),
13269 )
13270 .await
13271 .unwrap();
13272 module.stop().await.unwrap();
13273
13274 assert_eq!(parse_error(&response[0])["code"], "module_warming");
13275 let event = capture
13276 .events()
13277 .into_iter()
13278 .find(|event| {
13279 event.target == "control"
13280 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
13281 })
13282 .expect("route.open refusal event");
13283 assert_eq!(
13284 event.fields.get("module_id"),
13285 Some(&"\"warming\"".to_string())
13286 );
13287 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
13288 assert_eq!(
13289 event.fields.get("reason"),
13290 Some(&"\"supervised_not_registered\"".to_string())
13291 );
13292 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
13293 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
13294 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
13295 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13296 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13297 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
13298 assert_eq!(
13299 handler.counters().snapshot()["route_open_refused_by_code"],
13300 json!({ "module_warming": 1 })
13301 );
13302 }
13303
13304 const OUTAGE_START: &str = "route.open refusing module: not serving";
13305 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
13306
13307 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
13308 capture
13309 .events()
13310 .into_iter()
13311 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
13312 .collect()
13313 }
13314
13315 fn supervise_stub(
13316 registry: &Arc<Registry>,
13317 module_id: &str,
13318 enabled: bool,
13319 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
13320 let supervisor_handle = SupervisorHandle::new();
13321 let supervisor =
13322 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
13323 .with_handle(supervisor_handle.clone())
13324 .with_connection_file_path(std::env::temp_dir().join(format!(
13325 "subc-route-outage-{module_id}-{}",
13326 std::process::id()
13327 )));
13328 let module = supervisor
13329 .supervise_configured(
13330 ModuleSpec {
13331 module_id: module_id.to_string(),
13332 program: fake_aft_stub_path(),
13333 args: Vec::new(),
13334 env: Vec::new(),
13335 reserved: false,
13336 reserved_prefixes: Vec::new(),
13337 protocol: ModuleProtocol::Subc,
13338 overlap: Default::default(),
13339 },
13340 enabled,
13341 )
13342 .unwrap();
13343 (supervisor_handle, module)
13344 }
13345
13346 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
13347 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
13348 module_id: module_id.to_string(),
13349 drain_timeout_ms: Some(50),
13350 })
13351 .unwrap();
13352 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
13353 }
13354
13355 #[test]
13359 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
13360 let registry = Arc::new(Registry::default());
13361 let forwarding = Arc::new(ForwardingTable::default());
13362 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13363 let second = ControlHandler::with_forwarding(registry, forwarding);
13364 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
13365 }
13366
13367 #[tokio::test(flavor = "current_thread")]
13371 async fn route_open_unknown_module_refusals_add_no_outage_state() {
13372 let handler = ControlHandler::new(Arc::new(Registry::default()));
13373 let capture = EventCapture::default();
13374 let _subscriber =
13375 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13376 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
13377 for corr in 0..8 {
13378 let response = handler
13379 .handle_control_frame(
13380 &ctx,
13381 route_open_frame(
13382 380 + corr,
13383 &format!("nobody-{corr}"),
13384 unique_project_root("outage-unknown"),
13385 ),
13386 )
13387 .await
13388 .unwrap();
13389 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13390 }
13391
13392 assert_eq!(handler.route_outages.tracked_module_count(), 0);
13393 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
13394 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
13395 }
13396
13397 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13402 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
13403 let registry = Arc::new(Registry::default());
13404 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
13405 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13406 let capture = EventCapture::default();
13407 let _subscriber =
13408 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13409 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
13410 handler.route_outages.record_accepted("outage-restart");
13413
13414 let response = handler
13415 .handle_control_frame(
13416 &ctx,
13417 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
13418 )
13419 .await
13420 .unwrap();
13421 assert_eq!(response[0].header.ty, FrameType::Error);
13422 let starts = outage_lines(&capture, OUTAGE_START);
13423 assert_eq!(starts.len(), 1, "{starts:?}");
13424 assert_eq!(starts[0].level, tracing::Level::WARN);
13425 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
13426 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
13427 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
13428 handler.route_outages.record_accepted("outage-restart");
13429 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
13430
13431 let restart = handler
13432 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
13433 .await
13434 .unwrap();
13435 assert_eq!(
13436 restart[0].header.ty,
13437 FrameType::Response,
13438 "{:?}",
13439 parse_error(&restart[0])
13440 );
13441 handler
13442 .handle_control_frame(
13443 &ctx,
13444 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
13445 )
13446 .await
13447 .unwrap();
13448 module.stop().await.unwrap();
13449
13450 let starts = outage_lines(&capture, OUTAGE_START);
13451 assert_eq!(starts.len(), 2, "{starts:?}");
13452 assert_eq!(starts[1].level, tracing::Level::INFO);
13453 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
13454 }
13455
13456 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13460 async fn failed_operator_restart_leaves_no_operator_mark() {
13461 let registry = Arc::new(Registry::default());
13462 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
13463 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13464 let capture = EventCapture::default();
13465 let _subscriber =
13466 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13467 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
13468 handler.route_outages.record_accepted("outage-disabled");
13469
13470 let restart = handler
13471 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
13472 .await
13473 .unwrap();
13474 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
13475 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
13476
13477 handler
13478 .handle_control_frame(
13479 &ctx,
13480 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
13481 )
13482 .await
13483 .unwrap();
13484 let starts = outage_lines(&capture, OUTAGE_START);
13485 assert_eq!(starts.len(), 1, "{starts:?}");
13486 assert_eq!(starts[0].level, tracing::Level::WARN);
13487 }
13488
13489 #[tokio::test(flavor = "current_thread")]
13490 async fn route_open_unknown_module_escapes_target_module_id() {
13491 let handler = ControlHandler::new(Arc::new(Registry::default()));
13492 let capture = EventCapture::default();
13493 let _subscriber =
13494 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13495 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
13496 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13497 let response = handler
13498 .handle_control_frame(
13499 &ctx,
13500 route_open_frame(
13501 395,
13502 hostile_module_id,
13503 unique_project_root("hostile-target-module-id"),
13504 ),
13505 )
13506 .await
13507 .unwrap();
13508
13509 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13510 let event = capture
13511 .events()
13512 .into_iter()
13513 .find(|event| {
13514 event.target == "control"
13515 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13516 })
13517 .expect("route.open unknown-module refusal event");
13518 let logged = event.fields.get("module_id").expect("module_id field");
13519 assert!(!logged.bytes().any(|byte| byte < 0x20));
13520 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13521 }
13522
13523 #[tokio::test(flavor = "current_thread")]
13524 async fn route_open_module_rejection_uses_daemon_counter_key() {
13525 let registry = Arc::new(Registry::default());
13526 let forwarding = Arc::new(ForwardingTable::default());
13527 let handler =
13528 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13529 let module_connection = ConnectionId::new(95);
13530 let (module_ctx, mut module_rx) = route_ctx(module_connection);
13531 hello_via_sink(
13532 &handler,
13533 &module_ctx,
13534 &mut module_rx,
13535 hello_frame("aft", PROTOCOL_VERSION, 395),
13536 )
13537 .await;
13538
13539 let client_connection = ConnectionId::new(96);
13540 let (client_ctx, _client_rx) = route_ctx(client_connection);
13541 let capture = EventCapture::default();
13542 let _subscriber =
13543 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13544 let (route_task, bind) = relay_route_open(
13545 &handler,
13546 client_connection,
13547 &client_ctx.egress,
13548 &mut module_rx,
13549 396,
13550 "aft",
13551 "hostile-module-code",
13552 )
13553 .await;
13554 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
13555 let rejection = Frame::build(
13556 FrameType::Error,
13557 control_flags(),
13558 0,
13559 0,
13560 bind.header.corr,
13561 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
13562 )
13563 .unwrap();
13564 handler
13565 .handle_control_frame(&module_ctx, rejection)
13566 .await
13567 .unwrap();
13568
13569 let response = route_task.await.unwrap();
13570 assert_eq!(parse_error(&response[0])["code"], hostile_code);
13571 let counters = handler.counters().snapshot();
13572 assert_eq!(
13573 counters["route_open_refused_by_code"],
13574 json!({ "module_rejected": 1 })
13575 );
13576 assert!(counters["route_open_refused_by_code"]
13577 .get(hostile_code)
13578 .is_none());
13579
13580 let event = capture
13581 .events()
13582 .into_iter()
13583 .find(|event| {
13584 event.target == "control"
13585 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
13586 })
13587 .expect("route.open module-rejection refusal event");
13588 let logged = event.fields.get("module_code").expect("module_code field");
13589 assert!(!logged.bytes().any(|byte| byte < 0x20));
13590 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13591 }
13592
13593 #[tokio::test]
13594 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
13595 let registry = Arc::new(Registry::default());
13596 let supervisor_handle = SupervisorHandle::new();
13597 let missing_program = std::env::temp_dir().join(format!(
13598 "subc-route-open-missing-program-{}",
13599 std::process::id()
13600 ));
13601 let supervisor =
13602 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13603 .with_handle(supervisor_handle.clone());
13604 let module = supervisor
13605 .supervise_configured(
13606 ModuleSpec {
13607 module_id: "failed".to_string(),
13608 program: missing_program,
13609 args: Vec::new(),
13610 env: Vec::new(),
13611 reserved: false,
13612 reserved_prefixes: Vec::new(),
13613 protocol: ModuleProtocol::Subc,
13614 overlap: Default::default(),
13615 },
13616 true,
13617 )
13618 .unwrap();
13619 assert_eq!(module.state().unwrap(), ModuleState::Failed);
13620
13621 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13622 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
13623 let response = handler
13624 .handle_control_frame(
13625 &ctx,
13626 route_open_frame(305, "failed", unique_project_root("failed")),
13627 )
13628 .await
13629 .unwrap();
13630
13631 assert_eq!(response[0].header.ty, FrameType::Error);
13632 let error = parse_error(&response[0]);
13633 assert_eq!(error["code"], "target_unavailable");
13634 assert!(error["message"]
13635 .as_str()
13636 .unwrap()
13637 .contains("state=failed, enabled=true, live=false"));
13638 }
13639
13640 #[tokio::test]
13641 async fn route_open_role_mismatch_remains_target_unavailable() {
13642 let registry = Arc::new(Registry::default());
13643 let handler = ControlHandler::new(Arc::clone(®istry));
13644 handler
13645 .handle_control(
13646 ConnectionId::new(41),
13647 non_routable_hello_frame_with_control_ops("health-only", 306, None),
13648 )
13649 .unwrap();
13650
13651 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
13652 let response = handler
13653 .handle_control_frame(
13654 &ctx,
13655 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
13656 )
13657 .await
13658 .unwrap();
13659
13660 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13661 assert!(parse_error(&response[0])["message"]
13662 .as_str()
13663 .unwrap()
13664 .contains("does not provide the requested target"));
13665 }
13666
13667 #[tokio::test]
13668 async fn route_open_inactive_registration_remains_target_unavailable() {
13669 let registry = Arc::new(Registry::default());
13670 let handler = ControlHandler::new(Arc::clone(®istry));
13671 handler
13672 .handle_control(
13673 ConnectionId::new(43),
13674 hello_frame("inactive", PROTOCOL_VERSION, 308),
13675 )
13676 .unwrap();
13677 assert!(registry
13678 .set_module_state_for_test("inactive", ChannelState::Closed)
13679 .unwrap());
13680
13681 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
13682 let response = handler
13683 .handle_control_frame(
13684 &ctx,
13685 route_open_frame(309, "inactive", unique_project_root("inactive")),
13686 )
13687 .await
13688 .unwrap();
13689
13690 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13691 assert!(parse_error(&response[0])["message"]
13692 .as_str()
13693 .unwrap()
13694 .contains("is not active"));
13695 }
13696
13697 #[tokio::test]
13698 async fn late_health_reply_is_recorded_through_the_module_response_path() {
13699 let registry = Arc::new(Registry::default());
13700 let forwarding = Arc::new(ForwardingTable::default());
13701 let supervisor_handle = SupervisorHandle::new();
13702 let supervisor =
13703 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
13704 .with_forwarding(Arc::clone(&forwarding))
13705 .with_handle(supervisor_handle.clone());
13706 let module = supervisor
13707 .supervise_configured(
13708 crate::ModuleSpec {
13709 module_id: "late-health-response".to_string(),
13710 program: PathBuf::from("disabled-module"),
13711 args: Vec::new(),
13712 env: Vec::new(),
13713 reserved: false,
13714 reserved_prefixes: Vec::new(),
13715 protocol: ModuleProtocol::Subc,
13716 overlap: Default::default(),
13717 },
13718 false,
13719 )
13720 .unwrap();
13721 let handler =
13722 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13723 .with_supervisor(supervisor_handle);
13724 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
13725 handler
13726 .handle_control_frame(
13727 &module_ctx,
13728 hello_frame_with_control_ops(
13729 "late-health-response",
13730 PROTOCOL_VERSION,
13731 7,
13732 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13733 ),
13734 )
13735 .await
13736 .unwrap();
13737 let probe_started_at = Instant::now() - Duration::from_millis(80);
13738 let pending = forwarding
13739 .begin_health_probe_rpc_for(
13740 "late-health-response",
13741 MODULE_CONTROL_OP_HEALTH_CHECK,
13742 probe_started_at,
13743 Instant::now() - Duration::from_millis(1),
13744 )
13745 .unwrap();
13746 assert!(forwarding
13747 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
13748 .unwrap());
13749
13750 let responses = handler
13751 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
13752 .await
13753 .unwrap();
13754
13755 assert!(responses.is_empty());
13756 let health = module.status().unwrap().health;
13757 assert_eq!(health.late_answer_count, 1);
13758 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
13759 }
13760
13761 #[tokio::test]
13762 async fn health_probe_timeout_and_module_death_are_typed() {
13763 let registry = Arc::new(Registry::default());
13764 let forwarding = Arc::new(ForwardingTable::default());
13765 let handler =
13766 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13767 .with_health_probe_timeout(Duration::from_millis(50));
13768 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
13769 hello_via_sink(
13770 &handler,
13771 &module_ctx,
13772 &mut module_rx,
13773 hello_frame_with_control_ops(
13774 "aft",
13775 PROTOCOL_VERSION,
13776 7,
13777 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13778 ),
13779 )
13780 .await;
13781
13782 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
13783 let responses = handler
13784 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
13785 .await
13786 .unwrap();
13787 assert_eq!(responses[0].header.ty, FrameType::Error);
13788 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
13789 let _ = module_rx.try_recv();
13790
13791 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
13792 let health_handler = handler.clone();
13793 let death_task = tokio::spawn(async move {
13794 health_handler
13795 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
13796 .await
13797 .unwrap()
13798 });
13799 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13800 .await
13801 .unwrap()
13802 .unwrap();
13803 handler
13804 .cleanup_connection(module_ctx.connection_id)
13805 .unwrap();
13806 let responses = death_task.await.unwrap();
13807 assert_eq!(responses[0].header.ty, FrameType::Error);
13808 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
13809 }
13810
13811 #[test]
13812 fn hello_requires_exact_protocol_version() {
13813 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
13814 let registry = Arc::new(Registry::default());
13815 let handler = ControlHandler::new(Arc::clone(®istry));
13816 let responses = handler
13817 .handle_control(
13818 ConnectionId::new(connection),
13819 hello_frame("aft", offered, 9),
13820 )
13821 .unwrap();
13822
13823 assert_eq!(responses.len(), 1);
13824 assert_eq!(responses[0].header.ty, FrameType::Error);
13825 let error = parse_error(&responses[0]);
13826 assert_eq!(error["code"], "version_unsupported");
13827 assert!(registry.get_module("aft").unwrap().is_none());
13828 assert_eq!(registry.active_registration_count().unwrap(), 0);
13829 }
13830 }
13831
13832 #[test]
13833 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
13834 let registry = Arc::new(Registry::default());
13835 let forwarding = Arc::new(ForwardingTable::default());
13836 let handler =
13837 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13838 let module_connection = ConnectionId::new(301);
13839 let registration = registry
13840 .register_with_control_ops(
13841 manifest("aft-push", PROTOCOL_VERSION),
13842 PROTOCOL_VERSION,
13843 module_connection,
13844 module_baseline_control_ops(),
13845 )
13846 .unwrap();
13847 let (module_tx, _module_rx) = mpsc::channel(8);
13848 let endpoint = forwarding
13849 .register_module_connection(
13850 module_connection,
13851 "aft-push".to_string(),
13852 PROTOCOL_VERSION,
13853 manifest_concurrency(®istration.manifest),
13854 FrameSink::new(module_tx),
13855 )
13856 .unwrap();
13857
13858 let unknown = Frame::build(
13860 FrameType::Push,
13861 control_flags(),
13862 0,
13863 0,
13864 5,
13865 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
13866 )
13867 .unwrap();
13868 let out = handler.handle_status_update(endpoint, unknown).unwrap();
13869 assert!(
13870 out.is_empty(),
13871 "unknown push op must be ignored, got {out:?}"
13872 );
13873
13874 let malformed = Frame::build(
13876 FrameType::Push,
13877 control_flags(),
13878 0,
13879 0,
13880 6,
13881 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
13882 )
13883 .unwrap();
13884 let out = handler.handle_status_update(endpoint, malformed).unwrap();
13885 assert_eq!(out.len(), 1);
13886 assert_eq!(out[0].header.ty, FrameType::Error);
13887 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
13888 }
13889
13890 #[test]
13891 fn hello_rejected_when_connection_already_owns_client_routes() {
13892 let registry = Arc::new(Registry::default());
13893 let forwarding = Arc::new(ForwardingTable::default());
13894 let handler =
13895 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13896 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
13898 let client_connection = ConnectionId::new(202);
13899
13900 let responses = handler
13903 .handle_control(
13904 client_connection,
13905 hello_frame("aft-second", PROTOCOL_VERSION, 9),
13906 )
13907 .unwrap();
13908 assert_eq!(responses[0].header.ty, FrameType::Error);
13909 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
13910 assert!(registry.get_module("aft-second").unwrap().is_none());
13911 }
13912
13913 #[tokio::test]
13914 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
13915 let registry = Arc::new(Registry::default());
13916 let forwarding = Arc::new(ForwardingTable::default());
13917 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
13918 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
13919 hello_via_sink(
13920 &handler,
13921 &module_ctx,
13922 &mut module_rx,
13923 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
13924 )
13925 .await;
13926 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
13927 let pending = forwarding
13928 .begin_route_bind_relay_for_test(
13929 client_ctx.connection_id,
13930 client_ctx.egress.clone(),
13931 2,
13932 "alpha",
13933 )
13934 .unwrap();
13935 forwarding
13936 .complete_pending_relay(
13937 module_ctx.connection_id,
13938 pending.corr,
13939 RouteBindRelayOutcome::Accepted,
13940 )
13941 .unwrap();
13942 client_rx.try_recv().unwrap();
13943 for module_id in ["beta", "alpha"] {
13944 let replies = handler
13945 .handle_control_frame(
13946 &module_ctx,
13947 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
13948 )
13949 .await
13950 .unwrap();
13951 assert_eq!(replies.len(), 1, "second HELLO must be refused");
13952 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
13953 }
13954 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
13955 assert!(registry.get_module("beta").unwrap().is_none());
13956 assert!(matches!(
13957 forwarding
13958 .lookup_data_route(
13959 client_ctx.connection_id,
13960 pending.client_channel,
13961 pending.client_epoch,
13962 )
13963 .unwrap(),
13964 DataRoute::Client(DataRouteState::Bound(_))
13965 ));
13966 assert!(handler
13967 .hello_launch_nonces
13968 .lock()
13969 .unwrap()
13970 .presented(module_ctx.connection_id, Some("alpha-nonce")));
13971 assert!(module_rx.try_recv().is_err());
13972 }
13973
13974 #[test]
13975 fn reserved_module_hello_requires_matching_launch_nonce() {
13976 let registry = Arc::new(Registry::default());
13977 let supervisor = SupervisorHandle::new();
13978 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
13981 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
13982
13983 let no_nonce = handler
13985 .handle_control(
13986 ConnectionId::new(1),
13987 hello_frame("vault", PROTOCOL_VERSION, 1),
13988 )
13989 .unwrap();
13990 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
13991 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
13992 assert!(registry.get_module("vault").unwrap().is_none());
13993
13994 let wrong = handler
13996 .handle_control(
13997 ConnectionId::new(2),
13998 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
13999 )
14000 .unwrap();
14001 assert_eq!(wrong[0].header.ty, FrameType::Error);
14002 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
14003 assert!(registry.get_module("vault").unwrap().is_none());
14004
14005 let ok = handler
14007 .handle_control(
14008 ConnectionId::new(3),
14009 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
14010 )
14011 .unwrap();
14012 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
14013 assert!(registry.get_module("vault").unwrap().is_some());
14014 }
14015
14016 #[test]
14017 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
14018 let registry = Arc::new(Registry::default());
14019 let supervisor = SupervisorHandle::new();
14020 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14021 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14022 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14023
14024 let squat = handler
14025 .handle_control(
14026 ConnectionId::new(1),
14027 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
14028 )
14029 .unwrap();
14030 assert_eq!(squat[0].header.ty, FrameType::Error);
14031 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
14032 assert!(parse_error(&squat[0])["message"]
14033 .as_str()
14034 .unwrap()
14035 .contains("fed:"));
14036
14037 let accepted_peer = handler
14038 .handle_control(
14039 ConnectionId::new(2),
14040 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
14041 )
14042 .unwrap();
14043 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
14044
14045 let accepted_short = handler
14046 .handle_control(
14047 ConnectionId::new(3),
14048 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
14049 )
14050 .unwrap();
14051 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
14052
14053 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
14054 let response = handler
14055 .handle_control(
14056 ConnectionId::new(conn),
14057 hello_frame(module_id, PROTOCOL_VERSION, conn),
14058 )
14059 .unwrap();
14060 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
14061 }
14062 }
14063
14064 #[test]
14065 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
14066 let registry = Arc::new(Registry::default());
14067 let supervisor = SupervisorHandle::new();
14068 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14069 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14070 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
14071 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14072
14073 let owner_nonce = handler
14074 .handle_control(
14075 ConnectionId::new(1),
14076 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
14077 )
14078 .unwrap();
14079 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
14080 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
14081 assert!(registry.get_module("fed:special").unwrap().is_none());
14082
14083 let exact_nonce = handler
14084 .handle_control(
14085 ConnectionId::new(2),
14086 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
14087 )
14088 .unwrap();
14089 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
14090 assert!(registry.get_module("fed:special").unwrap().is_some());
14091 }
14092
14093 #[test]
14094 fn non_reserved_module_ignores_launch_nonce() {
14095 let registry = Arc::new(Registry::default());
14096 let handler = ControlHandler::new(Arc::clone(®istry));
14099 let no_nonce = handler
14100 .handle_control(
14101 ConnectionId::new(1),
14102 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
14103 )
14104 .unwrap();
14105 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
14106 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
14107
14108 let echoed_nonce = handler
14109 .handle_control(
14110 ConnectionId::new(2),
14111 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
14112 )
14113 .unwrap();
14114 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
14115 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
14116 }
14117
14118 #[test]
14119 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
14120 let handler = ControlHandler::default();
14121 let conn = ConnectionId::new(1);
14122 let malformed = Frame::build(
14123 FrameType::Hello,
14124 control_flags(),
14125 0,
14126 0,
14127 3,
14128 b"{not json".to_vec(),
14129 )
14130 .unwrap();
14131
14132 let error = handler.handle_control(conn, malformed).unwrap();
14133 assert_eq!(error[0].header.ty, FrameType::Error);
14134 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
14135
14136 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14137 let pong = handler.handle_control(conn, ping).unwrap();
14138 assert_eq!(pong[0].header.ty, FrameType::Pong);
14139 assert_eq!(pong[0].header.corr, 4);
14140 }
14141
14142 #[test]
14143 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
14144 let registry = Arc::new(Registry::default());
14145 let handler = ControlHandler::new(Arc::clone(®istry));
14146
14147 handler
14148 .handle_control(
14149 ConnectionId::new(1),
14150 hello_frame("aft", PROTOCOL_VERSION, 1),
14151 )
14152 .unwrap();
14153 let duplicate = handler
14154 .handle_control(
14155 ConnectionId::new(2),
14156 hello_frame("aft", PROTOCOL_VERSION, 2),
14157 )
14158 .unwrap();
14159
14160 assert_eq!(duplicate[0].header.ty, FrameType::Error);
14161 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14162 let registration = registry.get_module("aft").unwrap().unwrap();
14163 assert_eq!(registration.connection_id, ConnectionId::new(1));
14164 }
14165
14166 #[test]
14167 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
14168 let registry = Arc::new(Registry::default());
14169 let forwarding = Arc::new(ForwardingTable::default());
14170 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
14171 let handler =
14172 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14173 .with_process_liveness(process_liveness);
14174 let (ctx, route_channel, route_epoch) =
14175 bind_liveness_route(®istry, &forwarding, "aft-dead");
14176 let responses = handler
14177 .handle_route_poll(
14178 &ctx,
14179 route_poll_frame(41, PollKind::Liveness, route_channel),
14180 route_channel,
14181 route_epoch,
14182 PollKind::Liveness,
14183 )
14184 .unwrap();
14185
14186 assert_eq!(responses.len(), 1);
14187 assert_eq!(responses[0].header.ty, FrameType::Response);
14188 assert_route_poll_liveness(&responses[0], false);
14189 }
14190
14191 #[test]
14192 fn liveness_poll_without_process_source_uses_bound_route() {
14193 let registry = Arc::new(Registry::default());
14194 let forwarding = Arc::new(ForwardingTable::default());
14195 let handler =
14196 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14197 let (ctx, route_channel, route_epoch) =
14198 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
14199 let responses = handler
14200 .handle_route_poll(
14201 &ctx,
14202 route_poll_frame(42, PollKind::Liveness, route_channel),
14203 route_channel,
14204 route_epoch,
14205 PollKind::Liveness,
14206 )
14207 .unwrap();
14208
14209 assert_route_poll_liveness(&responses[0], true);
14210 }
14211
14212 #[test]
14213 fn liveness_poll_untracked_process_source_uses_bound_route() {
14214 let registry = Arc::new(Registry::default());
14215 let forwarding = Arc::new(ForwardingTable::default());
14216 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
14217 let handler =
14218 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14219 .with_process_liveness(process_liveness);
14220 let (ctx, route_channel, route_epoch) =
14221 bind_liveness_route(®istry, &forwarding, "aft-untracked");
14222 let responses = handler
14223 .handle_route_poll(
14224 &ctx,
14225 route_poll_frame(43, PollKind::Liveness, route_channel),
14226 route_channel,
14227 route_epoch,
14228 PollKind::Liveness,
14229 )
14230 .unwrap();
14231
14232 assert_route_poll_liveness(&responses[0], true);
14233 }
14234
14235 #[tokio::test]
14236 async fn unknown_op_returns_unknown_control_op() {
14237 let handler = ControlHandler::default();
14238 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
14239 let request = Frame::build(
14240 FrameType::Request,
14241 control_flags(),
14242 0,
14243 0,
14244 55,
14245 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
14246 )
14247 .unwrap();
14248
14249 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14250
14251 assert_eq!(response.len(), 1);
14252 assert_eq!(response[0].header.ty, FrameType::Error);
14253 assert_eq!(response[0].header.corr, 55);
14254 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
14255 }
14256
14257 #[tokio::test]
14258 async fn supervisor_provenance_rejects_unknown_exact_module() {
14259 let handler = ControlHandler::default();
14260 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
14261 let request = Frame::build(
14262 FrameType::Request,
14263 control_flags(),
14264 0,
14265 0,
14266 57,
14267 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
14268 )
14269 .unwrap();
14270
14271 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14272
14273 assert_eq!(response.len(), 1);
14274 assert_eq!(response[0].header.ty, FrameType::Error);
14275 assert_eq!(response[0].header.corr, 57);
14276 let error = parse_error(&response[0]);
14277 assert_eq!(error["code"], "unknown_module");
14278 assert_eq!(error["message"], "module_id 'missing' is not supervised");
14279 }
14280
14281 #[test]
14282 fn provenance_probe_override_keeps_handler_tests_deterministic() {
14283 let expected = subc_control::RunningImageAgreement::Unavailable {
14284 reason: subc_control::RunningImageUnavailableReason::HashFailed,
14285 };
14286 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
14287 assert_eq!(handler.provenance_probe_override, Some(expected));
14288 }
14289
14290 #[test]
14291 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
14292 let verdict = reload_verdict(
14293 std::path::Path::new("/bin/new"),
14294 Some(std::path::Path::new("/bin/old")),
14295 subc_control::RunningImageAgreement::Unavailable {
14296 reason: subc_control::RunningImageUnavailableReason::HashFailed,
14297 },
14298 );
14299 assert!(matches!(
14300 verdict.path,
14301 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
14302 if configured == std::path::Path::new("/bin/new")
14303 && spawned_from == std::path::Path::new("/bin/old")
14304 ));
14305 }
14306
14307 #[test]
14308 fn launch_spec_without_running_digest_is_unavailable_not_match() {
14309 use subc_control::{
14310 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
14311 };
14312 let configured = crate::supervise::ModuleSpec {
14313 module_id: "legacy".into(),
14314 program: "/module".into(),
14315 args: Vec::new(),
14316 env: Vec::new(),
14317 reserved: false,
14318 reserved_prefixes: Vec::new(),
14319 protocol: subc_control::ModuleProtocol::Subc,
14320 overlap: Default::default(),
14321 };
14322 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
14323 assert_eq!(
14324 launch_spec_agreement(&configured, false, Some(&digest)),
14325 Agreement::Unavailable {
14326 reason: Reason::NotRunning
14327 }
14328 );
14329 assert_eq!(
14330 launch_spec_agreement(&configured, true, None),
14331 Agreement::Unavailable {
14332 reason: Reason::SpawnedSpecUnavailable
14333 }
14334 );
14335 assert_eq!(
14336 launch_spec_agreement(&configured, true, Some(&digest)),
14337 Agreement::Match
14338 );
14339 }
14340
14341 #[cfg(windows)]
14342 #[test]
14343 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
14344 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
14345 let configured = dir.join("module.exe");
14346 std::fs::write(&configured, b"executable path fixture").unwrap();
14347 let spawned = std::fs::canonicalize(&configured).unwrap();
14348 assert_ne!(
14349 configured, spawned,
14350 "fixture must exercise different path spellings"
14351 );
14352 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
14353 let lower_case = configured.to_string_lossy().to_lowercase();
14354 for spelling in [
14355 &configured,
14356 Path::new(&forward_slashes),
14357 Path::new(&lower_case),
14358 ] {
14359 let verdict = reload_verdict(
14360 spelling,
14361 Some(&spawned),
14362 subc_control::RunningImageAgreement::Unavailable {
14363 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14364 },
14365 );
14366 assert_eq!(
14367 verdict.path,
14368 ReloadPathAgreement::Match,
14369 "configured {spelling:?}, spawned {spawned:?}"
14370 );
14371 }
14372 }
14373
14374 #[cfg(windows)]
14375 #[test]
14376 fn windows_reload_path_keeps_different_existing_program_pending() {
14377 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
14378 let old = dir.join("old.exe");
14379 let new = dir.join("new.exe");
14380 std::fs::write(&old, b"same executable content").unwrap();
14381 std::fs::copy(&old, &new).unwrap();
14382 let spawned = std::fs::canonicalize(&old).unwrap();
14383 let image = subc_control::RunningImageAgreement::Unavailable {
14384 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14385 };
14386 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
14387 assert!(
14388 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
14389 if configured == new && spawned_from == spawned)
14390 );
14391 let missing = dir.join("missing.exe");
14392 assert!(matches!(
14393 reload_verdict(&missing, Some(&spawned), image).path,
14394 ReloadPathAgreement::Mismatch { .. }
14395 ));
14396 }
14397
14398 #[test]
14399 fn reload_verdict_detects_replaced_image_at_same_path() {
14400 let image = subc_control::RunningImageAgreement::Mismatch {
14401 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
14402 digest: "old".into(),
14403 },
14404 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
14405 digest: "new".into(),
14406 },
14407 };
14408 let verdict = reload_verdict(
14409 std::path::Path::new("/bin/same"),
14410 Some(std::path::Path::new("/bin/same")),
14411 image.clone(),
14412 );
14413 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14414 assert_eq!(verdict.image, image);
14415 }
14416
14417 #[test]
14418 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
14419 let image = subc_control::RunningImageAgreement::Unavailable {
14420 reason: subc_control::RunningImageUnavailableReason::NotRunning,
14421 };
14422 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
14423 assert_eq!(
14424 verdict.path,
14425 subc_control::ReloadPathAgreement::Unavailable {
14426 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
14427 }
14428 );
14429 assert_eq!(verdict.image, image);
14430
14431 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
14432 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14433 };
14434 let verdict = reload_verdict(
14435 std::path::Path::new("/bin/same"),
14436 Some(std::path::Path::new("/bin/same")),
14437 unconfirmed.clone(),
14438 );
14439 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14440 assert_eq!(verdict.image, unconfirmed);
14441 }
14442
14443 #[test]
14444 fn reload_verdict_preserves_each_image_unavailability_reason() {
14445 use subc_control::RunningImageUnavailableReason as Reason;
14446
14447 for reason in [
14448 Reason::NotRunning,
14449 Reason::UnsupportedPlatform,
14450 Reason::RunningExecutableUnreadable,
14451 Reason::SpawnedPathUnreadable,
14452 Reason::HashFailed,
14453 Reason::ProcessIdentityUnconfirmed,
14454 Reason::Unknown("future_probe_reason".to_string()),
14455 ] {
14456 let image = subc_control::RunningImageAgreement::Unavailable {
14457 reason: reason.clone(),
14458 };
14459 let verdict = reload_verdict(
14460 std::path::Path::new("/bin/same"),
14461 Some(std::path::Path::new("/bin/same")),
14462 image.clone(),
14463 );
14464 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14465 assert_eq!(verdict.image, image, "{reason:?}");
14466 }
14467 }
14468
14469 #[tokio::test]
14470 async fn malformed_control_bodies_return_invalid_control_body() {
14471 let handler = ControlHandler::default();
14472 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
14473
14474 for (corr, body) in [
14475 (56, br#"{"route_channel":1}"#.as_slice()),
14476 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
14477 (
14478 58,
14479 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
14480 ),
14481 ] {
14482 let request = Frame::build(
14483 FrameType::Request,
14484 control_flags(),
14485 0,
14486 0,
14487 corr,
14488 body.to_vec(),
14489 )
14490 .unwrap();
14491 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14492
14493 assert_eq!(response.len(), 1);
14494 assert_eq!(response[0].header.ty, FrameType::Error);
14495 assert_eq!(response[0].header.corr, corr);
14496 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
14497 }
14498 }
14499
14500 #[tokio::test]
14501 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
14502 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14503 let registry = Arc::new(Registry::default());
14504 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
14505 let router = Router::with_control_handler(Arc::clone(&control));
14506 let connection = router.begin_connection();
14507 let (ctx, mut rx) = route_ctx(connection.id());
14508
14509 router
14510 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
14511 .await
14512 .unwrap();
14513 let response = rx.recv().await.unwrap();
14514 let ack = parse_ack(&response);
14515 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14516 let channel = 1;
14517
14518 let goodbye =
14519 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
14520 router.route_for_connection(&ctx, goodbye).await.unwrap();
14521 assert!(rx.try_recv().is_err());
14522 assert!(registry.get_module("aft").unwrap().is_none());
14523
14524 router
14525 .route_for_connection(&ctx, channel_request(channel, 13))
14526 .await
14527 .unwrap();
14528 let error_frame = rx.recv().await.unwrap();
14529 assert_eq!(error_frame.header.ty, FrameType::Error);
14530 assert_eq!(error_frame.header.channel, channel);
14531 let captured = crate::router::test_log::captured_logs(&logs);
14532 assert_eq!(
14533 captured
14534 .lines()
14535 .filter(|line| {
14536 line.contains("module_id=aft")
14537 && line.contains("reason=explicit_goodbye")
14538 && line.contains("module registration ended")
14539 })
14540 .count(),
14541 1,
14542 "unexpected GOODBYE registry log: {captured}"
14543 );
14544 }
14545
14546 #[tokio::test]
14547 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
14548 let registry = Arc::new(Registry::default());
14549 let handler = ControlHandler::new(registry).with_capability_config(
14550 [("prov".to_string(), true), ("cons".to_string(), true)],
14551 BTreeMap::new(),
14552 );
14553 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
14554 register_capability_manifest(
14555 &handler,
14556 &provider_ctx,
14557 &mut provider_rx,
14558 capability_manifest("prov", &["thing/v1"], &[]),
14559 1,
14560 )
14561 .await;
14562 let mut consumer = capability_manifest("cons", &[], &[]);
14563 consumer.capabilities.as_mut().unwrap().requires.push(
14564 subc_protocol::manifest::CapabilityRequirement {
14565 capability: "thing/v1".to_string(),
14566 need: subc_protocol::manifest::CapabilityNeed::Required,
14567 },
14568 );
14569 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
14570 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
14571 assert_eq!(
14572 handler.capability_evaluator.verdict("cons", "thing/v1"),
14573 Some(CapabilityVerdict::Provided)
14574 );
14575 let (mut client_rx, _) = open_route_for_capability_test(
14576 &handler,
14577 &provider_ctx,
14578 &mut provider_rx,
14579 703,
14580 3,
14581 "prov",
14582 None,
14583 )
14584 .await;
14585 let goodbye =
14586 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14587 handler
14588 .handle_control_frame(&provider_ctx, goodbye)
14589 .await
14590 .unwrap();
14591 assert_eq!(
14592 handler.capability_evaluator.verdict("cons", "thing/v1"),
14593 Some(CapabilityVerdict::NeverProvided)
14594 );
14595 let closed = client_rx
14596 .try_recv()
14597 .expect("GOODBYE pushes route.closed before route GOODBYE");
14598 assert!(
14599 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
14600 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
14601 );
14602 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
14603 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14604 }
14605
14606 struct RouteClosedFixture {
14607 handler: ControlHandler,
14608 #[cfg(unix)]
14609 supervisor: Supervisor,
14610 module_ctx: RouteCtx,
14611 #[cfg(unix)]
14612 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14613 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14614 route: Arc<crate::forwarding::RouteBinding>,
14615 }
14616
14617 async fn route_closed_fixture() -> RouteClosedFixture {
14618 let registry = Arc::new(Registry::default());
14619 let handler = ControlHandler::new(Arc::clone(®istry));
14620 #[cfg(unix)]
14621 let supervisor = Supervisor::new(registry, RestartPolicy::default())
14622 .with_forwarding(Arc::clone(&handler.forwarding));
14623 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
14624 register_capability_manifest(
14625 &handler,
14626 &module_ctx,
14627 &mut module_rx,
14628 capability_manifest("prov", &[], &[]),
14629 1,
14630 )
14631 .await;
14632 let (client_rx, bind) = open_route_for_capability_test(
14633 &handler,
14634 &module_ctx,
14635 &mut module_rx,
14636 702,
14637 2,
14638 "prov",
14639 None,
14640 )
14641 .await;
14642 let ModuleControlRequest::RouteBind {
14643 route_channel: channel,
14644 epoch: route_epoch,
14645 ..
14646 } = bind
14647 else {
14648 panic!("expected route.bind");
14649 };
14650 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
14651 handler
14652 .forwarding
14653 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
14654 .unwrap()
14655 else {
14656 panic!("the acknowledged route must be bound");
14657 };
14658 RouteClosedFixture {
14659 handler,
14660 #[cfg(unix)]
14661 supervisor,
14662 module_ctx,
14663 #[cfg(unix)]
14664 module_rx,
14665 client_rx,
14666 route,
14667 }
14668 }
14669
14670 impl RouteClosedFixture {
14671 #[cfg(unix)]
14672 async fn cut(&mut self) {
14673 let notices = async {
14674 let mut draining = self.module_rx.recv().await.unwrap();
14675 assert!(matches!(
14676 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
14677 &draining.body
14678 )
14679 .unwrap(),
14680 subc_protocol::session::ModuleControlCommand::Draining {
14681 reason: RouteCloseReason::Restart,
14682 ..
14683 }
14684 ));
14685 draining.flushed.take().unwrap().send(()).unwrap();
14686 let mut closing = self.client_rx.recv().await.unwrap();
14687 assert!(matches!(
14688 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
14689 ClientControlPush::RouteClosing {
14690 reason: RouteCloseReason::Restart,
14691 ..
14692 }
14693 ));
14694 closing.flushed.take().unwrap().send(()).unwrap();
14695 };
14696 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
14697 result.unwrap();
14698 }
14699
14700 fn disconnect(&mut self) -> ClientControlPush {
14701 self.handler
14702 .cleanup_connection(self.module_ctx.connection_id)
14703 .unwrap();
14704 let closed = self.client_rx.try_recv().unwrap();
14705 assert_eq!(closed.header.ty, FrameType::Push);
14706 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
14707 assert!(matches!(
14708 &push,
14709 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
14710 if channels == &[self.route.client_channel]
14711 ));
14712 assert_eq!(
14713 self.client_rx.try_recv().unwrap().header.ty,
14714 FrameType::Goodbye
14715 );
14716 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
14717 push
14718 }
14719 }
14720
14721 #[cfg(unix)]
14722 #[tokio::test(start_paused = true)]
14723 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
14724 let mut fixture = route_closed_fixture().await;
14725 fixture.cut().await;
14726 assert!(matches!(
14727 fixture.disconnect(),
14728 ClientControlPush::RouteClosed {
14729 reason: RouteCloseReason::Restart,
14730 drained: true,
14731 ..
14732 }
14733 ));
14734 }
14735
14736 #[cfg(unix)]
14737 #[tokio::test(start_paused = true)]
14738 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
14739 let mut fixture = route_closed_fixture().await;
14740 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
14741 let started = tokio::time::Instant::now();
14742 fixture.cut().await;
14743 assert!(started.elapsed() >= Duration::from_secs(2));
14744 assert_eq!(fixture.route.flow.in_flight(), 1);
14745 assert!(matches!(
14746 fixture.disconnect(),
14747 ClientControlPush::RouteClosed {
14748 reason: RouteCloseReason::Restart,
14749 drained: false,
14750 ..
14751 }
14752 ));
14753 }
14754
14755 #[tokio::test]
14756 async fn module_connection_crash_reports_crash_and_not_drained() {
14757 let mut fixture = route_closed_fixture().await;
14758 assert!(matches!(
14762 fixture.disconnect(),
14763 ClientControlPush::RouteClosed {
14764 reason: RouteCloseReason::Crash,
14765 drained: false,
14766 ..
14767 }
14768 ));
14769 }
14770
14771 #[tokio::test]
14772 async fn dropping_router_connection_releases_registration() {
14773 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14774 let registry = Arc::new(Registry::default());
14775 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
14776 let router = Router::with_control_handler(Arc::clone(&control));
14777 let connection = router.begin_connection();
14778 let connection_id = connection.id();
14779 let (ctx, mut rx) = route_ctx(connection_id);
14780
14781 router
14782 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
14783 .await
14784 .unwrap();
14785 let response = rx.recv().await.unwrap();
14786 let ack = parse_ack(&response);
14787 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14788 assert!(registry.get_module("aft").unwrap().is_some());
14789
14790 drop(connection);
14791
14792 assert!(registry.get_module("aft").unwrap().is_none());
14793 assert_eq!(registry.active_registration_count().unwrap(), 0);
14794
14795 control
14796 .cleanup_connection(ConnectionId::new(u64::MAX))
14797 .unwrap();
14798 let captured = crate::router::test_log::captured_logs(&logs);
14799 println!("captured registry lifecycle logs:\n{captured}");
14800 let events: Vec<_> = captured
14801 .lines()
14802 .filter(|line| {
14803 line.contains("module registered module_id=aft ")
14804 || line.contains("module registration ended")
14805 })
14806 .collect();
14807 assert_eq!(
14808 events.len(),
14809 2,
14810 "unexpected registry lifecycle logs: {captured}"
14811 );
14812 assert!(events[0].contains("module registered module_id=aft "));
14813 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
14814 assert!(events[1].contains(&format!(
14815 "module_id=aft connection_id={}",
14816 connection_id.get()
14817 )));
14818 assert!(events[1].contains("reason=connection_closed"));
14819 assert!(events[1].contains("module registration ended"));
14820 assert_eq!(
14821 captured
14822 .lines()
14823 .filter(|line| line.contains("module registered module_id=aft "))
14824 .count(),
14825 1,
14826 "legacy registration admission line must appear once: {captured}"
14827 );
14828 }
14829
14830 #[tokio::test]
14831 async fn hello_registration_keeps_legacy_module_registered_line_once() {
14832 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14833 let registry = Arc::new(Registry::default());
14834 let control = ControlHandler::new(Arc::clone(®istry));
14835 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
14836 hello_via_sink(
14837 &control,
14838 &ctx,
14839 &mut rx,
14840 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
14841 )
14842 .await;
14843
14844 let captured = crate::router::test_log::captured_logs(&logs);
14845 let admissions: Vec<_> = captured
14846 .lines()
14847 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
14848 .collect();
14849 assert_eq!(
14850 admissions.len(),
14851 1,
14852 "legacy admission line must remain exactly once: {captured}"
14853 );
14854 assert!(admissions[0].contains("routable_provider=true"));
14855 assert!(admissions[0].contains("connection_id=777"));
14856 }
14857
14858 fn capability_manifest(
14859 module_id: &str,
14860 provides: &[&str],
14861 must_never_reach: &[&str],
14862 ) -> ModuleManifest {
14863 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
14864 manifest.capabilities = Some(CapabilityDeclarations {
14865 provides: provides
14866 .iter()
14867 .map(|capability| (*capability).to_string())
14868 .collect(),
14869 requires: Vec::new(),
14870 must_never_reach: must_never_reach
14871 .iter()
14872 .map(|capability| (*capability).to_string())
14873 .collect(),
14874 });
14875 manifest
14876 }
14877
14878 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
14879 Frame::build(
14880 FrameType::Hello,
14881 control_flags(),
14882 0,
14883 0,
14884 corr,
14885 serde_json::to_vec(&ModuleHelloBody {
14886 protocol_ver: manifest.protocol_ver,
14887 manifest,
14888 control_ops: None,
14889 launch_nonce: None,
14890 })
14891 .expect("capability test HELLO serializes"),
14892 )
14893 .expect("capability test HELLO frame builds")
14894 }
14895
14896 fn catalog_update_with_capabilities_frame(
14897 corr: u64,
14898 capabilities: CapabilityDeclarations,
14899 ) -> Frame {
14900 Frame::build(
14901 FrameType::Request,
14902 control_flags(),
14903 0,
14904 0,
14905 corr,
14906 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14907 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
14908 capabilities: Some(capabilities),
14909 ready: None,
14910 })
14911 .expect("capability catalog.update serializes"),
14912 )
14913 .expect("capability catalog.update frame builds")
14914 }
14915
14916 async fn register_capability_manifest(
14917 handler: &ControlHandler,
14918 ctx: &RouteCtx,
14919 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14920 manifest: ModuleManifest,
14921 corr: u64,
14922 ) {
14923 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
14924 }
14925
14926 async fn open_route_for_capability_test(
14927 handler: &ControlHandler,
14928 target_ctx: &RouteCtx,
14929 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14930 client_connection_id: u64,
14931 corr: u64,
14932 target_module_id: &str,
14933 consumer_identity: Option<ConsumerIdentity>,
14934 ) -> (
14935 mpsc::Receiver<crate::router::OutboundFrame>,
14936 ModuleControlRequest,
14937 ) {
14938 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
14939 let route_handler = handler.clone();
14940 let target_module_id = target_module_id.to_string();
14941 let route_task = tokio::spawn(async move {
14942 route_handler
14943 .handle_control_frame(
14944 &client_ctx,
14945 route_open_frame_with_admission_facts(
14946 corr,
14947 &target_module_id,
14948 unique_project_root("admission-facts"),
14949 consumer_identity,
14950 None,
14951 ),
14952 )
14953 .await
14954 .expect("capability test route.open succeeds")
14955 });
14956 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
14957 .await
14958 .expect("capability test route.open must reach route.bind")
14959 .expect("target control receiver stays open");
14960 let bind_request: ModuleControlRequest =
14961 serde_json::from_slice(&bind.body).expect("route.bind decodes");
14962 handler
14963 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
14964 .await
14965 .expect("capability test route.bind ACK succeeds");
14966 assert!(route_task.await.expect("route.open task joins").is_empty());
14967 let opened = client_rx
14968 .recv()
14969 .await
14970 .expect("successful route.open publishes a response");
14971 assert!(matches!(
14972 serde_json::from_slice::<ClientControlResponse>(&opened.body),
14973 Ok(ClientControlResponse::RouteOpen { .. })
14974 ));
14975 (client_rx, bind_request)
14976 }
14977
14978 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
14979 assert_eq!(frame.header.ty, FrameType::Push);
14980 assert_eq!(frame.header.channel, 0);
14981 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
14982 .expect("route.closed control push decodes");
14983 let ClientControlPush::RouteClosed { channels, .. } = &push else {
14984 panic!("expected route.closed");
14985 };
14986 assert_eq!(channels.len(), 1, "exactly one violating route closed");
14987 let closed_channel = channels[0];
14988 let channels = channels.clone();
14989 assert_eq!(
14990 push,
14991 ClientControlPush::RouteClosed {
14992 module_id: target_module_id.to_string(),
14993 channels,
14994 reason: RouteCloseReason::CapabilityDenied,
14995 drained: false,
14996 abandoned: 0,
14997 excluded_subscriptions: 0,
14998 terminal: Some(false),
14999 }
15000 );
15001 closed_channel
15002 }
15003
15004 #[tokio::test]
15005 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
15006 let registry = Arc::new(Registry::default());
15007 let forwarding = Arc::new(ForwardingTable::default());
15008 let supervisor = SupervisorHandle::new();
15009 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15010 let handler =
15011 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15012 .with_supervisor(supervisor);
15013 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
15014 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
15015 register_capability_manifest(
15016 &handler,
15017 &target_ctx,
15018 &mut target_rx,
15019 capability_manifest("target", &["credentials-provider/v1"], &[]),
15020 1,
15021 )
15022 .await;
15023 register_capability_manifest(
15024 &handler,
15025 &opener_ctx,
15026 &mut opener_rx,
15027 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15028 2,
15029 )
15030 .await;
15031
15032 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
15033 let replies = handler
15034 .handle_control_frame(
15035 &client_ctx,
15036 route_open_frame_with_admission_facts(
15037 3,
15038 "target",
15039 unique_project_root("admission-facts"),
15040 Some(ConsumerIdentity {
15041 module_id: "opener".to_string(),
15042 launch_nonce: "opener-nonce".to_string(),
15043 }),
15044 None,
15045 ),
15046 )
15047 .await
15048 .expect("denied route.open returns a typed frame");
15049 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15050 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15051 assert!(
15052 target_rx.try_recv().is_err(),
15053 "forbidden route.open must not relay route.bind"
15054 );
15055 }
15056
15057 #[tokio::test]
15058 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
15059 let registry = Arc::new(Registry::default());
15060 let forwarding = Arc::new(ForwardingTable::default());
15061 let supervisor = SupervisorHandle::new();
15062 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15063 let handler =
15064 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15065 .with_supervisor(supervisor);
15066 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
15067 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
15068 register_capability_manifest(
15069 &handler,
15070 &target_ctx,
15071 &mut target_rx,
15072 capability_manifest("target", &["credentials-provider/v1"], &[]),
15073 1,
15074 )
15075 .await;
15076 register_capability_manifest(
15077 &handler,
15078 &old_opener_ctx,
15079 &mut old_opener_rx,
15080 capability_manifest("opener", &[], &[]),
15081 2,
15082 )
15083 .await;
15084 let (mut client_rx, _) = open_route_for_capability_test(
15085 &handler,
15086 &target_ctx,
15087 &mut target_rx,
15088 712,
15089 3,
15090 "target",
15091 Some(ConsumerIdentity {
15092 module_id: "opener".to_string(),
15093 launch_nonce: "opener-nonce".to_string(),
15094 }),
15095 )
15096 .await;
15097 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15098
15099 handler
15100 .cleanup_connection(old_opener_ctx.connection_id)
15101 .expect("old opener registration cleans up");
15102 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
15103 register_capability_manifest(
15104 &handler,
15105 &new_opener_ctx,
15106 &mut new_opener_rx,
15107 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15108 4,
15109 )
15110 .await;
15111
15112 let closed_channel = assert_capability_denied_push(
15113 client_rx
15114 .try_recv()
15115 .expect("HELLO deny addition must emit route.closed")
15116 .frame,
15117 "target",
15118 );
15119 let goodbye = client_rx
15120 .try_recv()
15121 .expect("HELLO deny addition must send the client a route GOODBYE")
15122 .frame;
15123 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15124 assert_eq!(goodbye.header.channel, closed_channel);
15125 assert_ne!(goodbye.header.channel, 0);
15126 assert_ne!(goodbye.header.epoch, 0);
15127 assert!(client_rx.try_recv().is_err());
15128 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15129 assert!(matches!(
15130 target_rx.try_recv(),
15131 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15132 ));
15133 }
15134
15135 #[tokio::test]
15136 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
15137 let registry = Arc::new(Registry::default());
15138 let forwarding = Arc::new(ForwardingTable::default());
15139 let supervisor = SupervisorHandle::new();
15140 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15141 let handler =
15142 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15143 .with_supervisor(supervisor);
15144 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
15145 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
15146 register_capability_manifest(
15147 &handler,
15148 &target_ctx,
15149 &mut target_rx,
15150 capability_manifest("target", &[], &[]),
15151 1,
15152 )
15153 .await;
15154 register_capability_manifest(
15155 &handler,
15156 &opener_ctx,
15157 &mut opener_rx,
15158 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15159 2,
15160 )
15161 .await;
15162 let (mut client_rx, _) = open_route_for_capability_test(
15163 &handler,
15164 &target_ctx,
15165 &mut target_rx,
15166 722,
15167 3,
15168 "target",
15169 Some(ConsumerIdentity {
15170 module_id: "opener".to_string(),
15171 launch_nonce: "opener-nonce".to_string(),
15172 }),
15173 )
15174 .await;
15175 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15176
15177 let replies = handler
15178 .handle_control_frame(
15179 &target_ctx,
15180 catalog_update_with_capabilities_frame(
15181 4,
15182 CapabilityDeclarations {
15183 provides: vec!["credentials-provider/v1".to_string()],
15184 requires: Vec::new(),
15185 must_never_reach: Vec::new(),
15186 },
15187 ),
15188 )
15189 .await
15190 .expect("claim catalog.update succeeds");
15191 assert!(matches!(
15192 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
15193 Ok(ModuleControlResponseToModule::CatalogUpdate {})
15194 ));
15195 let closed_channel = assert_capability_denied_push(
15196 client_rx
15197 .try_recv()
15198 .expect("claim addition must emit route.closed")
15199 .frame,
15200 "target",
15201 );
15202 let goodbye = client_rx
15203 .try_recv()
15204 .expect("claim addition must send the client a route GOODBYE")
15205 .frame;
15206 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15207 assert_eq!(goodbye.header.channel, closed_channel);
15208 assert_ne!(goodbye.header.channel, 0);
15209 assert_ne!(goodbye.header.epoch, 0);
15210 assert!(client_rx.try_recv().is_err());
15211 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15212 assert!(matches!(
15213 target_rx.try_recv(),
15214 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15215 ));
15216 }
15217
15218 #[tokio::test]
15219 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
15220 let registry = Arc::new(Registry::default());
15221 let forwarding = Arc::new(ForwardingTable::default());
15222 let supervisor = SupervisorHandle::new();
15223 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15224 let handler =
15225 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15226 .with_supervisor(supervisor);
15227 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
15228 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
15229 register_capability_manifest(
15230 &handler,
15231 &target_ctx,
15232 &mut target_rx,
15233 capability_manifest("target", &["credentials-provider/v1"], &[]),
15234 1,
15235 )
15236 .await;
15237 register_capability_manifest(
15238 &handler,
15239 &opener_ctx,
15240 &mut opener_rx,
15241 capability_manifest("opener", &[], &[]),
15242 2,
15243 )
15244 .await;
15245 let (mut client_rx, _) = open_route_for_capability_test(
15246 &handler,
15247 &target_ctx,
15248 &mut target_rx,
15249 732,
15250 3,
15251 "target",
15252 Some(ConsumerIdentity {
15253 module_id: "opener".to_string(),
15254 launch_nonce: "opener-nonce".to_string(),
15255 }),
15256 )
15257 .await;
15258 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15259
15260 handler
15261 .handle_control_frame(
15262 &target_ctx,
15263 catalog_update_with_capabilities_frame(
15264 4,
15265 CapabilityDeclarations {
15266 provides: Vec::new(),
15267 requires: Vec::new(),
15268 must_never_reach: Vec::new(),
15269 },
15270 ),
15271 )
15272 .await
15273 .expect("claim removal catalog.update succeeds");
15274 assert_eq!(
15275 forwarding.active_binding_count().unwrap(),
15276 1,
15277 "removing an attested target claim must leave the route census unchanged"
15278 );
15279 assert!(
15280 client_rx.try_recv().is_err(),
15281 "claim removal must not emit route.closed or a client route GOODBYE"
15282 );
15283 assert!(
15284 target_rx.try_recv().is_err(),
15285 "claim removal must not send the target a route GOODBYE"
15286 );
15287 }
15288
15289 #[tokio::test]
15292 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
15293 let registry = Arc::new(Registry::default());
15294 let forwarding = Arc::new(ForwardingTable::default());
15295 let supervisor = SupervisorHandle::new();
15296 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15297 let handler =
15298 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15299 .with_supervisor(supervisor);
15300 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
15301 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
15302 register_capability_manifest(
15303 &handler,
15304 &target_ctx,
15305 &mut target_rx,
15306 capability_manifest("target", &["credentials-provider/v1"], &[]),
15307 1,
15308 )
15309 .await;
15310 register_capability_manifest(
15311 &handler,
15312 &opener_ctx,
15313 &mut opener_rx,
15314 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15315 2,
15316 )
15317 .await;
15318
15319 let (_client_rx, bind) = open_route_for_capability_test(
15320 &handler,
15321 &target_ctx,
15322 &mut target_rx,
15323 742,
15324 3,
15325 "target",
15326 None,
15327 )
15328 .await;
15329 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
15330 panic!("direct scope-honesty route must bind");
15331 };
15332 assert_eq!(principal, Some(Principal::Direct));
15333 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15334 }
15335
15336 #[tokio::test]
15339 async fn must_never_reach_self_route_is_capability_forbidden() {
15340 let registry = Arc::new(Registry::default());
15341 let forwarding = Arc::new(ForwardingTable::default());
15342 let supervisor = SupervisorHandle::new();
15343 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
15344 let handler =
15345 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15346 .with_supervisor(supervisor);
15347 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
15348 register_capability_manifest(
15349 &handler,
15350 &self_ctx,
15351 &mut self_rx,
15352 capability_manifest(
15353 "self-provider",
15354 &["credentials-provider/v1"],
15355 &["credentials-provider/v1"],
15356 ),
15357 1,
15358 )
15359 .await;
15360
15361 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
15362 let replies = handler
15363 .handle_control_frame(
15364 &client_ctx,
15365 route_open_frame_with_admission_facts(
15366 2,
15367 "self-provider",
15368 unique_project_root("admission-facts"),
15369 Some(ConsumerIdentity {
15370 module_id: "self-provider".to_string(),
15371 launch_nonce: "self-nonce".to_string(),
15372 }),
15373 None,
15374 ),
15375 )
15376 .await
15377 .expect("self-route refusal returns a typed frame");
15378 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15379 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15380 assert!(
15381 self_rx.try_recv().is_err(),
15382 "self denial must not relay route.bind"
15383 );
15384 }
15385
15386 #[test]
15387 fn unsupported_channel_zero_frame_returns_error() {
15388 let handler = ControlHandler::default();
15389 let request = Frame::build(
15390 FrameType::Request,
15391 control_flags(),
15392 0,
15393 0,
15394 21,
15395 b"opaque".to_vec(),
15396 )
15397 .unwrap();
15398
15399 let response = handler
15400 .handle_control(ConnectionId::new(1), request)
15401 .unwrap();
15402
15403 assert_eq!(response[0].header.ty, FrameType::Error);
15404 assert_eq!(
15405 parse_error(&response[0])["code"],
15406 "unsupported_control_frame"
15407 );
15408 }
15409
15410 mod swap {
15415 use super::*;
15416
15417 const INCUMBENT: ConnectionId = ConnectionId::new(30);
15418 const CANDIDATE: ConnectionId = ConnectionId::new(40);
15419
15420 struct Swap {
15421 registry: Arc<Registry>,
15422 forwarding: Arc<ForwardingTable>,
15423 handler: ControlHandler,
15424 incumbent_ctx: RouteCtx,
15425 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15426 candidate_ctx: RouteCtx,
15427 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15428 }
15429
15430 async fn swap_with_incumbent() -> Swap {
15431 let registry = Arc::new(Registry::default());
15432 let forwarding = Arc::new(ForwardingTable::default());
15433 let handler =
15434 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15435 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
15436 hello_via_sink(
15437 &handler,
15438 &incumbent_ctx,
15439 &mut incumbent_rx,
15440 hello_frame("aft", PROTOCOL_VERSION, 7),
15441 )
15442 .await;
15443 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
15444 Swap {
15445 registry,
15446 forwarding,
15447 handler,
15448 incumbent_ctx,
15449 incumbent_rx,
15450 candidate_ctx,
15451 candidate_rx,
15452 }
15453 }
15454
15455 fn register_candidate(swap: &Swap, ready: Option<bool>) {
15456 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
15457 candidate_manifest.ready = ready;
15458 let registration = swap
15459 .registry
15460 .register_candidate_with_control_ops(
15461 candidate_manifest,
15462 PROTOCOL_VERSION,
15463 CANDIDATE,
15464 module_baseline_control_ops(),
15465 )
15466 .unwrap();
15467 swap.forwarding
15468 .register_candidate_module_connection(
15469 CANDIDATE,
15470 "aft".to_string(),
15471 PROTOCOL_VERSION,
15472 manifest_concurrency(®istration.manifest),
15473 swap.candidate_ctx.egress.clone(),
15474 )
15475 .unwrap();
15476 }
15477
15478 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
15479 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
15480 swap.registry.promote_candidate("aft").unwrap().unwrap();
15481 cutover.incumbent.unwrap()
15482 }
15483
15484 #[tokio::test]
15493 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
15494 let mut swap = swap_with_incumbent().await;
15495 let hold = Arc::new(std::sync::Barrier::new(2));
15496 let handler = swap
15497 .handler
15498 .clone()
15499 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
15500 handler
15501 .supervisor
15502 .open_swap("aft", "candidate-nonce".to_string());
15503 let hello = {
15504 let handler = handler.clone();
15505 let candidate_ctx = swap.candidate_ctx.clone();
15506 std::thread::spawn(move || {
15507 handler
15508 .handle_hello(
15509 candidate_ctx.connection_id,
15510 Some(candidate_ctx.egress.clone()),
15511 hello_frame_with_nonce(
15512 "aft",
15513 PROTOCOL_VERSION,
15514 8,
15515 Some("candidate-nonce"),
15516 ),
15517 )
15518 .unwrap()
15519 })
15520 };
15521
15522 hold.wait();
15523 assert!(swap
15526 .registry
15527 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
15528 .unwrap()
15529 .is_some());
15530 assert!(swap
15531 .registry
15532 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15533 .unwrap()
15534 .is_none());
15535 assert!(swap
15536 .forwarding
15537 .module_endpoint_for_connection(CANDIDATE)
15538 .unwrap()
15539 .is_none());
15540
15541 hold.wait();
15542 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
15543 assert_eq!(
15544 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
15545 FrameType::HelloAck
15546 );
15547 let published = swap
15548 .registry
15549 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15550 .unwrap()
15551 .expect("candidate is published once its HELLO completes");
15552 assert_eq!(published.connection_id, CANDIDATE);
15553 assert!(swap
15554 .forwarding
15555 .module_endpoint_for_connection(CANDIDATE)
15556 .unwrap()
15557 .is_some());
15558 }
15559
15560 fn keyed_total(counters: &Value, key: &str) -> u64 {
15561 counters[key]
15562 .as_object()
15563 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
15564 .unwrap_or(0)
15565 }
15566
15567 #[tokio::test]
15568 async fn replacement_logs_old_end_and_new_admission_once() {
15569 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15570 let mut swap = swap_with_incumbent().await;
15571 swap.handler
15572 .supervisor
15573 .open_swap("aft", "candidate-nonce".to_string());
15574 hello_via_sink(
15575 &swap.handler,
15576 &swap.candidate_ctx,
15577 &mut swap.candidate_rx,
15578 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
15579 )
15580 .await;
15581 cutover(&swap);
15582 swap.handler.cleanup_connection(INCUMBENT).unwrap();
15583
15584 let captured = crate::router::test_log::captured_logs(&logs);
15585 println!("captured replacement registry logs:\n{captured}");
15586 let new_admissions: Vec<_> = captured
15587 .lines()
15588 .filter(|line| {
15589 line.contains("connection_id=40")
15590 && line.contains("swap candidate registered; not routable until cutover")
15591 })
15592 .collect();
15593 assert_eq!(
15594 new_admissions.len(),
15595 1,
15596 "unexpected admission logs: {captured}"
15597 );
15598 assert!(new_admissions[0].contains("module_id=aft"));
15599 assert!(new_admissions[0].contains("connection_id=40"));
15600 assert!(
15601 new_admissions[0].contains("swap candidate registered; not routable until cutover")
15602 );
15603 assert!(new_admissions[0].contains("ready=true"));
15604
15605 let old_admissions: Vec<_> = captured
15606 .lines()
15607 .filter(|line| {
15608 line.contains("module registered module_id=aft ")
15609 && line.contains("connection_id=30")
15610 })
15611 .collect();
15612 assert_eq!(
15613 old_admissions.len(),
15614 1,
15615 "unexpected incumbent admission: {captured}"
15616 );
15617
15618 let old_ends: Vec<_> = captured
15619 .lines()
15620 .filter(|line| {
15621 line.contains("connection_id=30") && line.contains("module registration ended")
15622 })
15623 .collect();
15624 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
15625 assert!(old_ends[0].contains("module_id=aft"));
15626 assert!(old_ends[0].contains("connection_id=30"));
15627 assert!(old_ends[0].contains("reason=replaced"));
15628 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
15629 assert_eq!(
15630 captured
15631 .lines()
15632 .filter(|line| line.contains("module registration promoted"))
15633 .count(),
15634 1,
15635 "unexpected promotion logs: {captured}"
15636 );
15637 }
15638
15639 #[tokio::test]
15645 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
15646 let mut swap = swap_with_incumbent().await;
15647 let handler = swap.handler.clone();
15648
15649 let cotenant = ConnectionId::new(31);
15651 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
15652 let (cotenant_task, cotenant_bind) = relay_route_open(
15653 &handler,
15654 cotenant,
15655 &cotenant_ctx.egress,
15656 &mut swap.incumbent_rx,
15657 100,
15658 "aft",
15659 "swap-cotenant",
15660 )
15661 .await;
15662 handler
15663 .handle_control_frame(
15664 &swap.incumbent_ctx,
15665 route_bind_ack(cotenant_bind.header.corr),
15666 )
15667 .await
15668 .unwrap();
15669 assert!(cotenant_task.await.unwrap().is_empty());
15670 let (cotenant_channel, cotenant_epoch) =
15671 published_route(&cotenant_rx.recv().await.unwrap());
15672
15673 let caller = ConnectionId::new(32);
15675 let (caller_ctx, mut caller_rx) = route_ctx(caller);
15676 let (caller_task, caller_bind) = relay_route_open(
15677 &handler,
15678 caller,
15679 &caller_ctx.egress,
15680 &mut swap.incumbent_rx,
15681 101,
15682 "aft",
15683 "swap-caller",
15684 )
15685 .await;
15686 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
15687
15688 register_candidate(&swap, None);
15689 cutover(&swap);
15690
15691 let ack = handler
15693 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
15694 .await;
15695 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
15696 if module_loop_error.is_some() {
15697 handler.cleanup_connection(INCUMBENT).unwrap();
15700 }
15701
15702 assert!(
15704 cotenant_rx.try_recv().is_err(),
15705 "the co-tenant route on the incumbent was torn down by one late ack: \
15706 {module_loop_error:?}"
15707 );
15708 assert!(matches!(
15709 swap.forwarding
15710 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
15711 .unwrap(),
15712 DataRoute::Client(DataRouteState::Bound(_))
15713 ));
15714 assert_eq!(module_loop_error, None);
15715 assert!(swap
15716 .registry
15717 .get_module_by_connection(INCUMBENT)
15718 .unwrap()
15719 .is_some());
15720
15721 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
15723 .await
15724 .expect("the incumbent is told to drop the abandoned binding")
15725 .unwrap()
15726 .frame;
15727 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15728 assert_eq!(goodbye.header.channel, abandoned_channel);
15729 assert_eq!(goodbye.header.epoch, abandoned_epoch);
15730 assert!(swap.incumbent_rx.try_recv().is_err());
15731
15732 let response = caller_task.await.unwrap();
15734 assert_eq!(response.len(), 1);
15735 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15736 assert!(caller_rx.try_recv().is_err());
15737
15738 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
15742 let counters = handler.counters().snapshot();
15743 assert_eq!(
15744 keyed_total(&counters, "route_open_accepted_by_principal"),
15745 1
15746 );
15747 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
15748 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
15749 }
15750
15751 #[tokio::test]
15755 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
15756 let mut swap = swap_with_incumbent().await;
15757 register_candidate(&swap, None);
15758 let incumbent = cutover(&swap);
15759 swap.forwarding
15760 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
15761 .unwrap()
15762 .expect("the incumbent is still registered");
15763
15764 let client = ConnectionId::new(33);
15765 let (client_ctx, mut client_rx) = route_ctx(client);
15766 let route_handler = swap.handler.clone();
15767 let open_ctx = RouteCtx {
15768 connection_id: client,
15769 egress: client_ctx.egress.clone(),
15770 };
15771 let mut route_task = tokio::spawn(async move {
15772 route_handler
15773 .handle_control_frame(
15774 &open_ctx,
15775 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
15776 )
15777 .await
15778 .unwrap()
15779 });
15780 let bind = tokio::select! {
15781 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
15782 response = &mut route_task => {
15783 let response = response.unwrap();
15784 panic!(
15785 "post-cutover route.open was refused instead of relayed to the candidate: {}",
15786 parse_error(&response[0])["code"]
15787 );
15788 }
15789 };
15790 swap.handler
15791 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
15792 .await
15793 .unwrap();
15794 assert!(route_task.await.unwrap().is_empty());
15795 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
15796 match swap
15797 .forwarding
15798 .lookup_data_route(client, channel, epoch)
15799 .unwrap()
15800 {
15801 DataRoute::Client(DataRouteState::Bound(route)) => {
15802 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
15803 }
15804 other => panic!("expected a bound route on the candidate, got {other:?}"),
15805 }
15806 assert!(swap.incumbent_rx.try_recv().is_err());
15807 }
15808
15809 #[tokio::test]
15814 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
15815 let swap = swap_with_incumbent().await;
15816 register_candidate(&swap, Some(false));
15817 let update = Frame::build(
15818 FrameType::Request,
15819 control_flags(),
15820 0,
15821 0,
15822 55,
15823 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
15824 provides: manifest("aft", PROTOCOL_VERSION).provides,
15825 capabilities: None,
15826 ready: Some(true),
15827 })
15828 .unwrap(),
15829 )
15830 .unwrap();
15831
15832 let replies = swap
15833 .handler
15834 .handle_control_frame(&swap.candidate_ctx, update)
15835 .await
15836 .unwrap();
15837
15838 assert_eq!(replies.len(), 1);
15839 assert_eq!(
15840 replies[0].header.ty,
15841 FrameType::Response,
15842 "candidate catalog.update was refused: {:?}",
15843 serde_json::from_slice::<Value>(&replies[0].body).ok()
15844 );
15845 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
15846 assert_eq!(
15847 swap.registry
15848 .get_module("aft")
15849 .unwrap()
15850 .unwrap()
15851 .connection_id,
15852 INCUMBENT
15853 );
15854 }
15855 }
15856
15857 mod swap_admission {
15861 use super::*;
15862
15863 const INCUMBENT_NONCE: &str = "incumbent-nonce";
15864 const CANDIDATE_NONCE: &str = "candidate-nonce";
15865
15866 fn handler_with_incumbent(
15867 module_id: &str,
15868 reserved: bool,
15869 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
15870 let registry = Arc::new(Registry::default());
15871 let supervisor = SupervisorHandle::new();
15872 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
15873 if reserved {
15874 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
15875 }
15876 let handler =
15877 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
15878 let incumbent = handler
15879 .handle_control(
15880 ConnectionId::new(1),
15881 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
15882 )
15883 .unwrap();
15884 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
15885 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
15886 (registry, supervisor, handler)
15887 }
15888
15889 #[test]
15895 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
15896 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
15897
15898 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
15899 let replies = handler
15900 .handle_control(
15901 ConnectionId::new(connection),
15902 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
15903 )
15904 .unwrap();
15905 assert_eq!(replies[0].header.ty, FrameType::Error);
15906 assert_eq!(
15907 parse_error(&replies[0])["code"],
15908 "swap_token_invalid",
15909 "nonce {nonce:?}"
15910 );
15911 }
15912 assert!(registry.get_candidate("aft").unwrap().is_none());
15913 assert_eq!(
15914 registry.get_module("aft").unwrap().unwrap().connection_id,
15915 ConnectionId::new(1)
15916 );
15917
15918 let admitted = handler
15921 .handle_control(
15922 ConnectionId::new(4),
15923 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
15924 )
15925 .unwrap();
15926 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
15927 assert_eq!(
15928 registry
15929 .get_candidate("aft")
15930 .unwrap()
15931 .unwrap()
15932 .connection_id,
15933 ConnectionId::new(4)
15934 );
15935 assert_eq!(
15936 registry.get_module("aft").unwrap().unwrap().connection_id,
15937 ConnectionId::new(1),
15938 "the candidate must not take the active slot"
15939 );
15940 let replayed = handler
15941 .handle_control(
15942 ConnectionId::new(5),
15943 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
15944 )
15945 .unwrap();
15946 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
15947
15948 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
15952 let squatter = handler
15953 .handle_control(
15954 ConnectionId::new(6),
15955 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
15956 )
15957 .unwrap();
15958 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
15959 assert!(
15960 registry.get_module("aft").unwrap().is_none(),
15961 "a squatter took the active slot of an id being swapped"
15962 );
15963 }
15964
15965 #[test]
15970 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
15971 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
15972
15973 let replies = handler
15974 .handle_control(
15975 ConnectionId::new(2),
15976 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15977 )
15978 .unwrap();
15979
15980 assert_eq!(
15981 replies[0].header.ty,
15982 FrameType::HelloAck,
15983 "reserved candidate refused: {:?}",
15984 serde_json::from_slice::<Value>(&replies[0].body).ok()
15985 );
15986 assert_eq!(
15987 registry
15988 .get_candidate("vault")
15989 .unwrap()
15990 .unwrap()
15991 .connection_id,
15992 ConnectionId::new(2)
15993 );
15994 }
15995
15996 #[test]
15999 fn without_an_open_swap_the_ordinary_gates_decide() {
16000 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
16001 supervisor.close_swap("vault");
16002
16003 let candidate = handler
16004 .handle_control(
16005 ConnectionId::new(2),
16006 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16007 )
16008 .unwrap();
16009 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
16010 let duplicate = handler
16011 .handle_control(
16012 ConnectionId::new(3),
16013 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
16014 )
16015 .unwrap();
16016 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
16017 assert!(registry.get_candidate("vault").unwrap().is_none());
16018 }
16019 }
16020
16021 mod scopes {
16025 mod idle_tests {
16026 use super::*;
16027 include!("scope_idle_tests.rs");
16028 }
16029 mod route_open_timings {
16030 use super::*;
16031 include!("route_open_timing_tests.rs");
16032 }
16033 mod route_open_refusal_log {
16034 use super::*;
16035 include!("route_open_refusal_log_tests.rs");
16036 }
16037 mod activity {
16038 include!("activity_scope_tests.rs");
16039 }
16040 mod delta {
16041 include!("scope_handler_delta_tests.rs");
16042 }
16043 mod call_modes {
16044 include!("scope_call_mode_tests.rs");
16045 }
16046 #[tokio::test]
16047 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
16048 let rig = rig().await;
16049 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
16050 "s".into(),
16051 1,
16052 "tool".into(),
16053 ))
16054 .unwrap();
16055 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
16056 let (direct, _rx) = wide_ctx(999);
16057 for (ctx, expected) in [
16058 (&rig.owner, FrameType::Response),
16059 (&direct, FrameType::Error),
16060 ] {
16061 let frame =
16062 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
16063 .unwrap();
16064 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
16065 assert_eq!(replies[0].header.ty, expected);
16066 if expected == FrameType::Response {
16067 let reply: subc_control::ScopeServedByReply =
16068 serde_json::from_slice(&replies[0].body).unwrap();
16069 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
16070 } else {
16071 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
16072 }
16073 }
16074 let reply = call(
16075 &rig.handler,
16076 &rig.owner,
16077 &ModuleControlRequestFromModule::ScopeDescribe {
16078 owner: Principal::Reserved {
16079 module_id: OWNER.into(),
16080 },
16081 scope_ref: "s".into(),
16082 },
16083 )
16084 .await;
16085 assert!(matches!(
16086 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
16087 ModuleControlResponseToModule::ScopeDescribe { .. }
16088 ));
16089 }
16090
16091 use subc_protocol::scope::{
16092 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
16093 ScopeStatus,
16094 };
16095
16096 use super::*;
16097
16098 const OWNER: &str = "prefrontal-core";
16099
16100 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
16101 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
16102 }
16103
16104 async fn call(
16105 handler: &ControlHandler,
16106 ctx: &RouteCtx,
16107 request: &ModuleControlRequestFromModule,
16108 ) -> Frame {
16109 let body = serde_json::to_vec(request).unwrap();
16110 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
16111 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
16112 assert_eq!(replies.len(), 1, "{replies:?}");
16113 replies.pop().unwrap()
16114 }
16115
16116 async fn sync(
16117 handler: &ControlHandler,
16118 ctx: &RouteCtx,
16119 generation: u64,
16120 scopes: Vec<ScopeRecord>,
16121 ) -> Result<ModuleControlResponseToModule, String> {
16122 let reply = call(
16123 handler,
16124 ctx,
16125 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
16126 )
16127 .await;
16128 match reply.header.ty {
16129 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
16130 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
16131 }
16132 }
16133
16134 async fn describe(
16135 handler: &ControlHandler,
16136 ctx: &RouteCtx,
16137 owner: &str,
16138 scope_ref: &str,
16139 ) -> ModuleControlResponseToModule {
16140 let reply = call(
16141 handler,
16142 ctx,
16143 &ModuleControlRequestFromModule::ScopeDescribe {
16144 owner: Principal::Reserved {
16145 module_id: owner.to_string(),
16146 },
16147 scope_ref: scope_ref.to_string(),
16148 },
16149 )
16150 .await;
16151 assert_eq!(
16152 reply.header.ty,
16153 FrameType::Response,
16154 "{:?}",
16155 parse_error(&reply)
16156 );
16157 serde_json::from_slice(&reply.body).unwrap()
16158 }
16159
16160 #[tokio::test]
16161 async fn scope_describe_never_echoes_bind_proof_or_hash() {
16162 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16163 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16164 let mut rig = rig().await;
16165 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16166 .await;
16167 let _bound = rig
16168 .bound(
16169 Some(AFT),
16170 PLEXUS,
16171 Some(
16172 rig.selector("s", Some(1))
16173 .with_bind_proof(Some(PROOF.into())),
16174 ),
16175 )
16176 .await;
16177 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
16178 let output = serde_json::to_string(&reply).unwrap();
16179 assert!(output.contains("\"status\":\"live\""));
16180 assert!(!output.contains(PROOF));
16181 assert!(!output.contains(HASH));
16182 }
16183
16184 async fn module(
16186 handler: &ControlHandler,
16187 connection: u64,
16188 module_id: &str,
16189 nonce: Option<&str>,
16190 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16191 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
16192 hello_via_sink(
16193 handler,
16194 &ctx,
16195 &mut rx,
16196 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
16197 )
16198 .await;
16199 (ctx, rx)
16200 }
16201
16202 #[tokio::test]
16205 async fn a_client_connection_cannot_sync_or_describe() {
16206 let handler = ControlHandler::new(Arc::new(Registry::default()));
16207 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
16208 for request in [
16209 ModuleControlRequestFromModule::ScopeSync {
16210 generation: 1,
16211 scopes: vec![head("s", 1)],
16212 },
16213 ModuleControlRequestFromModule::ScopeDescribe {
16214 owner: Principal::Direct,
16215 scope_ref: "s".to_string(),
16216 },
16217 ] {
16218 let reply = call(&handler, &ctx, &request).await;
16219 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
16220 }
16221 assert!(
16222 !handler
16223 .scopes
16224 .read()
16225 .unwrap()
16226 .describe(
16227 &Principal::Reserved {
16228 module_id: OWNER.to_string()
16229 },
16230 "s"
16231 )
16232 .owner_synced
16233 );
16234 }
16235
16236 #[tokio::test]
16239 async fn a_module_without_a_supervised_launch_cannot_sync() {
16240 let handler = ControlHandler::new(Arc::new(Registry::default()));
16241 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
16242 assert_eq!(
16243 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
16244 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16245 );
16246 }
16247
16248 #[tokio::test]
16249 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
16250 let supervisor = SupervisorHandle::new();
16251 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16252 let handler = ControlHandler::new(Arc::new(Registry::default()))
16253 .with_supervisor(supervisor.clone());
16254 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16255 sync(&handler, &incumbent, 1, vec![head("s", 1)])
16256 .await
16257 .expect("the current launch syncs");
16258
16259 supervisor.open_swap(OWNER, "n2".to_string());
16262 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
16263 assert_eq!(
16264 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16265 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16266 );
16267 sync(&handler, &incumbent, 2, vec![head("s", 1)])
16268 .await
16269 .expect("the serving owner syncs during the swap");
16270
16271 supervisor.close_swap(OWNER);
16274 assert_eq!(
16275 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16276 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16277 );
16278 sync(&handler, &incumbent, 3, vec![head("s", 1)])
16279 .await
16280 .expect("the serving owner syncs after the rollback");
16281 handler.cleanup_connection(candidate.connection_id).unwrap();
16282
16283 supervisor.open_swap(OWNER, "n3".to_string());
16288 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
16289 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
16290 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
16291 .await
16292 .expect("the promoted launch takes authority");
16293 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16294 panic!("unexpected reply {reply:?}");
16295 };
16296 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
16297 assert_eq!(
16298 sync(&handler, &incumbent, 4, Vec::new()).await,
16299 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16300 );
16301 }
16302
16303 #[tokio::test]
16306 async fn closing_the_authority_connection_frees_sync_authority() {
16307 let supervisor = SupervisorHandle::new();
16308 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16309 let handler = ControlHandler::new(Arc::new(Registry::default()))
16310 .with_supervisor(supervisor.clone());
16311 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16312 sync(&handler, &first, 10, vec![head("s", 1)])
16313 .await
16314 .unwrap();
16315 handler.cleanup_connection(first.connection_id).unwrap();
16316
16317 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
16318 sync(&handler, &second, 1, vec![head("s", 1)])
16319 .await
16320 .expect("the next connection takes the released authority");
16321 }
16322
16323 #[tokio::test]
16324 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
16325 let supervisor = SupervisorHandle::new();
16326 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16327 let handler =
16328 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
16329 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
16330 sync(&handler, &first, 10, vec![head("s", 1)])
16331 .await
16332 .unwrap();
16333 handler
16334 .handle_control_frame(
16335 &first,
16336 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
16337 )
16338 .await
16339 .unwrap();
16340 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
16341 sync(&handler, &second, 1, vec![head("s", 1)])
16342 .await
16343 .expect("GOODBYE releases authority even if the old socket remains open");
16344 }
16345
16346 #[tokio::test]
16347 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
16348 let registry = Arc::new(Registry::default());
16349 let supervisor_handle = SupervisorHandle::new();
16350 let supervisor =
16351 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
16352 .with_handle(supervisor_handle.clone())
16353 .with_daemon_incarnation("incarnation-7".to_string());
16354 supervisor
16357 .supervise_configured(
16358 ModuleSpec {
16359 module_id: OWNER.to_string(),
16360 program: PathBuf::from("/nonexistent/prefrontal-core"),
16361 args: Vec::new(),
16362 env: Vec::new(),
16363 reserved: false,
16364 reserved_prefixes: Vec::new(),
16365 protocol: ModuleProtocol::Subc,
16366 overlap: Default::default(),
16367 },
16368 false,
16369 )
16370 .unwrap();
16371 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
16372 let handler =
16373 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
16374 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
16375
16376 let ModuleControlResponseToModule::ScopeDescribe {
16378 status,
16379 daemon_incarnation,
16380 owner_synced,
16381 owner_configured,
16382 scope,
16383 ..
16384 } = describe(&handler, &reader, OWNER, "s").await
16385 else {
16386 panic!("not a describe reply");
16387 };
16388 assert_eq!(status, ScopeStatus::NotLive);
16389 assert_eq!(daemon_incarnation, "incarnation-7");
16390 assert!(!owner_synced);
16391 assert!(owner_configured);
16392 assert!(scope.is_none());
16393
16394 let ModuleControlResponseToModule::ScopeDescribe {
16397 status,
16398 owner_configured,
16399 ..
16400 } = describe(&handler, &reader, "ghost", "s").await
16401 else {
16402 panic!("not a describe reply");
16403 };
16404 assert_eq!(status, ScopeStatus::NotLive);
16405 assert!(!owner_configured);
16406
16407 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
16409 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
16410 let ModuleControlResponseToModule::ScopeDescribe {
16411 status,
16412 scope_epoch,
16413 owner_synced,
16414 scope,
16415 ..
16416 } = describe(&handler, &reader, OWNER, "s").await
16417 else {
16418 panic!("not a describe reply");
16419 };
16420 assert_eq!(status, ScopeStatus::Live);
16421 assert_eq!(scope_epoch, Some(4));
16422 assert!(owner_synced);
16423 let stamp = scope.expect("a live scope carries its stamp");
16424 assert!(
16425 stamp.owner_authorized,
16426 "prefrontal-core is the default authority"
16427 );
16428 assert_eq!(stamp.kind, ScopeKind::Head);
16429 }
16430
16431 #[tokio::test]
16432 async fn scope_authority_owners_decides_owner_authorized() {
16433 let supervisor = SupervisorHandle::new();
16434 supervisor.set_spawn_nonce("broca", "b1".to_string());
16435 let handler = ControlHandler::new(Arc::new(Registry::default()))
16436 .with_supervisor(supervisor)
16437 .with_scope_authority_owners(vec!["broca".to_string()]);
16438 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
16439 let mut gated = head("s", 1);
16440 gated.attributes.agent_id = Some("agent".to_string());
16441 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
16442 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
16443 describe(&handler, &broca, "broca", "s").await
16444 else {
16445 panic!("not a describe reply");
16446 };
16447 assert!(scope.unwrap().owner_authorized);
16448 }
16449
16450 #[tokio::test]
16454 async fn scope_ops_and_the_scopes_capability_are_advertised() {
16455 let handler = ControlHandler::new(Arc::new(Registry::default()));
16456 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
16457 let ack = hello_via_sink(
16458 &handler,
16459 &ctx,
16460 &mut rx,
16461 hello_frame("m", PROTOCOL_VERSION, 1),
16462 )
16463 .await;
16464 let ack = parse_ack(&ack);
16465 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
16466 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
16467 }
16468 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
16469
16470 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
16471 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16472 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16473 let reply = handler
16474 .handle_control_frame(&client, frame)
16475 .await
16476 .unwrap()
16477 .pop()
16478 .unwrap();
16479 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16480 serde_json::from_slice(&reply.body).unwrap()
16481 else {
16482 panic!("not a server.describe reply");
16483 };
16484 assert!(
16485 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
16486 "{capabilities:?}"
16487 );
16488 }
16489
16490 const PLEXUS: &str = "plexus";
16493 const OTHER: &str = "other";
16494 const AFT: &str = "aft";
16495 const BROCA: &str = "broca";
16496 const MAGIC: &str = "magic-context";
16497
16498 fn nonce(module_id: &str) -> String {
16499 format!("nonce-{module_id}")
16500 }
16501
16502 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16503 let (tx, rx) = mpsc::channel(64);
16504 (
16505 RouteCtx {
16506 connection_id: ConnectionId::new(connection),
16507 egress: FrameSink::new(tx),
16508 },
16509 rx,
16510 )
16511 }
16512
16513 struct Rig {
16517 handler: ControlHandler,
16518 forwarding: Arc<ForwardingTable>,
16519 owner: RouteCtx,
16520 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16521 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
16522 generation: u64,
16523 next_connection: u64,
16524 _supervisor: Supervisor,
16525 }
16526
16527 async fn rig() -> Rig {
16528 rig_with_flow_support(true).await
16529 }
16530
16531 async fn rig_with_flow_support(flow_support: bool) -> Rig {
16532 rig_with_scope_capabilities(flow_support, false).await
16533 }
16534
16535 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
16536 let mut provides = Vec::new();
16537 if flow_support {
16538 provides.push("flow-scopes/v1");
16539 }
16540 if run_support {
16541 provides.push("agent-run-scopes/v1");
16542 }
16543 rig_with_target_capabilities(&provides).await
16544 }
16545
16546 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
16547 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
16548 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16549 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16552 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
16553 Frame::build(
16554 FrameType::Hello,
16555 control_flags(),
16556 0,
16557 0,
16558 connection,
16559 serde_json::to_vec(&body).unwrap(),
16560 )
16561 .unwrap()
16562 }
16563
16564 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
16565 let registry = Arc::new(Registry::default());
16566 let forwarding = Arc::new(ForwardingTable::default());
16567 let supervisor_handle = SupervisorHandle::new();
16568 let supervisor =
16569 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
16570 .with_handle(supervisor_handle.clone());
16571 supervisor
16572 .supervise_configured(
16573 ModuleSpec {
16574 module_id: OWNER.to_string(),
16575 program: PathBuf::from("/nonexistent/prefrontal-core"),
16576 args: Vec::new(),
16577 env: Vec::new(),
16578 reserved: false,
16579 reserved_prefixes: Vec::new(),
16580 protocol: ModuleProtocol::Subc,
16581 overlap: Default::default(),
16582 },
16583 false,
16584 )
16585 .unwrap();
16586 for module_id in [OWNER, AFT, BROCA, MAGIC] {
16587 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
16588 }
16589 let handler =
16590 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16591 .with_supervisor(supervisor_handle);
16592 let (owner, mut owner_rx) = wide_ctx(1);
16593 hello_via_sink(
16594 &handler,
16595 &owner,
16596 &mut owner_rx,
16597 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
16598 )
16599 .await;
16600 let mut modules = BTreeMap::new();
16601 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
16602 let (ctx, mut rx) = wide_ctx(connection);
16603 let hello = hello_with_capabilities(module_id, connection, provides);
16604 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
16605 modules.insert(module_id.to_string(), (ctx, rx));
16606 }
16607 Rig {
16608 handler,
16609 forwarding,
16610 owner,
16611 _owner_rx: owner_rx,
16612 modules,
16613 generation: 0,
16614 next_connection: 100,
16615 _supervisor: supervisor,
16616 }
16617 }
16618
16619 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
16620 ScopeCarrier::new(Principal::Reserved {
16621 module_id: module_id.to_string(),
16622 })
16623 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
16624 }
16625
16626 fn session(scope_epoch: u64) -> ScopeRecord {
16629 let mut record = head("s", scope_epoch);
16630 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
16631 record.attributes.agent_id = Some("agent-1".to_string());
16632 record.attributes.delegates = true;
16633 record
16634 }
16635
16636 impl Rig {
16637 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
16638 self.generation += 1;
16639 sync(&self.handler, &self.owner, self.generation, scopes)
16640 .await
16641 .expect("the owner's sync is accepted");
16642 }
16643
16644 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16645 ScopeSelector::new(
16646 Principal::Reserved {
16647 module_id: OWNER.to_string(),
16648 },
16649 scope_ref.to_string(),
16650 scope_epoch,
16651 )
16652 }
16653
16654 fn open_frame(
16655 &mut self,
16656 opener: Option<&str>,
16657 target: &str,
16658 scope: Option<ScopeSelector>,
16659 ) -> (
16660 RouteCtx,
16661 mpsc::Receiver<crate::router::OutboundFrame>,
16662 Frame,
16663 ) {
16664 self.next_connection += 1;
16665 let (ctx, rx) = wide_ctx(self.next_connection);
16666 let root = unique_project_root("scoped-open");
16667 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
16668 target: RouteTarget::ToolProvider {
16669 module_id: target.to_string(),
16670 },
16671 identity: BindIdentity::new(
16672 root.path().to_path_buf(),
16673 "unit".to_string(),
16674 "session".to_string(),
16675 ),
16676 consumer_identity: opener.map(|module_id| ConsumerIdentity {
16677 module_id: module_id.to_string(),
16678 launch_nonce: nonce(module_id),
16679 }),
16680 consumer_capabilities: None,
16681 role_versions: None,
16682 admission_facts: None,
16683 scope,
16684 })
16685 .unwrap();
16686 let frame = Frame::build(
16687 FrameType::Request,
16688 control_flags(),
16689 0,
16690 0,
16691 self.next_connection,
16692 body,
16693 )
16694 .unwrap();
16695 (ctx, rx, frame)
16696 }
16697
16698 async fn refused(
16700 &mut self,
16701 opener: Option<&str>,
16702 target: &str,
16703 scope: Option<ScopeSelector>,
16704 ) -> String {
16705 self.refusal_body(opener, target, scope).await["code"]
16706 .as_str()
16707 .unwrap()
16708 .to_string()
16709 }
16710
16711 async fn refusal_body(
16712 &mut self,
16713 opener: Option<&str>,
16714 target: &str,
16715 scope: Option<ScopeSelector>,
16716 ) -> Value {
16717 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
16718 let replies = tokio::time::timeout(
16719 Duration::from_secs(2),
16720 self.handler.handle_control_frame(&ctx, frame),
16721 )
16722 .await
16723 .expect("the open must be refused before waiting for a bind ack")
16724 .unwrap();
16725 assert_eq!(replies.len(), 1, "{replies:?}");
16726 assert_eq!(replies[0].header.ty, FrameType::Error);
16727 let (_, module_rx) = self.modules.get_mut(target).unwrap();
16728 assert!(
16729 module_rx.try_recv().is_err(),
16730 "a refused open relays nothing"
16731 );
16732 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
16733 parse_error(&replies[0])
16734 }
16735
16736 async fn relayed(
16738 &mut self,
16739 opener: Option<&str>,
16740 target: &str,
16741 scope: Option<ScopeSelector>,
16742 ) -> Relayed {
16743 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
16744 let handler = self.handler.clone();
16745 let task_ctx = ctx.clone();
16746 let task = tokio::spawn(async move {
16747 handler
16748 .handle_control_frame(&task_ctx, frame)
16749 .await
16750 .unwrap()
16751 });
16752 let (_, module_rx) = self.modules.get_mut(target).unwrap();
16753 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
16754 .await
16755 .expect("the target receives the relayed route.bind")
16756 .unwrap()
16757 .frame;
16758 Relayed {
16759 target: target.to_string(),
16760 client: ctx,
16761 client_rx: rx,
16762 task,
16763 bind,
16764 }
16765 }
16766
16767 async fn ack(&self, relayed: &Relayed) {
16768 let (module, _) = &self.modules[&relayed.target];
16769 self.handler
16770 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
16771 .await
16772 .unwrap();
16773 }
16774
16775 async fn bound(
16777 &mut self,
16778 opener: Option<&str>,
16779 target: &str,
16780 scope: Option<ScopeSelector>,
16781 ) -> Bound {
16782 let relayed = self.relayed(opener, target, scope).await;
16783 self.ack(&relayed).await;
16784 let Relayed {
16785 target,
16786 client,
16787 mut client_rx,
16788 task,
16789 bind,
16790 } = relayed;
16791 assert!(
16792 task.await.unwrap().is_empty(),
16793 "the open is answered by commit"
16794 );
16795 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
16796 Bound {
16797 target,
16798 client,
16799 client_rx,
16800 channel,
16801 epoch,
16802 bind,
16803 }
16804 }
16805
16806 fn live(&self, route: &Bound) -> bool {
16807 matches!(
16808 self.forwarding
16809 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16810 .unwrap(),
16811 DataRoute::Client(DataRouteState::Bound(_))
16812 )
16813 }
16814 }
16815
16816 struct Relayed {
16817 target: String,
16818 client: RouteCtx,
16819 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16820 task: tokio::task::JoinHandle<Vec<Frame>>,
16821 bind: Frame,
16822 }
16823
16824 struct Bound {
16825 target: String,
16826 client: RouteCtx,
16827 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16828 channel: u16,
16829 epoch: u32,
16830 bind: Frame,
16831 }
16832
16833 impl Bound {
16834 fn closed_reason(&mut self) -> RouteCloseReason {
16837 let mut reason = None;
16838 let mut goodbye = false;
16839 while let Ok(outbound) = self.client_rx.try_recv() {
16840 let frame = outbound.frame;
16841 match frame.header.ty {
16842 FrameType::Goodbye => {
16843 assert_eq!(
16844 (frame.header.channel, frame.header.epoch),
16845 (self.channel, self.epoch)
16846 );
16847 goodbye = true;
16848 }
16849 FrameType::Push => {
16850 let ClientControlPush::RouteClosed {
16851 reason: r,
16852 module_id,
16853 ..
16854 } = serde_json::from_slice(&frame.body).unwrap()
16855 else {
16856 panic!("unexpected push");
16857 };
16858 assert_eq!(module_id, self.target);
16859 reason = Some(r);
16860 }
16861 other => panic!("unexpected frame {other:?}"),
16862 }
16863 }
16864 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
16865 reason.expect("the client is told why the route closed")
16866 }
16867
16868 fn untouched(&mut self) -> bool {
16869 self.client_rx.try_recv().is_err()
16870 }
16871
16872 fn stamp(&self) -> Option<ScopeStamp> {
16873 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
16874 ModuleControlRequest::RouteBind { scope, .. } => scope,
16875 other => panic!("expected a route.bind, got {other:?}"),
16876 }
16877 }
16878 }
16879
16880 #[tokio::test]
16881 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
16882 ) {
16883 let mut rig = rig().await;
16884 let mut record = session(1);
16885 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
16886 record.child_owners = vec![Principal::Reserved {
16887 module_id: MAGIC.to_string(),
16888 }];
16889 rig.sync(vec![record]).await;
16890 let scope = || Some(rig_selector("s", Some(1)));
16891
16892 rig.bound(Some(OWNER), PLEXUS, scope()).await;
16895 rig.bound(Some(AFT), OTHER, scope()).await;
16896 rig.bound(Some(BROCA), PLEXUS, scope()).await;
16897
16898 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
16902 assert_eq!(
16903 rig.refused(opener, target, scope()).await,
16904 error_codes::SCOPE_NOT_CARRIER,
16905 "{opener:?} -> {target}"
16906 );
16907 }
16908 }
16909
16910 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16911 ScopeSelector::new(
16912 Principal::Reserved {
16913 module_id: OWNER.to_string(),
16914 },
16915 scope_ref.to_string(),
16916 scope_epoch,
16917 )
16918 }
16919
16920 #[tokio::test]
16921 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
16922 let mut rig = rig().await;
16923 rig.sync(vec![session(1)]).await;
16924 for opener in [OWNER, AFT] {
16925 assert_eq!(
16926 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
16927 .await,
16928 error_codes::SCOPE_EPOCH_REQUIRED,
16929 "{opener}"
16930 );
16931 }
16932 }
16933
16934 #[tokio::test]
16935 async fn admission_separates_not_synced_not_live_and_ended() {
16936 let mut rig = rig().await;
16937 let code = rig
16939 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16940 .await;
16941 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
16942 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
16943
16944 let ghost = ScopeSelector::new(
16946 Principal::Reserved {
16947 module_id: "ghost".to_string(),
16948 },
16949 "s".to_string(),
16950 Some(1),
16951 );
16952 assert_eq!(
16953 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
16954 error_codes::SCOPE_NOT_LIVE
16955 );
16956
16957 rig.sync(vec![session(2)]).await;
16958 assert_eq!(
16959 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
16960 .await,
16961 error_codes::SCOPE_NOT_LIVE
16962 );
16963 for epoch in [1, 3] {
16964 assert_eq!(
16965 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
16966 .await,
16967 error_codes::SCOPE_ENDED,
16968 "epoch {epoch}"
16969 );
16970 }
16971 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
16973 .await;
16974 }
16975
16976 #[tokio::test]
16977 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
16978 let mut rig = rig().await;
16979 rig.sync(vec![session(1)]).await;
16980 let route = rig
16981 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16982 .await;
16983 let stamp = route.stamp().expect("a scoped bind carries the stamp");
16984 assert_eq!(stamp.scope_ref, "s");
16985 assert_eq!(stamp.scope_epoch, 1);
16986 assert_eq!(stamp.kind, ScopeKind::Head);
16987 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
16988 assert!(stamp.attributes.delegates);
16989 assert!(stamp.owner_authorized);
16990 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16991 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
16992
16993 let (broca, mut broca_rx) = wide_ctx(50);
16996 hello_via_sink(
16997 &rig.handler,
16998 &broca,
16999 &mut broca_rx,
17000 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
17001 )
17002 .await;
17003 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
17004 .await
17005 .unwrap();
17006 let own = ScopeSelector::new(
17007 Principal::Reserved {
17008 module_id: BROCA.to_string(),
17009 },
17010 "b".to_string(),
17011 Some(1),
17012 );
17013 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
17014 assert!(!route.stamp().unwrap().owner_authorized);
17015 }
17016
17017 #[tokio::test]
17018 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
17019 let mut rig = rig().await;
17020 let mut record = head("s", 1);
17021 record.carriers = vec![carrier(AFT, None)];
17022 let flow_id = "Flow:run-7/step_2!~";
17023 record.attributes.flow_id = Some(flow_id.to_string());
17024 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
17025 .await
17026 .unwrap();
17027 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17028 panic!("not a sync reply");
17029 };
17030 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
17031 let route = rig
17032 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17033 .await;
17034 assert!(rig.live(&route), "the stamped bind committed");
17035 let stamp = route.stamp().expect("a flow scope carries a stamp");
17036 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
17037 assert_eq!(stamp.attributes.agent_id, None);
17038 assert!(!stamp.attributes.delegates);
17039 assert!(stamp.owner_authorized);
17040 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17041 assert_eq!(unscoped.stamp(), None);
17042 }
17043
17044 #[tokio::test]
17045 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
17046 let mut rig = rig_with_flow_support(false).await;
17047 let mut record = session(1);
17048 record.attributes.flow_id = Some("flow:7".to_string());
17049 rig.sync(vec![record]).await;
17050 for opener in [OWNER, AFT] {
17051 let body = rig
17052 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17053 .await;
17054 assert_eq!(body["code"], "target_flow_unsupported");
17055 let message = body["message"].as_str().unwrap();
17056 for required in [PLEXUS, "flow-scopes/v1"] {
17057 assert!(message.contains(required), "{message}");
17058 }
17059 }
17060 }
17061
17062 #[tokio::test]
17063 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
17064 let mut rig = rig_with_flow_support(true).await;
17065 let mut record = session(1);
17066 record.attributes.flow_id = Some("flow:7".to_string());
17067 rig.sync(vec![record]).await;
17068 for opener in [OWNER, AFT] {
17069 let route = rig
17070 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17071 .await;
17072 assert!(rig.live(&route));
17073 assert_eq!(
17074 route.stamp().unwrap().attributes.flow_id.as_deref(),
17075 Some("flow:7")
17076 );
17077 }
17078 }
17079
17080 #[tokio::test]
17081 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
17082 use std::future::Future;
17083
17084 let mut rig = rig_with_flow_support(true).await;
17085 let mut record = session(1);
17086 record.attributes.flow_id = Some("flow:7".to_string());
17087 rig.sync(vec![record]).await;
17088 let (client, mut client_rx, frame) =
17089 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
17090 for _ in 0..64 {
17093 client.egress.try_send(route_bind_ack(1)).unwrap();
17094 }
17095 let handler = rig.handler.clone();
17096 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17097 std::future::poll_fn(|cx| {
17098 assert!(open.as_mut().poll(cx).is_pending());
17099 std::task::Poll::Ready(())
17100 })
17101 .await;
17102 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17103
17104 let old_connection = rig.modules[PLEXUS].0.connection_id;
17105 rig.handler.cleanup_connection(old_connection).unwrap();
17106 let (replacement, mut replacement_rx) = wide_ctx(200);
17107 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
17108 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17109 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17110 let hello = Frame::build(
17111 FrameType::Hello,
17112 control_flags(),
17113 0,
17114 0,
17115 200,
17116 serde_json::to_vec(&body).unwrap(),
17117 )
17118 .unwrap();
17119 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
17120
17121 client_rx.try_recv().unwrap();
17122 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17123 .await
17124 .expect("the replacement is refused without waiting for a bind ack")
17125 .unwrap();
17126 assert_eq!(replies.len(), 1);
17127 let body = parse_error(&replies[0]);
17128 assert_eq!(body["code"], "target_flow_unsupported");
17129 for required in [PLEXUS, "flow-scopes/v1"] {
17130 assert!(body["message"].as_str().unwrap().contains(required));
17131 }
17132 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
17133 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17134 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17135 assert!(rig
17136 .handler
17137 .registry
17138 .get_module_by_connection(replacement.connection_id)
17139 .unwrap()
17140 .is_some());
17141 }
17142
17143 #[tokio::test]
17144 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
17145 use std::future::Future;
17146 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17147 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17148 const NEXT_HASH: &str =
17149 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17150 let mut rig = rig().await;
17151 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
17152 record.carriers.push(ScopeCarrier::new(Principal::Direct));
17153 rig.sync(vec![record.clone()]).await;
17154 let (client, mut rx, frame) = rig.open_frame(
17155 None,
17156 PLEXUS,
17157 Some(
17158 rig.selector("s", Some(1))
17159 .with_bind_proof(Some(PROOF.into())),
17160 ),
17161 );
17162 for _ in 0..64 {
17165 client.egress.try_send(route_bind_ack(1)).unwrap();
17166 }
17167 let handler = rig.handler.clone();
17168 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17169 std::future::poll_fn(|cx| {
17170 assert!(open.as_mut().poll(cx).is_pending());
17171 std::task::Poll::Ready(())
17172 })
17173 .await;
17174 record.bind_proof_sha256 = Some(NEXT_HASH.into());
17175 rig.sync(vec![record]).await;
17176 rx.try_recv().unwrap();
17177 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17178 .await
17179 .unwrap()
17180 .unwrap();
17181 assert_eq!(
17182 parse_error(&replies[0])["code"],
17183 error_codes::SCOPE_BIND_PROOF_MISMATCH
17184 );
17185 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
17186 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17187 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17188 }
17189
17190 #[tokio::test]
17191 async fn scope_bind_proof_reserved_openers_ignore_proof() {
17192 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17193 let mut rig = rig().await;
17194 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17195 .await;
17196 for opener in [OWNER, AFT] {
17197 for proof in [None, Some("malformed+proof=")] {
17198 let route = rig
17199 .bound(
17200 Some(opener),
17201 PLEXUS,
17202 Some(
17203 rig.selector("s", Some(1))
17204 .with_bind_proof(proof.map(str::to_string)),
17205 ),
17206 )
17207 .await;
17208 assert!(rig.live(&route));
17209 }
17210 }
17211 }
17212
17213 #[tokio::test]
17214 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
17215 use std::future::Future;
17216 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17217 const NEXT_HASH: &str =
17218 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17219 let mut rig = rig().await;
17220 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17221 .await;
17222 let (client, mut rx, frame) =
17223 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
17224 for _ in 0..64 {
17225 client.egress.try_send(route_bind_ack(1)).unwrap();
17226 }
17227 let handler = rig.handler.clone();
17228 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17229 std::future::poll_fn(|cx| {
17230 assert!(open.as_mut().poll(cx).is_pending());
17231 std::task::Poll::Ready(())
17232 })
17233 .await;
17234 rig.sync(vec![
17235 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
17236 ])
17237 .await;
17238 rx.try_recv().unwrap();
17239 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17240 let bind = std::future::poll_fn(|cx| {
17241 assert!(open.as_mut().poll(cx).is_pending());
17242 module_rx.poll_recv(cx)
17243 })
17244 .await
17245 .unwrap()
17246 .frame;
17247 handler
17248 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
17249 .await
17250 .unwrap();
17251 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17252 .await
17253 .unwrap()
17254 .unwrap();
17255 assert_eq!(replies.len(), 1);
17256 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
17259 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17260 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
17261 let route = rig
17262 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
17263 .await;
17264 assert!(rig.live(&route));
17265 }
17266
17267 #[tokio::test]
17268 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
17269 {
17270 let mut rig = rig_with_flow_support(false).await;
17271 rig.sync(vec![session(1)]).await;
17272 let route = rig
17273 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17274 .await;
17275 assert!(rig.live(&route));
17276 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
17277 let mut record = session(1);
17278 record.attributes.flow_id = Some("flow:7".to_string());
17279 rig.sync(vec![record]).await;
17280 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
17281 assert!(rig.live(&unscoped));
17282 assert_eq!(unscoped.stamp(), None);
17283 }
17284
17285 #[tokio::test]
17286 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
17287 let mut rig = rig().await;
17288 let mut record = session(1);
17289 record.attributes.flow_id = Some("flow:7".to_string());
17290 rig.sync(vec![record.clone()]).await;
17291 let before = rig.forwarding.published_scope_tag(OWNER, "s");
17292 let mut owner_route = rig
17293 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17294 .await;
17295 let mut carrier_route = rig
17296 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
17297 .await;
17298 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17299 rig.sync(vec![record.clone()]).await;
17300 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17301 assert!(rig.live(&owner_route) && owner_route.untouched());
17302 assert!(rig.live(&carrier_route) && carrier_route.untouched());
17303
17304 record.attributes.flow_id = Some("flow:8".to_string());
17305 rig.sync(vec![record]).await;
17306 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
17307 let before = before.unwrap();
17308 assert_eq!(after.scope_epoch, before.scope_epoch);
17309 assert!(after.version > before.version);
17310 for route in [&mut owner_route, &mut carrier_route] {
17311 assert!(!rig.live(route));
17312 assert_eq!(
17313 route.closed_reason(),
17314 RouteCloseReason::ScopeDelegationChanged
17315 );
17316 }
17317 assert!(rig.live(&unscoped) && unscoped.untouched());
17318 for target in [PLEXUS, OTHER] {
17321 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
17322 let goodbye = module_rx
17323 .try_recv()
17324 .expect("the provider sees the drain")
17325 .frame;
17326 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17327 assert!(module_rx.try_recv().is_err());
17328 }
17329 let rebound = rig
17330 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17331 .await;
17332 assert_eq!(
17333 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
17334 Some("flow:8")
17335 );
17336 }
17337
17338 #[tokio::test]
17343 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
17344 let mut rig = rig().await;
17345 rig.sync(vec![session(1)]).await;
17346 let mut cotenant = rig
17347 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17348 .await;
17349
17350 let mut changed = session(1);
17351 changed.child_owners.push(Principal::Reserved {
17352 module_id: MAGIC.to_string(),
17353 });
17354 let mut ended = None;
17355 for (code, next) in [
17356 (error_codes::SCOPE_CHANGED, vec![changed]),
17357 (error_codes::SCOPE_ENDED, Vec::new()),
17358 ] {
17359 let relayed = rig
17360 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17361 .await;
17362 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
17363 ended = Some(next.is_empty());
17364 rig.sync(next).await;
17365 rig.ack(&relayed).await;
17366 let replies = relayed.task.await.unwrap();
17367 assert_eq!(replies.len(), 1, "{replies:?}");
17368 assert_eq!(parse_error(&replies[0])["code"], code);
17369 assert_eq!(
17370 subc_protocol::error_codes::is_retryable_route_open(code),
17371 code == error_codes::SCOPE_CHANGED
17372 );
17373 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17374 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17376 let mut goodbyes = Vec::new();
17379 while let Ok(outbound) = plexus_rx.try_recv() {
17380 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
17381 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
17382 }
17383 assert!(
17384 goodbyes.contains(&(bind_channel, bind_epoch)),
17385 "{goodbyes:?}"
17386 );
17387 assert!(rig
17388 .handler
17389 .registry
17390 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
17391 .unwrap()
17392 .is_some());
17393 }
17394 assert_eq!(ended, Some(true));
17395 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
17398 }
17399
17400 #[tokio::test]
17403 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
17404 struct Case {
17405 name: &'static str,
17406 change: fn(&mut ScopeRecord),
17407 closed: [Option<RouteCloseReason>; 4],
17410 }
17411 use RouteCloseReason::*;
17412 let cases = [
17413 Case {
17414 name: "a carrier entry removed",
17415 change: |r| {
17416 r.carriers.retain(|c| {
17417 c.principal
17418 != Principal::Reserved {
17419 module_id: AFT.to_string(),
17420 }
17421 })
17422 },
17423 closed: [None, Some(ScopeCarrierRemoved), None, None],
17424 },
17425 Case {
17426 name: "a target removed from a carrier",
17427 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
17428 closed: [None, None, None, Some(ScopeCarrierRemoved)],
17429 },
17430 Case {
17431 name: "a bare carrier narrowed to targets",
17432 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
17433 closed: [None, Some(ScopeCarrierRemoved), None, None],
17434 },
17435 Case {
17436 name: "delegates turned off",
17437 change: |r| r.attributes.delegates = false,
17438 closed: [Some(ScopeDelegationChanged); 4],
17439 },
17440 Case {
17441 name: "agent_id changed",
17442 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
17443 closed: [Some(ScopeDelegationChanged); 4],
17444 },
17445 Case {
17446 name: "a carrier added, child owners changed, the record re-sent",
17447 change: |r| {
17448 r.carriers.push(carrier(MAGIC, None));
17449 r.child_owners.push(Principal::Reserved {
17450 module_id: MAGIC.to_string(),
17451 });
17452 },
17453 closed: [None; 4],
17454 },
17455 Case {
17456 name: "a target added",
17457 change: |r| {
17458 r.carriers[1]
17459 .targets
17460 .as_mut()
17461 .unwrap()
17462 .push("third".to_string())
17463 },
17464 closed: [None; 4],
17465 },
17466 Case {
17467 name: "delegates turned on",
17468 change: |r| r.attributes.delegates = true,
17469 closed: [None; 4],
17470 },
17471 ];
17472 for case in cases {
17473 let mut rig = rig().await;
17474 rig.sync(vec![session(1)]).await;
17475 let scope = || Some(rig_selector("s", Some(1)));
17476 let mut routes = [
17477 rig.bound(Some(OWNER), PLEXUS, scope()).await,
17478 rig.bound(Some(AFT), PLEXUS, scope()).await,
17479 rig.bound(Some(BROCA), PLEXUS, scope()).await,
17480 rig.bound(Some(BROCA), OTHER, scope()).await,
17481 ];
17482 let mut record = session(1);
17483 (case.change)(&mut record);
17484 rig.sync(vec![record]).await;
17485 for (index, expected) in case.closed.iter().enumerate() {
17486 let route = &mut routes[index];
17487 match expected {
17488 Some(reason) => {
17489 assert!(!rig.live(route), "{}: route {index} still live", case.name);
17490 assert_eq!(
17491 route.closed_reason(),
17492 *reason,
17493 "{}: route {index}",
17494 case.name
17495 );
17496 }
17497 None => {
17498 assert!(rig.live(route), "{}: route {index} closed", case.name);
17499 assert!(
17500 route.untouched(),
17501 "{}: route {index} was told something",
17502 case.name
17503 );
17504 }
17505 }
17506 }
17507 }
17508 }
17509
17510 #[tokio::test]
17511 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
17512 for next in [Vec::new(), vec![session(2)]] {
17514 let mut rig = rig().await;
17515 rig.sync(vec![session(1)]).await;
17516 let mut route = rig
17517 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17518 .await;
17519 rig.sync(next).await;
17520 assert!(!rig.live(&route));
17521 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
17522 }
17523
17524 let mut rig = rig().await;
17527 let mut child = session(1);
17528 child.scope_ref = "child".to_string();
17529 child.kind = ScopeKind::Worker;
17530 child.parent = Some(ScopeParent::new(
17531 Principal::Reserved {
17532 module_id: OWNER.to_string(),
17533 },
17534 "s".to_string(),
17535 1,
17536 ));
17537 rig.sync(vec![session(1), child.clone()]).await;
17538 let mut child_route = rig
17539 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
17540 .await;
17541 assert_eq!(
17542 child_route.stamp().unwrap().parent_state,
17543 Some(ParentState::Linked)
17544 );
17545 rig.sync(vec![child]).await;
17546 assert!(!rig.live(&child_route));
17547 assert_eq!(
17548 child_route.closed_reason(),
17549 RouteCloseReason::ScopeParentEnded
17550 );
17551 }
17552
17553 #[tokio::test]
17554 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
17555 ) {
17556 let mut rig = rig().await;
17557 rig.sync(vec![session(1)]).await;
17558 let mut route = rig
17559 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17560 .await;
17561 let before = rig.forwarding.published_scope_tag(OWNER, "s");
17562 rig.sync(vec![session(1)]).await;
17563 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17564 assert!(rig.live(&route) && route.untouched());
17565
17566 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
17570 .forwarding
17571 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17572 .unwrap()
17573 else {
17574 panic!("the route is bound");
17575 };
17576 binding.flow.acquire_tagged(9, false).await.unwrap();
17577 let mut widened = session(1);
17578 widened.carriers.push(carrier(MAGIC, None));
17579 rig.sync(vec![widened]).await;
17580 assert!(rig.live(&route) && route.untouched());
17581 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17582 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
17583 assert_eq!(binding.flow.in_flight(), 1);
17586 binding
17587 .flow
17588 .acquire_tagged(10, false)
17589 .await
17590 .expect("the flow is still open");
17591 }
17592
17593 #[tokio::test]
17596 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
17597 let mut rig = rig().await;
17598 rig.sync(vec![session(1)]).await;
17599 let mut on_incumbent = rig
17600 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17601 .await;
17602
17603 let (candidate, _candidate_rx) = wide_ctx(9);
17606 let registration = rig
17607 .handler
17608 .registry
17609 .register_candidate_with_control_ops(
17610 manifest(PLEXUS, PROTOCOL_VERSION),
17611 PROTOCOL_VERSION,
17612 candidate.connection_id,
17613 module_baseline_control_ops(),
17614 )
17615 .unwrap();
17616 rig.forwarding
17617 .register_candidate_module_connection(
17618 candidate.connection_id,
17619 PLEXUS.to_string(),
17620 PROTOCOL_VERSION,
17621 manifest_concurrency(®istration.manifest),
17622 candidate.egress.clone(),
17623 )
17624 .unwrap();
17625 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
17626 rig.handler
17627 .registry
17628 .promote_candidate(PLEXUS)
17629 .unwrap()
17630 .unwrap();
17631 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
17632
17633 rig.sync(Vec::new()).await;
17634 assert!(!rig.live(&on_incumbent));
17635 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
17636 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17637 let goodbye = incumbent_rx
17638 .try_recv()
17639 .expect("the superseded endpoint is told")
17640 .frame;
17641 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17642 }
17643 }
17644}
17645
17646#[cfg(test)]
17647mod concurrency_default_exposure_tests {
17648 use super::*;
17649
17650 fn hello_body(role_json: &str) -> Vec<u8> {
17651 format!(
17652 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
17653 )
17654 .into_bytes()
17655 }
17656
17657 fn manifest_from(body: &[u8]) -> ModuleManifest {
17658 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
17659 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
17660 .expect("manifest parses")
17661 }
17662
17663 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
17664
17665 #[test]
17666 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
17667 let body = hello_body(&format!(
17668 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
17669 ));
17670 let manifest = manifest_from(&body);
17671 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17674 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
17675 }
17676
17677 #[test]
17678 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
17679 let body = hello_body(&format!(
17680 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
17681 ));
17682 let manifest = manifest_from(&body);
17683 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17684 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17685 }
17686
17687 #[test]
17688 fn non_management_roles_are_never_reported() {
17689 let body = hello_body(
17690 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
17691 );
17692 let manifest = manifest_from(&body);
17693 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17694 }
17695}