1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_event_declarations,
25 validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
26 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
27 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
28 },
29 scope::{
30 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
31 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
32 SCOPE_SYNC_OP,
33 },
34 session::{
35 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
36 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
37 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
38 ROLE_VERSIONS_FIELD,
39 },
40 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
41 Priority, RouteTarget, PROTOCOL_VERSION,
42};
43use tokio::time::{timeout_at, Instant};
44use tracing::{debug, info, warn};
45
46use crate::{
47 caller_identity::CallerIdentity,
48 capability_requirements::{
49 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
50 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
51 RuntimeModule,
52 },
53 child_roster::DaemonShutdownFlag,
54 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
55 forwarding::{
56 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
57 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
58 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
59 },
60 observability::{
61 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
62 },
63 privileged_prompt::{prompt_text, PrivilegedOperation},
64 provenance::{
65 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
66 },
67 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
68 router::{RouteCtx, RouterError},
69 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
70 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
71 stderr_tail::{CaptureState, TailEntry},
72 supervise::{
73 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
74 SupervisorHandle, SwapHelloAdmission,
75 },
76 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
77};
78
79pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
85
86const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
87const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
88const CAP_PING_PONG: &str = "ping_pong_v1";
89const CAP_SESSION_ATTACH: &str = "session_attach_v1";
90const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
91
92const SUBC_CONTROL_OPS: &[&str] = &[
93 ops::ACTIVITY_RECORD,
94 ops::ACTIVITY_QUERY,
95 ops::ACTIVITY_SUBSCRIBE,
96 ops::SERVER_DESCRIBE,
97 ops::CATALOG_LIST,
98 ops::ROUTE_OPEN,
99 ops::ROUTE_POLL,
100 ops::ROUTE_CLOSING,
101 ops::ROUTE_CLOSED,
102 ops::SUPERVISOR_LIST,
103 ops::SUPERVISOR_RESTART,
104 ops::SUPERVISOR_SWAP,
105 ops::SUPERVISOR_RELOAD,
106 ops::SUPERVISOR_RESCAN,
107 ops::SUPERVISOR_RELEASE_RESERVED,
108 ops::SUPERVISOR_SET_ENABLED,
109 ops::SUPERVISOR_HEALTH_PROBE,
110 ops::SUPERVISOR_HEALTH,
111 ops::SUPERVISOR_STDERR_TAIL,
112 ops::SUPERVISOR_TERMINALS,
113 ops::SUPERVISOR_ROUTES,
114 ops::SUPERVISOR_PROVENANCE,
115 ops::SUPERVISOR_SPAWN_SNAPSHOT,
116 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
117];
118
119const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
126
127mod rescan_checked;
128use rescan_checked::{RescanMode, RescanPlan};
129
130const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
131 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
132 "supervisor.live_roots",
133 SCOPE_SYNC_OP,
134 SCOPE_APPLY_OP,
135 SCOPE_DESCRIBE_OP,
136 "operator.confirm",
137];
138
139const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
143
144const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
145
146pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
154
155pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
170
171pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
189
190const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
191const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
192
193fn reload_verdict(
194 configured: &Path,
195 spawned_from: Option<&Path>,
196 image: subc_control::RunningImageAgreement,
197) -> PendingReloadVerdict {
198 let path = match spawned_from {
199 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
200 ReloadPathAgreement::Match
201 }
202 Some(spawned_from) => ReloadPathAgreement::Mismatch {
203 configured: configured.to_path_buf(),
204 spawned_from: spawned_from.to_path_buf(),
205 },
206 None => ReloadPathAgreement::Unavailable {
207 reason: if matches!(
208 image,
209 subc_control::RunningImageAgreement::Unavailable {
210 reason: subc_control::RunningImageUnavailableReason::NotRunning
211 }
212 ) {
213 ReloadPathUnavailableReason::NotRunning
214 } else {
215 ReloadPathUnavailableReason::SpawnedPathUnavailable
216 },
217 },
218 };
219 PendingReloadVerdict {
220 path,
221 image,
222 spec: None,
223 }
224}
225
226fn launch_spec_agreement(
227 configured: &crate::supervise::ModuleSpec,
228 running: bool,
229 spawned: Option<&crate::supervise::LaunchSpecDigest>,
230) -> subc_control::LaunchSpecAgreement {
231 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
232 if !running {
233 return Agreement::Unavailable {
234 reason: Reason::NotRunning,
235 };
236 }
237 match spawned {
238 None => Agreement::Unavailable {
239 reason: Reason::SpawnedSpecUnavailable,
240 },
241 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
242 Agreement::Match
243 }
244 Some(_) => Agreement::Mismatch,
245 }
246}
247
248fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
249 if configured == spawned_from {
250 return true;
251 }
252 #[cfg(windows)]
253 {
254 match (
260 std::fs::canonicalize(configured),
261 std::fs::canonicalize(spawned_from),
262 ) {
263 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
264 _ => false,
265 }
266 }
267 #[cfg(not(windows))]
268 {
269 false
270 }
271}
272
273#[derive(Clone)]
274struct DaemonProvenanceFacts {
275 build: DaemonBuildProvenance,
276 pid: Option<u32>,
277 started_at_ms: Option<u64>,
278 start_clock: Option<crate::clock::StartClock>,
279 executable_path: Option<PathBuf>,
280 executable_identity: Option<SpawnedFileIdentity>,
281 process_start_time: Option<u64>,
282 probe: ExecutableIdentityProbe,
283}
284
285impl Default for DaemonProvenanceFacts {
286 fn default() -> Self {
287 Self {
288 build: DaemonBuildProvenance {
289 build_git_sha: None,
290 build_lock_digest: None,
291 },
292 pid: None,
293 started_at_ms: None,
294 start_clock: None,
295 executable_path: None,
296 executable_identity: None,
297 process_start_time: None,
298 probe: ExecutableIdentityProbe::default(),
299 }
300 }
301}
302
303#[derive(Debug, Clone)]
304struct SupervisorRescanContext {
305 supervisor: Supervisor,
306 config_path: PathBuf,
307 configured_port: Option<u16>,
308 storage_config: Option<crate::daemon_config::StorageConfig>,
309 admission_facts_carrier_module_id: Option<String>,
310 admission_facts_targets: Option<Vec<String>>,
311 scope_authority_owners: Vec<String>,
312 privileged_operator_prompt: PrivilegedOperatorPrompt,
313 generation: Arc<std::sync::atomic::AtomicU64>,
319}
320
321const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
342 "reloading",
343 "supervisor_not_live",
344 "registration_not_active",
345 "no_forwarding_connection",
346 "relay_send_failed",
347];
348
349#[derive(Clone)]
351pub struct ControlHandler {
352 registry: Arc<Registry>,
353 activity: Option<crate::activity::ActivityLedger>,
354 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
355 forwarding: Arc<ForwardingTable>,
356 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
357 supervisor: SupervisorHandle,
358 subc_capabilities: Arc<[String]>,
359 route_bind_relay_timeout: Duration,
363 route_bind_relay_timeouts: BTreeMap<String, Duration>,
367 route_bind_breakers: RouteBindBreakers,
370 route_bind_concurrency: RouteBindConcurrency,
373 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
378 route_bind_breaker_threshold: u32,
380 route_bind_breaker_cooldown: Duration,
382 health_probe_timeout: Duration,
383 storage_config: Option<crate::daemon_config::StorageConfig>,
386 machine_id: Option<crate::machine_id::MachineId>,
390 admission_facts_carrier_module_id: Option<String>,
391 admission_facts_targets: Option<Vec<String>>,
392 scopes: Arc<RwLock<ScopeTable>>,
396 scope_deadline_added: Arc<tokio::sync::Notify>,
400 scope_authority_owners: Vec<String>,
403 privileged_operator_prompt: PrivilegedOperatorPrompt,
405 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
408 #[cfg(test)]
409 caller_lookup: Option<TestCallerLookup>,
410 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
413 rescan: Option<SupervisorRescanContext>,
414 connected_clients: ConnectedClients,
415 counters: DaemonCounters,
416 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
417 daemon_shutdown: DaemonShutdownFlag,
422 daemon_provenance: DaemonProvenanceFacts,
423 #[cfg(test)]
424 control_dispatch_delay: Option<Duration>,
425 #[cfg(test)]
426 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
427 #[cfg(test)]
431 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
432 #[cfg(test)]
438 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
439}
440
441impl fmt::Debug for ControlHandler {
442 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
443 f.debug_struct("ControlHandler")
444 .field("registry", &self.registry)
445 .field("forwarding", &self.forwarding)
446 .field("process_liveness", &self.process_liveness.is_some())
447 .field("supervisor", &self.supervisor)
448 .field("subc_capabilities", &self.subc_capabilities)
449 .finish()
450 }
451}
452
453struct RouteOpenRequest {
454 target: RouteTarget,
455 identity: BindIdentity,
456 consumer_identity: Option<ConsumerIdentity>,
457 consumer_capabilities: Option<Vec<String>>,
458 role_versions: Option<BTreeMap<String, String>>,
459 admission_facts: Option<serde_json::Value>,
460 scope: Option<ScopeSelector>,
461}
462
463#[derive(Clone, Copy)]
464enum RouteOpenPhase {
465 Admission,
466 ProjectRoot,
467 EgressReservation,
468 BindSend,
469 BindAckWait,
470}
471
472struct RouteOpenTimings {
476 module_id: String,
477 principal: String,
478 admission: Duration,
479 project_root: Duration,
480 egress_reservation: Duration,
481 bind_send: Duration,
482 bind_ack_wait: Duration,
483 phase: Option<RouteOpenPhase>,
484 phase_started_at: Instant,
485}
486
487impl RouteOpenTimings {
488 fn new() -> Self {
489 Self {
490 module_id: String::new(),
491 principal: "unresolved".to_string(),
493 admission: Duration::ZERO,
494 project_root: Duration::ZERO,
495 egress_reservation: Duration::ZERO,
496 bind_send: Duration::ZERO,
497 bind_ack_wait: Duration::ZERO,
498 phase: Some(RouteOpenPhase::Admission),
499 phase_started_at: Instant::now(),
500 }
501 }
502
503 fn finish(&mut self) {
504 let elapsed = self.phase_started_at.elapsed();
505 match self.phase.take() {
506 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
507 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
508 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
509 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
510 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
511 None => {}
512 }
513 }
514
515 fn enter(&mut self, phase: RouteOpenPhase) {
516 self.finish();
517 self.phase = Some(phase);
518 self.phase_started_at = Instant::now();
519 }
520}
521
522struct RouteBindReservationGuard {
523 forwarding: Arc<ForwardingTable>,
524 endpoint: ModuleEndpointId,
525 relay_corr: u64,
526 armed: bool,
527}
528
529struct ModuleControlRpcGuard {
530 forwarding: Arc<ForwardingTable>,
531 endpoint: ModuleEndpointId,
532 corr: u64,
533 armed: bool,
534}
535
536impl ModuleControlRpcGuard {
537 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
538 Self {
539 forwarding,
540 endpoint,
541 corr,
542 armed: true,
543 }
544 }
545
546 fn disarm(&mut self) {
547 self.armed = false;
548 }
549}
550
551impl Drop for ModuleControlRpcGuard {
552 fn drop(&mut self) {
553 if self.armed {
554 let _ = self
555 .forwarding
556 .cancel_module_control_rpc(self.endpoint, self.corr);
557 }
558 }
559}
560
561impl RouteBindReservationGuard {
562 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
563 Self {
564 forwarding,
565 endpoint,
566 relay_corr,
567 armed: true,
568 }
569 }
570
571 fn release_and_disarm(&mut self) {
572 if !self.armed {
573 return;
574 }
575 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
576 self.endpoint,
577 self.relay_corr,
578 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
579 ) {
580 send_goodbye_target_best_effort(
581 &self.forwarding.counters(),
582 &target,
583 "canceled route.bind",
584 );
585 }
586 self.armed = false;
587 }
588
589 fn disarm(&mut self) {
590 self.armed = false;
591 }
592}
593
594impl Drop for RouteBindReservationGuard {
595 fn drop(&mut self) {
596 self.release_and_disarm();
597 }
598}
599
600#[derive(Debug, Clone, Default)]
630pub(crate) struct RouteBindBreakers {
631 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
632}
633
634#[derive(Debug, Clone, Default)]
635pub(crate) struct RouteBindConcurrency {
636 modules: Arc<Mutex<HashMap<String, usize>>>,
637}
638
639struct RouteBindConcurrencyGuard {
640 concurrency: RouteBindConcurrency,
641 module_id: String,
642}
643
644impl RouteBindConcurrency {
645 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
648 let mut modules = self
649 .modules
650 .lock()
651 .expect("route.bind concurrency mutex poisoned");
652 let in_flight = modules.entry(module_id.to_string()).or_default();
653 if *in_flight >= limit {
654 return Err(*in_flight);
655 }
656 *in_flight += 1;
657 Ok(RouteBindConcurrencyGuard {
658 concurrency: self.clone(),
659 module_id: module_id.to_string(),
660 })
661 }
662}
663
664impl Drop for RouteBindConcurrencyGuard {
665 fn drop(&mut self) {
666 let mut modules = self
667 .concurrency
668 .modules
669 .lock()
670 .expect("route.bind concurrency mutex poisoned");
671 let remove = {
672 let in_flight = modules
673 .get_mut(&self.module_id)
674 .expect("admitted route.bind has a concurrency entry");
675 *in_flight -= 1;
676 *in_flight == 0
677 };
678 if remove {
679 modules.remove(&self.module_id);
680 }
681 }
682}
683
684#[derive(Debug, Default)]
685struct ModuleBreakerState {
686 consecutive_timeouts: u32,
688 cooldown_until: Option<Instant>,
691 probe_in_flight: Option<Arc<()>>,
696}
697
698enum RouteBindAdmission<'a> {
700 Admitted {
701 guard: RouteBindBreakerGuard<'a>,
702 probe: bool,
705 },
706 Refused {
707 consecutive_timeouts: u32,
708 retry_in: Duration,
712 probe_in_flight: bool,
713 },
714}
715
716struct RouteBindBreakerGuard<'a> {
723 breakers: RouteBindBreakers,
724 module_id: &'a str,
725 probe_token: Option<Arc<()>>,
726 settled: bool,
727}
728
729impl RouteBindBreakerGuard<'_> {
730 fn record_accepted(&mut self) -> bool {
734 self.settled = true;
735 self.breakers.record_accepted(self.module_id)
736 }
737
738 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
741 self.settled = true;
742 self.breakers.record_timeout(
743 self.module_id,
744 self.probe_token.as_ref(),
745 threshold,
746 cooldown,
747 )
748 }
749
750 fn record_inconclusive(&mut self) {
759 self.settled = true;
760 self.breakers
761 .record_inconclusive(self.module_id, self.probe_token.as_ref());
762 }
763}
764
765impl Drop for RouteBindBreakerGuard<'_> {
766 fn drop(&mut self) {
767 if !self.settled {
768 self.breakers
769 .record_inconclusive(self.module_id, self.probe_token.as_ref());
770 }
771 }
772}
773
774struct BreakerOpened {
778 consecutive_timeouts: u32,
779 reopened_after_probe: bool,
782}
783
784impl RouteBindBreakers {
785 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
786 self.modules
787 .lock()
788 .expect("route.bind breaker mutex poisoned")
789 }
790
791 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
794 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
795 probe: probe_token.is_some(),
796 guard: RouteBindBreakerGuard {
797 breakers: self.clone(),
798 module_id,
799 probe_token,
800 settled: false,
801 },
802 };
803
804 let mut modules = self.lock();
805 let Some(state) = modules.get_mut(module_id) else {
806 return admitted(None);
807 };
808 let Some(cooldown_until) = state.cooldown_until else {
809 return admitted(None);
810 };
811 if state.probe_in_flight.is_some() {
812 return RouteBindAdmission::Refused {
813 consecutive_timeouts: state.consecutive_timeouts,
814 retry_in: Duration::ZERO,
815 probe_in_flight: true,
816 };
817 }
818 let now = Instant::now();
819 if now < cooldown_until {
820 return RouteBindAdmission::Refused {
821 consecutive_timeouts: state.consecutive_timeouts,
822 retry_in: cooldown_until - now,
823 probe_in_flight: false,
824 };
825 }
826 let token = Arc::new(());
827 state.probe_in_flight = Some(Arc::clone(&token));
828 admitted(Some(token))
829 }
830
831 fn record_accepted(&self, module_id: &str) -> bool {
832 self.lock()
833 .remove(module_id)
834 .is_some_and(|state| state.cooldown_until.is_some())
835 }
836
837 fn record_timeout(
838 &self,
839 module_id: &str,
840 probe_token: Option<&Arc<()>>,
841 threshold: u32,
842 cooldown: Duration,
843 ) -> Option<BreakerOpened> {
844 let mut modules = self.lock();
845 let state = modules.entry(module_id.to_string()).or_default();
846 let was_open = state.cooldown_until.is_some();
847 let was_probe = Self::owns_probe(state, probe_token);
848 if was_probe {
849 state.probe_in_flight = None;
850 }
851 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
852 if state.consecutive_timeouts < threshold {
853 return None;
854 }
855 state.cooldown_until = Some(Instant::now() + cooldown);
856 Some(BreakerOpened {
857 consecutive_timeouts: state.consecutive_timeouts,
858 reopened_after_probe: was_open && was_probe,
859 })
860 }
861
862 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
863 state
867 .probe_in_flight
868 .as_ref()
869 .zip(token)
870 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
871 }
872
873 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
874 if let Some(state) = self.lock().get_mut(module_id) {
875 if Self::owns_probe(state, probe_token) {
876 state.probe_in_flight = None;
877 }
878 }
879 }
880
881 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
896 self.lock()
897 .remove(module_id)
898 .map(|state| state.consecutive_timeouts)
899 .filter(|discarded| *discarded > 0)
900 }
901
902 fn open_snapshot(&self) -> Option<serde_json::Value> {
909 let now = Instant::now();
910 let modules = self.lock();
911 let open = modules
912 .iter()
913 .filter_map(|(module_id, state)| {
914 let cooldown_until = state.cooldown_until?;
915 Some((
916 module_id.clone(),
917 serde_json::json!({
918 "consecutive_timeouts": state.consecutive_timeouts,
919 "cooldown_remaining_ms":
920 cooldown_until.saturating_duration_since(now).as_millis() as u64,
921 "probe_in_flight": state.probe_in_flight.is_some(),
922 }),
923 ))
924 })
925 .collect::<serde_json::Map<String, serde_json::Value>>();
926 (!open.is_empty()).then_some(serde_json::Value::Object(open))
927 }
928}
929
930impl ControlHandler {
931 pub fn new(registry: Arc<Registry>) -> Self {
932 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
933 }
934
935 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
936 let counters = forwarding.counters();
937 let route_bind_breakers = forwarding.route_bind_breakers();
941 let route_bind_concurrency = forwarding.route_bind_concurrency();
942 let route_outages = forwarding.route_outages();
943 Self {
944 registry,
945 activity: None,
946 activity_streams: Arc::new(Mutex::new(HashMap::new())),
947 forwarding,
948 process_liveness: None,
949 supervisor: SupervisorHandle::new(),
950 subc_capabilities: Arc::from([
951 CAP_MANIFEST_REGISTRATION.to_string(),
952 CAP_CHANNEL_LIFECYCLE.to_string(),
953 CAP_PING_PONG.to_string(),
954 CAP_SESSION_ATTACH.to_string(),
955 CAP_ADMISSION_FACTS_RELAY.to_string(),
956 CAP_SCOPES_V1.to_string(),
957 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
958 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
959 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
960 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
961 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
962 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
963 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
964 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
965 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
966 ]),
967 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
968 route_bind_relay_timeouts: BTreeMap::new(),
969 route_bind_breakers,
970 route_bind_concurrency,
971 route_outages,
972 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
973 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
974 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
975 storage_config: None,
976 machine_id: None,
977 admission_facts_carrier_module_id: None,
978 admission_facts_targets: None,
979 scopes: Arc::new(RwLock::new(ScopeTable::new(
980 crate::daemon_config::default_scope_authority_owners(),
981 ))),
982 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
983 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
984 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
985 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
986 #[cfg(test)]
987 caller_lookup: None,
988 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
989 rescan: None,
990 connected_clients: ConnectedClients::new(),
991 counters,
992 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
993 daemon_shutdown: DaemonShutdownFlag::default(),
994 daemon_provenance: DaemonProvenanceFacts::default(),
995 #[cfg(test)]
996 control_dispatch_delay: None,
997 #[cfg(test)]
998 provenance_probe_override: None,
999 #[cfg(test)]
1000 route_bind_relay_sent_hold: None,
1001 #[cfg(test)]
1002 hello_registry_to_forwarding_hold: None,
1003 }
1004 }
1005
1006 pub fn with_storage_config(
1009 mut self,
1010 storage_config: Option<crate::daemon_config::StorageConfig>,
1011 ) -> Self {
1012 self.storage_config = storage_config;
1013 self
1014 }
1015
1016 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1019 self.activity = Some(crate::activity::ActivityLedger::open(
1020 run_dir.join("activity.jsonl"),
1021 incarnation,
1022 ));
1023 self.supervisor.set_activity(self.activity.clone());
1024 self
1025 }
1026
1027 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1030 self.machine_id = machine_id;
1031 self
1032 }
1033
1034 pub fn with_admission_facts_config(
1038 mut self,
1039 carrier_module_id: Option<String>,
1040 targets: Option<Vec<String>>,
1041 ) -> Self {
1042 self.admission_facts_carrier_module_id = carrier_module_id;
1043 self.admission_facts_targets = targets;
1044 self
1045 }
1046
1047 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1051 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1052 self.scope_authority_owners = owners;
1053 self
1054 }
1055
1056 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1058 self.privileged_operator_prompt = mode;
1059 self
1060 }
1061
1062 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1065 self.route_bind_relay_timeout = timeout;
1066 self
1067 }
1068
1069 pub fn with_route_bind_relay_timeouts(
1075 mut self,
1076 timeouts: impl IntoIterator<Item = (String, Duration)>,
1077 ) -> Self {
1078 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1079 self
1080 }
1081
1082 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1088 self.route_bind_relay_timeouts
1089 .get(module_id)
1090 .copied()
1091 .unwrap_or(self.route_bind_relay_timeout)
1092 }
1093
1094 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1102 self.route_bind_breaker_threshold = threshold.max(1);
1103 self.route_bind_breaker_cooldown = cooldown;
1104 self
1105 }
1106
1107 #[cfg(test)]
1108 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1109 self.health_probe_timeout = timeout;
1110 self
1111 }
1112
1113 #[cfg(test)]
1114 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1115 self.control_dispatch_delay = Some(delay);
1116 self
1117 }
1118
1119 #[cfg(test)]
1123 pub(crate) fn with_route_bind_relay_sent_hold(
1124 mut self,
1125 hold: Arc<tokio::sync::Barrier>,
1126 ) -> Self {
1127 self.route_bind_relay_sent_hold = Some(hold);
1128 self
1129 }
1130
1131 #[cfg(test)]
1136 pub(crate) fn with_hello_registry_to_forwarding_hold(
1137 mut self,
1138 hold: Arc<std::sync::Barrier>,
1139 ) -> Self {
1140 self.hello_registry_to_forwarding_hold = Some(hold);
1141 self
1142 }
1143
1144 #[cfg(test)]
1145 fn hold_between_registry_and_forwarding(&self) {
1146 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1147 hold.wait();
1148 hold.wait();
1149 }
1150 }
1151
1152 pub fn with_process_liveness(
1153 mut self,
1154 process_liveness: Arc<dyn ModuleProcessLiveness>,
1155 ) -> Self {
1156 self.process_liveness = Some(process_liveness);
1157 self
1158 }
1159
1160 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1161 self.supervisor = supervisor;
1162 self.supervisor.set_activity(self.activity.clone());
1163 self
1164 }
1165
1166 pub fn with_daemon_provenance(
1167 mut self,
1168 pid: u32,
1169 started_at_ms: u64,
1170 executable_path: Option<PathBuf>,
1171 build_git_sha: Option<String>,
1172 build_lock_digest: Option<String>,
1173 ) -> Self {
1174 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1175 let process_start_time = process_start_time(pid);
1176 self.daemon_provenance = DaemonProvenanceFacts {
1177 build: DaemonBuildProvenance {
1178 build_git_sha,
1179 build_lock_digest,
1180 },
1181 pid: Some(pid),
1182 started_at_ms: Some(started_at_ms),
1183 start_clock: None,
1184 executable_path,
1185 executable_identity,
1186 process_start_time,
1187 probe: ExecutableIdentityProbe::default(),
1188 };
1189 self
1190 }
1191
1192 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1193 self.daemon_provenance.start_clock = Some(clock);
1194 self
1195 }
1196
1197 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1199 self.scopes
1200 .write()
1201 .unwrap_or_else(|poisoned| poisoned.into_inner())
1202 .set_wall_clock(Arc::new(clock));
1203 self
1204 }
1205
1206 #[cfg(test)]
1207 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1208 self.provenance_probe_override = Some(result);
1209 self
1210 }
1211
1212 pub fn with_capability_config(
1216 self,
1217 modules: impl IntoIterator<Item = (String, bool)>,
1218 reserved_capabilities: BTreeMap<String, String>,
1219 ) -> Self {
1220 self.capability_evaluator
1221 .configure(modules, reserved_capabilities);
1222 self
1223 }
1224
1225 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1229 self.daemon_shutdown = flag;
1230 self
1231 }
1232
1233 pub fn with_supervisor_rescan(
1234 mut self,
1235 supervisor: Supervisor,
1236 config_path: impl Into<PathBuf>,
1237 configured_port: Option<u16>,
1238 ) -> Self {
1239 self.rescan = Some(SupervisorRescanContext {
1240 supervisor,
1241 config_path: config_path.into(),
1242 configured_port,
1243 storage_config: self.storage_config.clone(),
1244 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1245 admission_facts_targets: self.admission_facts_targets.clone(),
1246 scope_authority_owners: self.scope_authority_owners.clone(),
1247 privileged_operator_prompt: self.privileged_operator_prompt,
1248 generation: Arc::default(),
1249 });
1250 self
1251 }
1252
1253 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1254 self.connected_clients = connected_clients;
1255 self
1256 }
1257
1258 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1259 Arc::clone(&self.forwarding)
1260 }
1261
1262 pub(crate) fn counters(&self) -> DaemonCounters {
1263 self.counters.clone()
1264 }
1265
1266 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1269 tokio::spawn(async move {
1270 loop {
1271 self.capability_evaluator
1272 .wait_for_change_or_deadline()
1273 .await;
1274 self.refresh_capability_requirements();
1275 }
1276 });
1277 }
1278
1279 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1285 tokio::spawn(async move {
1286 loop {
1287 let has_deadlines = self
1288 .scopes
1289 .read()
1290 .unwrap_or_else(|poisoned| poisoned.into_inner())
1291 .has_deadlines();
1292 if !has_deadlines {
1293 self.scope_deadline_added.notified().await;
1294 continue;
1295 }
1296 tokio::time::sleep(Duration::from_secs(1)).await;
1297 if let Err(error) = self.sweep_expired_scopes() {
1298 warn!(%error, "scope expiry sweep failed");
1299 }
1300 }
1301 });
1302 }
1303
1304 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1305 let mut table = self
1306 .scopes
1307 .write()
1308 .unwrap_or_else(|poisoned| poisoned.into_inner());
1309 let swept = table.sweep_expired();
1310 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1311 drop(table);
1312 self.close_scope_drained_routes(drained);
1313 Ok(())
1314 }
1315
1316 fn publish_scope_changes(
1317 &self,
1318 changes: &[crate::scopes::ScopeTagChange],
1319 expired: &[crate::scopes::ScopeExpired],
1320 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1321 if changes.is_empty() {
1322 return Ok(Vec::new());
1323 }
1324 let drained = self
1325 .forwarding
1326 .publish_scope_changes(changes)
1327 .map_err(RouterError::Forwarding)?;
1328 let mut counts = HashMap::new();
1329 for route in &drained {
1330 *counts
1331 .entry((
1332 route.scope.owner.as_str(),
1333 route.scope.scope_ref.as_str(),
1334 route.scope.tag.scope_epoch,
1335 ))
1336 .or_insert(0usize) += 1;
1337 }
1338 for scope in expired {
1339 let routes_closed = counts
1340 .get(&(
1341 scope.owner.as_str(),
1342 scope.scope_ref.as_str(),
1343 scope.scope_epoch,
1344 ))
1345 .copied()
1346 .unwrap_or(0);
1347 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1348 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1349 routes_closed, "scope expired");
1350 }
1351 Ok(drained)
1352 }
1353
1354 fn runtime_capability_snapshot(
1355 &self,
1356 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1357 let runtime = self
1358 .supervisor
1359 .list()
1360 .into_iter()
1361 .map(|module| {
1362 let status = module.status().map_err(|err| {
1363 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1364 })?;
1365 Ok(RuntimeModule {
1366 module_id: status.module_id,
1367 state: status.state,
1368 enabled: status.enabled,
1369 })
1370 })
1371 .collect::<Result<Vec<_>, RouterError>>()?;
1372 let registrations = self.registry.list_registered_modules().map_err(|err| {
1378 RouterError::backend(
1379 0,
1380 0,
1381 format!("failed to list capability registrations: {err}"),
1382 )
1383 })?;
1384 let registrations = registrations
1385 .into_iter()
1386 .map(|registration| RegisteredModule {
1387 module_id: registration.manifest.module_id,
1388 module_version: registration.manifest.module_version,
1389 capabilities: registration.manifest.capabilities,
1390 })
1391 .collect();
1392 Ok((runtime, registrations))
1393 }
1394
1395 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1402 let cached_registration = RegisteredModule {
1403 module_id: registration.manifest.module_id.clone(),
1404 module_version: registration.manifest.module_version.clone(),
1405 capabilities: registration.manifest.capabilities.clone(),
1406 };
1407 if self.capability_evaluator.record_hello(&cached_registration) {
1408 warn!(
1409 module_id = %cached_registration.module_id,
1410 "capability claims drifted from the cached manifest"
1411 );
1412 }
1413 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1414 self.enforce_capability_denies();
1415 }
1416 self.refresh_capability_requirements();
1417 }
1418
1419 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1423 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1424 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1425 self.supervisor.set_swap_promotion_observer(observer);
1426 }
1427
1428 pub fn refresh_capability_requirements(&self) {
1429 match self.runtime_capability_snapshot() {
1430 Ok((runtime, registrations)) => {
1431 log_requirement_events(
1432 self.capability_evaluator
1433 .evaluate_now(&runtime, ®istrations),
1434 &self.daemon_shutdown,
1435 );
1436 }
1437 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1438 }
1439 }
1440
1441 fn enforce_capability_denies(&self) {
1445 let registrations = match self.registry.list_registered_modules() {
1446 Ok(snapshot) => snapshot,
1447 Err(err) => {
1448 warn!(error = %err, "failed to read registrations for capability deny census");
1449 return;
1450 }
1451 };
1452 let manifests = registrations
1453 .into_iter()
1454 .map(|registration| {
1455 (
1456 registration.manifest.module_id.clone(),
1457 registration.manifest,
1458 )
1459 })
1460 .collect::<BTreeMap<_, _>>();
1461 let census = match self.forwarding.route_census(None) {
1462 Ok(census) => census,
1463 Err(err) => {
1464 warn!(error = %err, "failed to read route census for capability deny enforcement");
1465 return;
1466 }
1467 };
1468
1469 for (target_module_id, routes) in census {
1470 let Some(target_manifest) = manifests.get(&target_module_id) else {
1471 continue;
1472 };
1473 let mut closed_routes = Vec::new();
1474 let mut goodbyes = Vec::new();
1475 for route in routes {
1476 let Principal::Reserved {
1477 module_id: opening_module_id,
1478 } = &route.principal
1479 else {
1480 continue;
1481 };
1482 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1483 continue;
1484 };
1485 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1486 continue;
1487 };
1488
1489 match self.forwarding.release_client_route(
1490 route.goodbye_target.connection_id,
1491 route.goodbye_target.channel,
1492 route.goodbye_target.epoch,
1493 ) {
1494 Ok(RouteRelease::Removed(module_goodbye)) => {
1495 warn!(
1496 opening_module_id,
1497 target_module_id,
1498 capability,
1499 "force-closing route because an attested capability deny edge now matches"
1500 );
1501 goodbyes.push(module_goodbye);
1502 goodbyes.push(route.goodbye_target.clone());
1503 closed_routes.push(route);
1504 }
1505 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1506 Err(err) => warn!(
1507 opening_module_id,
1508 target_module_id,
1509 capability,
1510 error = %err,
1511 "failed to force-close capability-denied route"
1512 ),
1513 }
1514 }
1515
1516 if closed_routes.is_empty() {
1517 continue;
1518 }
1519 send_route_control_pushes(
1520 &self.forwarding,
1521 closed_routes,
1522 ClientControlPush::RouteClosed {
1523 module_id: target_module_id,
1524 channels: Vec::new(),
1525 reason: RouteCloseReason::CapabilityDenied,
1526 drained: false,
1527 abandoned: 0,
1528 excluded_subscriptions: 0,
1529 terminal: Some(false),
1530 },
1531 );
1532 self.emit_route_goodbyes(goodbyes);
1535 }
1536 }
1537
1538 fn not_ready_reason(
1544 &self,
1545 registration: &crate::registry::ModuleRegistration,
1546 ) -> Option<NotReadyReason> {
1547 if !registration.ready {
1548 return Some(NotReadyReason {
1549 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1550 capability: None,
1551 });
1552 }
1553 self.first_unprovided_required_capability(registration)
1554 .map(|capability| NotReadyReason {
1555 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1556 capability: Some(capability),
1557 })
1558 }
1559
1560 fn first_unprovided_required_capability(
1578 &self,
1579 registration: &crate::registry::ModuleRegistration,
1580 ) -> Option<String> {
1581 let required = registration
1582 .manifest
1583 .capabilities
1584 .iter()
1585 .flat_map(|declarations| declarations.requires.iter())
1586 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1587 .map(|requirement| requirement.capability.as_str())
1588 .collect::<BTreeSet<_>>();
1589 if required.is_empty() {
1590 return None;
1591 }
1592 let module_id = registration.manifest.module_id.as_str();
1593 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1594 if required
1595 .iter()
1596 .any(|capability| verdict(capability).is_none())
1597 {
1598 self.refresh_capability_requirements();
1599 }
1600 required
1601 .into_iter()
1602 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1603 .map(str::to_string)
1604 }
1605
1606 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1607 self.capability_evaluator
1608 .statuses()
1609 .into_iter()
1610 .map(capability_requirement_status)
1611 .collect()
1612 }
1613
1614 fn deregister_connection(
1621 &self,
1622 connection_id: ConnectionId,
1623 reason: RegistrationEndReason,
1624 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1625 self.registry
1626 .deregister_connection_with_reason(connection_id, reason)
1627 }
1628
1629 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1630 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1631 return None;
1632 }
1633 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1634 parse_client_control_request(&frame.body)
1635 else {
1636 return None;
1637 };
1638 Some(target_module_id(&target).to_string())
1639 }
1640
1641 pub(crate) fn route_open_capacity_refusal(
1642 &self,
1643 ctx: &RouteCtx,
1644 frame: &Frame,
1645 target_module_id: &str,
1646 in_flight: usize,
1647 limit: usize,
1648 ) -> Result<Frame, RouterError> {
1649 let caller = self.route_open_caller_from_frame(frame);
1652 self.route_open_admission_refusal_frame(
1653 ctx,
1654 &caller,
1655 frame,
1656 target_module_id,
1657 "open_admission_full",
1658 (in_flight, limit),
1659 format!(
1660 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1661 ),
1662 )
1663 }
1664
1665 fn route_open_target_capacity_refusal(
1666 &self,
1667 ctx: &RouteCtx,
1668 caller: &RouteOpenCaller,
1669 frame: &Frame,
1670 target_module_id: &str,
1671 in_flight: usize,
1672 ) -> Result<Frame, RouterError> {
1673 self.route_open_admission_refusal_frame(
1674 ctx,
1675 caller,
1676 frame,
1677 target_module_id,
1678 "target_binds_full",
1679 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1680 format!(
1681 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1682 ),
1683 )
1684 }
1685
1686 #[allow(clippy::too_many_arguments)]
1694 fn route_open_admission_refusal_frame(
1695 &self,
1696 ctx: &RouteCtx,
1697 caller: &RouteOpenCaller,
1698 frame: &Frame,
1699 target_module_id: &str,
1700 reason: &'static str,
1701 (in_flight, limit): (usize, usize),
1702 message: impl Into<String>,
1703 ) -> Result<Frame, RouterError> {
1704 let code = error_codes::TARGET_UNAVAILABLE;
1705 self.counters.increment_route_open_refused(code);
1706 info!(
1707 target: "control",
1708 code,
1709 reason,
1710 module_id = ?target_module_id,
1711 connection_id = ctx.connection_id.get(),
1712 principal = %caller.principal,
1713 scope = %caller.scope,
1714 harness = %caller.harness,
1715 in_flight,
1716 limit,
1717 "route.open refused"
1718 );
1719 control_error_frame(frame, code, message.into())
1720 }
1721
1722 #[cfg(test)]
1728 pub fn handle_control(
1729 &self,
1730 connection_id: ConnectionId,
1731 frame: Frame,
1732 ) -> Result<Vec<Frame>, RouterError> {
1733 match frame.header.ty {
1734 FrameType::Ping => Ok(vec![pong(&frame)?]),
1735 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1736 FrameType::Goodbye => self.handle_goodbye(connection_id),
1737 ty => Ok(vec![control_error_frame(
1738 &frame,
1739 "unsupported_control_frame",
1740 format!("unsupported channel-0 frame {ty:?}"),
1741 )?]),
1742 }
1743 }
1744
1745 pub async fn handle_control_frame(
1746 &self,
1747 ctx: &RouteCtx,
1748 frame: Frame,
1749 ) -> Result<Vec<Frame>, RouterError> {
1750 self.handle_control_frame_timed(ctx, frame, None).await
1751 }
1752
1753 pub(crate) fn record_caller_endpoints(
1754 &self,
1755 connection: ConnectionId,
1756 endpoints: Option<(SocketAddr, SocketAddr)>,
1757 ) {
1758 let mut callers = self
1759 .caller_endpoints
1760 .lock()
1761 .unwrap_or_else(|p| p.into_inner());
1762 callers.remove(&connection);
1763 if let Some(endpoints) = endpoints {
1764 callers.insert(connection, endpoints);
1765 }
1766 }
1767
1768 #[cfg(test)]
1769 pub(crate) fn with_caller_lookup_for_test(
1770 mut self,
1771 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1772 ) -> Self {
1773 self.caller_lookup = Some(Arc::new(lookup));
1774 self
1775 }
1776
1777 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1778 let endpoints = self
1779 .caller_endpoints
1780 .lock()
1781 .unwrap_or_else(|p| p.into_inner())
1782 .get(&connection)
1783 .copied();
1784 #[cfg(test)]
1785 let lookup = self.caller_lookup.clone();
1786 tokio::task::spawn_blocking(move || {
1787 #[cfg(test)]
1788 if let Some(lookup) = lookup {
1789 return lookup(endpoints);
1790 }
1791 endpoints
1792 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1793 .unwrap_or_default()
1794 })
1795 .await
1796 .unwrap_or_default()
1797 }
1798
1799 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1800 frame.header.channel == 0
1801 && frame.header.ty == FrameType::Request
1802 && self
1803 .forwarding
1804 .module_endpoint_for_connection(ctx.connection_id)
1805 .is_ok_and(|endpoint| endpoint.is_none())
1806 && parse_client_control_request(&frame.body)
1807 .ok()
1808 .is_some_and(|request| privileged_operation(&request).is_some())
1809 }
1810
1811 pub(crate) async fn handle_control_frame_timed(
1812 &self,
1813 ctx: &RouteCtx,
1814 frame: Frame,
1815 dispatch_started_at: Option<StdInstant>,
1816 ) -> Result<Vec<Frame>, RouterError> {
1817 match frame.header.ty {
1818 FrameType::Ping => Ok(vec![pong(&frame)?]),
1819 FrameType::Hello => {
1820 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1821 }
1822 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1823 FrameType::Cancel => {
1824 if self
1827 .forwarding
1828 .operator_confirms()
1829 .cancel(ctx.connection_id, frame.header.corr)
1830 || self
1831 .supervisor
1832 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1833 || self.cancel_activity_subscription(ctx, &frame)?
1834 {
1835 Ok(Vec::new())
1836 } else {
1837 Ok(vec![control_error_frame(
1838 &frame,
1839 "unknown_subscription",
1840 "no subscription has this correlation id",
1841 )?])
1842 }
1843 }
1844 FrameType::Request => {
1845 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1848 if op
1849 .as_ref()
1850 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1851 {
1852 return self.handle_activity_record(ctx, frame).await;
1853 }
1854 if op.as_ref().is_some_and(|probe| {
1855 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1856 }) {
1857 return self.handle_activity_read(ctx, frame).await;
1858 }
1859 if op
1860 .as_ref()
1861 .is_some_and(|probe| probe.op == "operator.confirm")
1862 {
1863 return self.handle_operator_confirm(ctx, frame);
1864 }
1865 if op
1866 .as_ref()
1867 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1868 {
1869 return self.handle_scope_served_by(ctx.connection_id, frame);
1870 }
1871 if self
1872 .forwarding
1873 .module_endpoint_for_connection(ctx.connection_id)
1874 .map_err(RouterError::Forwarding)?
1875 .is_some()
1876 {
1877 if !is_known_module_request_op(&frame.body) {
1878 return Ok(vec![control_error_frame(
1879 &frame,
1880 "unsupported_control_frame",
1881 "module-originated channel-0 REQUEST is not supported",
1882 )?]);
1883 }
1884 let request = match parse_module_control_request_from_module(&frame.body) {
1885 Ok(request) => request,
1886 Err((err, ControlRequestBodyError::UnknownOp)) => {
1887 return Ok(vec![control_error_frame(
1888 &frame,
1889 "unsupported_control_frame",
1890 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1891 )?])
1892 }
1893 Err((err, ControlRequestBodyError::InvalidBody)) => {
1894 return Ok(vec![control_error_frame(
1895 &frame,
1896 "invalid_control_body",
1897 format!("malformed module control body: {err}"),
1898 )?])
1899 }
1900 };
1901 let op = module_control_request_op(&request);
1902 let corr = frame.header.corr;
1903 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1904 let result =
1905 self.handle_module_control_request(ctx.connection_id, frame, request);
1906 log_slow_control_dispatch(
1907 dispatch_started_at,
1908 op,
1909 ctx.connection_id,
1910 corr,
1911 None,
1912 );
1913 return result;
1914 }
1915
1916 if is_known_module_request_op(&frame.body) {
1917 return Ok(vec![control_error_frame(
1918 &frame,
1919 "not_registered",
1920 "catalog.update requires an active module registration owned by this connection",
1921 )?]);
1922 }
1923
1924 let request = match parse_client_control_request(&frame.body) {
1925 Ok(request) => request,
1926 Err((err, ControlRequestBodyError::UnknownOp)) => {
1927 return Ok(vec![control_error_frame(
1928 &frame,
1929 "unknown_control_op",
1930 format!("unknown client control op: {err}"),
1931 )?])
1932 }
1933 Err((err, ControlRequestBodyError::InvalidBody)) => {
1934 return Ok(vec![control_error_frame(
1935 &frame,
1936 "invalid_control_body",
1937 format!("malformed client control body: {err}"),
1938 )?])
1939 }
1940 };
1941 let op = client_control_request_op(&request);
1942 let corr = frame.header.corr;
1943 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1944 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1945 #[cfg(test)]
1946 if let Some(delay) = self.control_dispatch_delay {
1947 tokio::time::sleep(delay).await;
1948 }
1949 let result = self
1950 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1951 .await;
1952 if let Some(timings) = route_open_timings.as_mut() {
1953 timings.finish();
1954 }
1955 log_slow_control_dispatch(
1956 dispatch_started_at,
1957 op,
1958 ctx.connection_id,
1959 corr,
1960 route_open_timings.as_ref(),
1961 );
1962 result
1963 }
1964 FrameType::Push => {
1965 let Some(endpoint) = self
1966 .forwarding
1967 .module_endpoint_for_connection(ctx.connection_id)
1968 .map_err(RouterError::Forwarding)?
1969 else {
1970 return Ok(vec![control_error_frame(
1971 &frame,
1972 "unsupported_control_frame",
1973 "client-originated channel-0 PUSH is not supported",
1974 )?]);
1975 };
1976 self.handle_status_update(endpoint, frame)
1977 }
1978 FrameType::Response | FrameType::Error
1979 if self
1980 .forwarding
1981 .module_endpoint_for_connection(ctx.connection_id)
1982 .map_err(RouterError::Forwarding)?
1983 .is_some() =>
1984 {
1985 self.handle_module_relay_response(ctx.connection_id, frame)
1986 }
1987 ty => Ok(vec![control_error_frame(
1988 &frame,
1989 "unsupported_control_frame",
1990 format!("unsupported channel-0 frame {ty:?}"),
1991 )?]),
1992 }
1993 }
1994
1995 pub fn cleanup_connection(
1996 &self,
1997 connection_id: ConnectionId,
1998 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1999 self.caller_endpoints
2000 .lock()
2001 .unwrap_or_else(|p| p.into_inner())
2002 .remove(&connection_id);
2003 self.forwarding
2004 .operator_confirms()
2005 .connection_closed(connection_id);
2006 self.cleanup_connection_with_end_reason(
2007 connection_id,
2008 RegistrationEndReason::ConnectionClosed,
2009 )
2010 }
2011
2012 fn cleanup_connection_with_end_reason(
2013 &self,
2014 connection_id: ConnectionId,
2015 requested_reason: RegistrationEndReason,
2016 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2017 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2018 self.registry
2019 .get_module_by_connection(connection_id)?
2020 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2021 .and_then(|module| module.registration_end_reason().ok().flatten())
2022 .unwrap_or(requested_reason)
2023 } else {
2024 requested_reason
2025 };
2026 let crash_closed = self
2027 .registry
2028 .get_module_by_connection(connection_id)?
2029 .and_then(|registration| {
2030 self.forwarding
2031 .module_endpoint_for_connection(connection_id)
2032 .ok()
2033 .flatten()
2034 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2035 .map(|routes| (registration.manifest.module_id, routes))
2036 });
2037 let crash_closed = crash_closed.map(|(module_id, routes)| {
2038 let terminal = match self.supervisor.get(&module_id) {
2039 None => false,
2040 Some(module) => match module.will_recover_after_connection_loss() {
2041 Ok(will_recover) => !will_recover,
2042 Err(err) => {
2043 warn!(
2044 %module_id,
2045 error = %err,
2046 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2047 );
2048 false
2049 }
2050 },
2051 };
2052 let (reason, drained) = match self
2056 .forwarding
2057 .daemon_drain_result_for_connection(connection_id)
2058 {
2059 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2060 Ok(None) => (RouteCloseReason::Crash, false),
2061 Err(err) => {
2062 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2063 (RouteCloseReason::Crash, false)
2064 }
2065 };
2066 (module_id, routes, reason, drained, terminal)
2067 });
2068 let registrations = self.deregister_connection(connection_id, end_reason);
2069 let cleanup = if crash_closed.is_some() {
2070 self.forwarding.cleanup_connection_counted(connection_id)
2071 } else {
2072 self.forwarding
2078 .cleanup_connection(connection_id)
2079 .map(|released| crate::forwarding::ConnectionCleanup {
2080 released,
2081 abandoned_relays: 0,
2082 })
2083 };
2084 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2089 let abandoned = cleanup
2090 .as_ref()
2091 .map_or(0, |cleanup| cleanup.abandoned_relays);
2092 send_route_control_pushes(
2093 &self.forwarding,
2094 routes,
2095 ClientControlPush::RouteClosed {
2096 module_id,
2097 channels: Vec::new(),
2098 reason,
2099 drained,
2100 abandoned,
2101 excluded_subscriptions: 0,
2102 terminal: Some(terminal),
2103 },
2104 );
2105 }
2106 if let Ok(cleanup) = cleanup {
2107 self.emit_route_goodbyes(cleanup.released);
2108 }
2109 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2113 crate::supervise::notify_registration_release();
2114 self.capability_evaluator.wake_deadline_loop();
2115 self.refresh_capability_requirements();
2116 }
2117 self.supervisor.remove_spawn_subscribers(connection_id);
2118 self.activity_streams
2119 .lock()
2120 .unwrap_or_else(|p| p.into_inner())
2121 .retain(|(id, _), task| {
2122 if *id == connection_id {
2123 task.abort();
2124 false
2125 } else {
2126 true
2127 }
2128 });
2129 self.hello_launch_nonces
2132 .lock()
2133 .unwrap_or_else(|poisoned| poisoned.into_inner())
2134 .forget(connection_id);
2135 self.scopes
2136 .write()
2137 .unwrap_or_else(|poisoned| poisoned.into_inner())
2138 .release_connection(connection_id);
2139 registrations
2140 }
2141
2142 pub(crate) fn handle_route_goodbye(
2143 &self,
2144 connection_id: ConnectionId,
2145 route_channel: u16,
2146 route_epoch: u32,
2147 ) -> Result<bool, RouterError> {
2148 debug!(
2149 connection_id = connection_id.get(),
2150 route_channel, route_epoch, "handling route GOODBYE"
2151 );
2152 let RouteRelease::Removed(released_route) = self
2153 .forwarding
2154 .release_client_route(connection_id, route_channel, route_epoch)
2155 .map_err(RouterError::Forwarding)?
2156 else {
2157 return Ok(false);
2158 };
2159 self.emit_route_goodbyes(vec![released_route]);
2160 Ok(true)
2161 }
2162
2163 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2164 for released in released_routes {
2165 let frame = match Frame::build_with_version(
2166 released.negotiated_ver,
2167 FrameType::Goodbye,
2168 control_flags(),
2169 released.channel,
2170 released.epoch,
2171 0,
2172 Vec::new(),
2173 ) {
2174 Ok(frame) => frame,
2175 Err(err) => {
2176 warn!(
2177 route_channel = released.channel,
2178 error = %err,
2179 "failed to build route GOODBYE frame"
2180 );
2181 continue;
2182 }
2183 };
2184 if !released.close_on_delivery_failure() {
2185 crate::forwarding::send_module_route_goodbye(
2186 &self.counters,
2187 &released.sink,
2188 frame,
2189 released.module_id.as_deref(),
2190 "client route released",
2191 );
2192 continue;
2193 }
2194 if let Err(err) = released.sink.try_send(frame) {
2195 warn!(
2196 target_connection_id = released.connection_id.get(),
2197 route_channel = released.channel,
2198 error = %err,
2199 "route GOODBYE was not delivered to client; closing target connection"
2200 );
2201 if self
2202 .forwarding
2203 .escalate_client_delivery_failure(
2204 released.connection_id,
2205 released.channel,
2206 released.epoch,
2207 CloseReason::new(
2208 "route_goodbye_delivery_failed",
2209 format!(
2210 "failed to enqueue route GOODBYE for channel {}: {err}",
2211 released.channel
2212 ),
2213 ),
2214 crate::forwarding::UndeliveredFrame {
2215 module_id: released.module_id.as_deref(),
2216 sink: &released.sink,
2217 },
2218 )
2219 .unwrap_or(false)
2220 {
2221 self.counters.increment_goodbye_relay_client_failed();
2222 }
2223 }
2224 }
2225 }
2226
2227 fn send_abandoned_route_bind_goodbye(
2239 &self,
2240 module_sink: &crate::FrameSink,
2241 negotiated_ver: u8,
2242 module_channel: u16,
2243 module_epoch: u32,
2244 ) {
2245 let frame = match Frame::build_with_version(
2246 negotiated_ver,
2247 FrameType::Goodbye,
2248 control_flags(),
2249 module_channel,
2250 module_epoch,
2251 0,
2252 Vec::new(),
2253 ) {
2254 Ok(frame) => frame,
2255 Err(err) => {
2256 warn!(
2257 route_channel = module_channel,
2258 error = %err,
2259 "failed to build GOODBYE for abandoned route.bind"
2260 );
2261 return;
2262 }
2263 };
2264 crate::forwarding::send_module_route_goodbye(
2265 &self.counters,
2266 module_sink,
2267 frame,
2268 None,
2269 "abandoned route.bind",
2270 );
2271 }
2272
2273 fn handle_hello(
2274 &self,
2275 connection_id: ConnectionId,
2276 sink: Option<crate::FrameSink>,
2277 frame: Frame,
2278 ) -> Result<Vec<Frame>, RouterError> {
2279 debug!(
2280 connection_id = connection_id.get(),
2281 corr = frame.header.corr,
2282 "handling HELLO"
2283 );
2284 if self
2288 .registry
2289 .get_module_by_connection(connection_id)
2290 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2291 .is_some()
2292 {
2293 return Ok(vec![control_error_frame(
2294 &frame,
2295 "invalid_hello",
2296 "connection is already registered as a module",
2297 )?]);
2298 }
2299 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2300 Ok(value) => value,
2301 Err(err) => {
2302 return Ok(vec![control_error_frame(
2303 &frame,
2304 "invalid_hello",
2305 format!("malformed HELLO body: {err}"),
2306 )?])
2307 }
2308 };
2309 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2310 return Ok(vec![control_error_frame(
2311 &frame,
2312 "invalid_capability_grammar",
2313 err.to_string(),
2314 )?]);
2315 }
2316 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2317 return Ok(vec![control_error_frame(
2318 &frame,
2319 "invalid_manifest",
2320 err.to_string(),
2321 )?]);
2322 }
2323 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2324 return Ok(vec![control_error_frame(
2325 &frame,
2326 "invalid_event_declaration",
2327 err.to_string(),
2328 )?]);
2329 }
2330 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2331 return Ok(vec![control_error_frame(
2332 &frame,
2333 "invalid_manifest",
2334 err.to_string(),
2335 )?]);
2336 }
2337 if let Some(provenance) = hello_value
2338 .get("manifest")
2339 .and_then(|manifest| manifest.get("provenance"))
2340 {
2341 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2342 return Ok(vec![control_error_frame(
2343 &frame,
2344 "invalid_manifest",
2345 format!("malformed manifest provenance: {err}"),
2346 )?]);
2347 }
2348 }
2349 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2350 let module_id = hello_value["manifest"]["module_id"]
2351 .as_str()
2352 .unwrap_or("<unknown>");
2353 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2354 }
2355 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2356 Ok(hello) => hello,
2357 Err(err) => {
2358 return Ok(vec![control_error_frame(
2359 &frame,
2360 "invalid_hello",
2361 format!("malformed HELLO body: {err}"),
2362 )?])
2363 }
2364 };
2365
2366 if hello.protocol_ver != hello.manifest.protocol_ver {
2367 return Ok(vec![control_error_frame(
2368 &frame,
2369 "invalid_manifest",
2370 format!(
2371 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2372 hello.protocol_ver, hello.manifest.protocol_ver
2373 ),
2374 )?]);
2375 }
2376
2377 if hello.manifest.module_id.trim().is_empty() {
2378 return Ok(vec![control_error_frame(
2379 &frame,
2380 "invalid_manifest",
2381 "manifest module_id must not be empty",
2382 )?]);
2383 }
2384
2385 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2386 Ok(negotiated_ver) => negotiated_ver,
2387 Err(message) => {
2388 return Ok(vec![control_error_frame(
2389 &frame,
2390 "version_unsupported",
2391 message,
2392 )?])
2393 }
2394 };
2395
2396 let swap_admission = self
2404 .supervisor
2405 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2406 if swap_admission == SwapHelloAdmission::Refused {
2407 warn!(
2408 module_id = %hello.manifest.module_id,
2409 connection_id = connection_id.get(),
2410 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2411 );
2412 return Ok(vec![control_error_frame(
2413 &frame,
2414 "swap_token_invalid",
2415 format!(
2416 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2417 hello.manifest.module_id
2418 ),
2419 )?]);
2420 }
2421 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2422
2423 let nonce_verified = if swap_candidate {
2428 true
2429 } else {
2430 match self.supervisor.reserved_hello_verification(
2431 &hello.manifest.module_id,
2432 hello.launch_nonce.as_deref(),
2433 ) {
2434 Ok(verified) => verified,
2435 Err(rejection) => {
2436 let message = match rejection {
2437 ReservedHelloRejection::Exact { module_id } => format!(
2438 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2439 ),
2440 ReservedHelloRejection::Prefix {
2441 prefix,
2442 owner_module_id,
2443 } => format!(
2444 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2445 hello.manifest.module_id
2446 ),
2447 };
2448 return Ok(vec![control_error_frame(
2449 &frame,
2450 "reserved_module",
2451 message,
2452 )?]);
2453 }
2454 }
2455 };
2456
2457 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2458 &hello.manifest.module_id,
2459 hello.manifest.capabilities.as_ref(),
2460 );
2461 if let Some(refusal) = reserved_capability_refusals.first() {
2462 let capability = refusal.capability.clone();
2463 let bound_module = refusal.claimants[0].clone();
2464 log_duplicate_claim_events(reserved_capability_refusals);
2465 return Ok(vec![control_error_frame(
2466 &frame,
2467 "reserved_capability",
2468 format!(
2469 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2470 capability, bound_module, hello.manifest.module_id
2471 ),
2472 )?]);
2473 }
2474
2475 if self
2478 .forwarding
2479 .connection_has_client_routes(connection_id)
2480 .map_err(RouterError::Forwarding)?
2481 {
2482 return Ok(vec![control_error_frame(
2483 &frame,
2484 "invalid_hello",
2485 "connection has open client routes and cannot also register as a module",
2486 )?]);
2487 }
2488
2489 self.hello_launch_nonces
2494 .lock()
2495 .unwrap_or_else(|poisoned| poisoned.into_inner())
2496 .record(connection_id, hello.launch_nonce.as_deref());
2497 let control_ops = effective_module_control_ops(hello.control_ops);
2498 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2501 if swap_candidate {
2502 return self.register_swap_candidate(
2503 connection_id,
2504 sink,
2505 &frame,
2506 hello.manifest,
2507 negotiated_ver,
2508 control_ops,
2509 hello_ack,
2510 );
2511 }
2512 let registration = match self
2519 .registry
2520 .register_unpublished_verified_with_control_ops(
2521 hello.manifest,
2522 negotiated_ver,
2523 connection_id,
2524 control_ops,
2525 nonce_verified,
2526 ) {
2527 Ok(registration) => registration,
2528 Err(RegistryError::DuplicateModuleId { module_id }) => {
2529 return Ok(vec![control_error_frame(
2530 &frame,
2531 "duplicate_module_id",
2532 format!(
2533 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2534 ),
2535 )?])
2536 }
2537 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2538 return Ok(vec![control_error_frame(
2539 &frame,
2540 "invalid_module_id",
2541 err.to_string(),
2542 )?])
2543 }
2544 Err(err) => {
2545 return Ok(vec![control_error_frame(
2546 &frame,
2547 "registry_error",
2548 err.to_string(),
2549 )?])
2550 }
2551 };
2552 #[cfg(test)]
2553 self.hold_between_registry_and_forwarding();
2554
2555 let reply = if let Some(sink) = sink {
2556 let concurrency = manifest_concurrency(®istration.manifest);
2572 if let Err(err) = self.forwarding.register_module_connection_acked(
2573 connection_id,
2574 registration.manifest.module_id.clone(),
2575 negotiated_ver,
2576 concurrency,
2577 sink,
2578 hello_ack,
2579 ) {
2580 if matches!(
2585 self.deregister_connection(
2586 connection_id,
2587 RegistrationEndReason::RegistrationFailed,
2588 ),
2589 Ok(r) if !r.is_empty()
2590 ) {
2591 crate::supervise::notify_registration_release();
2592 }
2593 return Ok(vec![control_error_frame(
2594 &frame,
2595 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2596 "invalid_hello"
2597 } else {
2598 forwarding_error_code(&err)
2599 },
2600 err.to_string(),
2601 )?]);
2602 }
2603 Vec::new()
2604 } else {
2605 vec![hello_ack]
2608 };
2609
2610 if !self
2615 .registry
2616 .publish_registration(connection_id)
2617 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2618 {
2619 debug!(
2620 module_id = %registration.manifest.module_id,
2621 connection_id = connection_id.get(),
2622 "registration was removed before it could be published"
2623 );
2624 }
2625
2626 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2635 info!(
2636 module_id = %registration.manifest.module_id,
2637 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2638 );
2639 }
2640
2641 self.apply_registration_capabilities(®istration);
2642
2643 self.supervisor
2644 .record_activity_registration(®istration.manifest.module_id);
2645
2646 info!(
2647 module_id = %registration.manifest.module_id,
2648 module_version = %registration.manifest.module_version,
2649 negotiated_ver,
2650 routable_provider = manifest_provides_routable_role(®istration.manifest),
2651 connection_id = connection_id.get(),
2652 "module registered"
2653 );
2654
2655 Ok(reply)
2656 }
2657
2658 #[allow(clippy::too_many_arguments)]
2675 fn register_swap_candidate(
2676 &self,
2677 connection_id: ConnectionId,
2678 sink: Option<crate::FrameSink>,
2679 frame: &Frame,
2680 manifest: ModuleManifest,
2681 negotiated_ver: u8,
2682 control_ops: Vec<String>,
2683 hello_ack: Frame,
2684 ) -> Result<Vec<Frame>, RouterError> {
2685 let module_id = manifest.module_id.clone();
2686 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2687 manifest,
2688 negotiated_ver,
2689 connection_id,
2690 control_ops,
2691 ) {
2692 Ok(registration) => registration,
2693 Err(RegistryError::DuplicateModuleId { module_id }) => {
2694 return Ok(vec![control_error_frame(
2695 frame,
2696 "duplicate_module_id",
2697 format!(
2698 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2699 ),
2700 )?])
2701 }
2702 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2703 return Ok(vec![control_error_frame(
2704 frame,
2705 "invalid_module_id",
2706 err.to_string(),
2707 )?])
2708 }
2709 Err(err) => {
2710 return Ok(vec![control_error_frame(
2711 frame,
2712 "registry_error",
2713 err.to_string(),
2714 )?])
2715 }
2716 };
2717 #[cfg(test)]
2718 self.hold_between_registry_and_forwarding();
2719 let reply = if let Some(sink) = sink {
2720 let concurrency = manifest_concurrency(®istration.manifest);
2724 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2725 connection_id,
2726 module_id.clone(),
2727 negotiated_ver,
2728 concurrency,
2729 sink,
2730 hello_ack,
2731 ) {
2732 if matches!(
2733 self.deregister_connection(
2734 connection_id,
2735 RegistrationEndReason::RegistrationFailed,
2736 ),
2737 Ok(r) if !r.is_empty()
2738 ) {
2739 crate::supervise::notify_registration_release();
2740 }
2741 return Ok(vec![control_error_frame(
2742 frame,
2743 forwarding_error_code(&err),
2744 err.to_string(),
2745 )?]);
2746 }
2747 Vec::new()
2748 } else {
2749 vec![hello_ack]
2750 };
2751 if !self
2752 .registry
2753 .publish_registration(connection_id)
2754 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2755 {
2756 debug!(
2757 module_id = %module_id,
2758 connection_id = connection_id.get(),
2759 "swap candidate registration was removed before it could be published"
2760 );
2761 }
2762 self.supervisor.mark_swap_candidate_admitted(&module_id);
2763 info!(
2764 module_id = %module_id,
2765 module_version = %registration.manifest.module_version,
2766 negotiated_ver,
2767 ready = registration.ready,
2768 connection_id = connection_id.get(),
2769 "swap candidate registered; not routable until cutover"
2770 );
2771 Ok(reply)
2772 }
2773
2774 fn build_hello_ack(
2775 &self,
2776 frame: &Frame,
2777 negotiated_ver: u8,
2778 module_id: &str,
2779 ) -> Result<Frame, RouterError> {
2780 let ack = ModuleHelloAckBody {
2781 negotiated_ver,
2782 subc_ops: module_subc_ops(),
2783 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2784 storage: self
2785 .storage_config
2786 .as_ref()
2787 .map(|cfg| cfg.descriptor_for(module_id)),
2788 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2789 };
2790 let body = serde_json::to_vec(&ack).map_err(|err| {
2791 RouterError::backend(
2792 0,
2793 frame.header.corr,
2794 format!("failed to encode HELLO_ACK: {err}"),
2795 )
2796 })?;
2797
2798 Frame::build_with_version(
2799 negotiated_ver,
2800 FrameType::HelloAck,
2801 control_flags(),
2802 0,
2803 0,
2804 frame.header.corr,
2805 body,
2806 )
2807 .map_err(RouterError::FrameBuild)
2808 }
2809
2810 async fn handle_client_control_request(
2811 &self,
2812 ctx: &RouteCtx,
2813 frame: Frame,
2814 request: ClientControlRequest,
2815 route_open_timings: &mut Option<RouteOpenTimings>,
2816 ) -> Result<Vec<Frame>, RouterError> {
2817 if let Some(operation) = privileged_operation(&request) {
2818 let caller = self.resolve_caller(ctx.connection_id).await;
2819 let op = client_control_request_op(&request);
2820 let text = prompt_text(operation, &caller);
2821 match self.privileged_operator_prompt {
2822 PrivilegedOperatorPrompt::Log => {
2823 info!(target: "subc_daemon::privileged_prompt", operation = op,
2824 target = ?operation, caller = %caller.description(), summary = %text,
2825 "privileged request would prompt the operator");
2826 }
2827 PrivilegedOperatorPrompt::Enforce => {
2828 let key =
2829 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2830 let outcome = self
2831 .forwarding
2832 .operator_confirms()
2833 .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2834 .await;
2835 if outcome.code != "confirmed" {
2836 return Ok(vec![outcome.frame(&frame)]);
2837 }
2838 }
2839 }
2840 }
2841 match request {
2842 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2843 ClientControlRequest::CatalogList { module_id } => {
2844 self.handle_catalog_list(frame, module_id)
2845 }
2846 ClientControlRequest::RouteOpen {
2847 target,
2848 identity,
2849 consumer_identity,
2850 consumer_capabilities,
2851 role_versions,
2852 admission_facts,
2853 scope,
2854 } => {
2855 self.handle_route_open(
2856 ctx,
2857 frame,
2858 RouteOpenRequest {
2859 target,
2860 identity,
2861 consumer_identity,
2862 consumer_capabilities,
2863 role_versions,
2864 admission_facts,
2865 scope,
2866 },
2867 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2868 )
2869 .await
2870 }
2871 ClientControlRequest::RoutePoll {
2872 route_channel,
2873 route_epoch,
2874 kind,
2875 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2876 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2877 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2878 self.handle_supervisor_spawn_snapshot(frame)
2879 }
2880 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2881 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2882 }
2883 ClientControlRequest::SupervisorRestart {
2884 module_id,
2885 drain_timeout_ms,
2886 } => {
2887 self.log_supervisor_request_received(
2888 ctx,
2889 frame.header.corr,
2890 ops::SUPERVISOR_RESTART,
2891 Some(&module_id),
2892 None,
2893 )?;
2894 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2895 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2896 .await
2897 }
2898 ClientControlRequest::SupervisorSwap {
2899 module_id,
2900 ready_timeout_ms,
2901 } => {
2902 self.log_supervisor_request_received(
2903 ctx,
2904 frame.header.corr,
2905 ops::SUPERVISOR_SWAP,
2906 Some(&module_id),
2907 None,
2908 )?;
2909 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2910 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2911 .await
2912 }
2913 ClientControlRequest::SupervisorReload { module_id } => {
2914 self.log_supervisor_request_received(
2915 ctx,
2916 frame.header.corr,
2917 ops::SUPERVISOR_RELOAD,
2918 Some(&module_id),
2919 None,
2920 )?;
2921 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2922 self.handle_supervisor_reload(frame, module_id, requester)
2923 .await
2924 }
2925 ClientControlRequest::SupervisorRescan { preview } => {
2926 if !preview {
2927 self.log_supervisor_request_received(
2928 ctx,
2929 frame.header.corr,
2930 ops::SUPERVISOR_RESCAN,
2931 None,
2932 None,
2933 )?;
2934 }
2935 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2936 self.handle_supervisor_rescan(
2937 frame,
2938 if preview {
2939 RescanMode::Preview
2940 } else {
2941 RescanMode::Execute
2942 },
2943 requester,
2944 )
2945 .await
2946 }
2947 ClientControlRequest::SupervisorRescanChecked {
2948 preview,
2949 candidate,
2950 expect,
2951 } => {
2952 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
2953 .await
2954 }
2955 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2956 self.log_supervisor_request_received(
2957 ctx,
2958 frame.header.corr,
2959 ops::SUPERVISOR_RELEASE_RESERVED,
2960 Some(&module_id),
2961 None,
2962 )?;
2963 self.handle_supervisor_release_reserved(frame, module_id)
2964 .await
2965 }
2966 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2967 self.log_supervisor_request_received(
2968 ctx,
2969 frame.header.corr,
2970 ops::SUPERVISOR_SET_ENABLED,
2971 Some(&module_id),
2972 Some(enabled),
2973 )?;
2974 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2975 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2976 .await
2977 }
2978 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2979 self.handle_supervisor_health_probe(frame, module_id).await
2980 }
2981 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2982 ClientControlRequest::SupervisorRoutes { module_id } => {
2983 self.handle_supervisor_routes(frame, module_id)
2984 }
2985 ClientControlRequest::SupervisorProvenance { module_id } => {
2986 self.handle_supervisor_provenance(frame, module_id).await
2987 }
2988 ClientControlRequest::SupervisorStderrTail {
2989 module_id,
2990 max_lines,
2991 max_bytes,
2992 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2993 ClientControlRequest::SupervisorTerminals { module_id } => {
2994 self.handle_supervisor_terminals(frame, module_id).await
2995 }
2996 }
2997 }
2998
2999 fn log_supervisor_request_received(
3000 &self,
3001 ctx: &RouteCtx,
3002 corr: u64,
3003 op: &'static str,
3004 module_id: Option<&str>,
3005 enabled: Option<bool>,
3006 ) -> Result<(), RouterError> {
3007 let caller = self.supervisor_requester(ctx, corr)?;
3008 let caller = principal_label(&caller);
3009 info!(
3012 target: "control",
3013 op,
3014 module_id,
3015 enabled,
3016 connection_id = ctx.connection_id.get(),
3017 caller = %caller,
3018 "supervisor request received"
3019 );
3020 Ok(())
3021 }
3022
3023 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3024 Ok(self
3025 .registry
3026 .get_module_by_connection(ctx.connection_id)
3027 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3028 .map(|registration| Principal::Reserved {
3029 module_id: registration.manifest.module_id,
3030 })
3031 .unwrap_or(Principal::Direct))
3032 }
3033
3034 fn handle_module_control_request(
3035 &self,
3036 connection_id: ConnectionId,
3037 frame: Frame,
3038 request: ModuleControlRequestFromModule,
3039 ) -> Result<Vec<Frame>, RouterError> {
3040 match request {
3041 ModuleControlRequestFromModule::CatalogUpdate {
3042 provides,
3043 capabilities,
3044 ready,
3045 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3046 ModuleControlRequestFromModule::LiveRoots {} => {
3047 let registered = self
3048 .registry
3049 .get_module_by_connection(connection_id)
3050 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3051 let Some(registration) = registered else {
3052 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3053 };
3054 let response = self
3055 .forwarding
3056 .live_roots(®istration.manifest.module_id)
3057 .map_err(RouterError::Forwarding)?;
3058 Ok(vec![control_response_body_frame(
3059 &frame,
3060 &response,
3061 "ModuleControlResponseToModule::LiveRoots",
3062 )?])
3063 }
3064 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3065 self.handle_scope_sync(connection_id, frame, generation, scopes)
3066 }
3067 ModuleControlRequestFromModule::ScopeApply {
3068 generation,
3069 upsert,
3070 end,
3071 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3072 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3073 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3074 }
3075 }
3076 }
3077
3078 fn handle_scope_served_by(
3079 &self,
3080 connection_id: ConnectionId,
3081 frame: Frame,
3082 ) -> Result<Vec<Frame>, RouterError> {
3083 let registered = self
3084 .registry
3085 .get_module_by_connection(connection_id)
3086 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3087 let Some(registered) = registered else {
3088 return Ok(vec![control_error_frame(
3089 &frame,
3090 "not_registered",
3091 "scope.served_by requires an active module registration owned by this connection",
3092 )?]);
3093 };
3094 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3095 {
3096 Ok(request) => request,
3097 Err(err) => {
3098 return Ok(vec![control_error_frame(
3099 &frame,
3100 "invalid_control_body",
3101 format!("malformed scope.served_by body: {err}"),
3102 )?])
3103 }
3104 };
3105 let answer = self
3106 .scopes
3107 .read()
3108 .unwrap_or_else(|poisoned| poisoned.into_inner())
3109 .served_by(
3110 ®istered.manifest.module_id,
3111 &request.scope_ref,
3112 request.scope_epoch,
3113 &request.module_id,
3114 );
3115 let reply = subc_control::ScopeServedByReply::new(
3116 answer,
3117 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3118 );
3119 Ok(vec![control_response_body_frame(
3120 &frame,
3121 &reply,
3122 "ScopeServedByReply",
3123 )?])
3124 }
3125
3126 fn handle_operator_confirm(
3127 &self,
3128 ctx: &RouteCtx,
3129 frame: Frame,
3130 ) -> Result<Vec<Frame>, RouterError> {
3131 use crate::operator_confirm::{audit, Outcome};
3132 let registration = self
3133 .registry
3134 .get_module_by_connection(ctx.connection_id)
3135 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3136 let Some(registration) = registration else {
3137 let outcome = Outcome::refusal("not_registered");
3138 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3139 return Ok(vec![outcome.frame(&frame)]);
3140 };
3141 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3142 Ok(request) => request,
3143 Err(_) => {
3144 let outcome = Outcome::refusal("invalid_control_body");
3145 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3146 return Ok(vec![outcome.frame(&frame)]);
3147 }
3148 };
3149 let module_id = registration.manifest.module_id;
3150 let nonce = self
3154 .hello_launch_nonces
3155 .lock()
3156 .unwrap_or_else(|p| p.into_inner())
3157 .nonce(ctx.connection_id)
3158 .map(str::to_owned);
3159 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3160 self.supervisor
3161 .spawned_consumer_authorized(&module_id, nonce)
3162 });
3163 let confirms = self.forwarding.operator_confirms();
3164 self.forwarding
3165 .with_operator_route(
3166 ctx.connection_id,
3167 request.route_channel,
3168 request.route_epoch,
3169 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3170 )
3171 .map_err(RouterError::Forwarding)
3172 }
3173
3174 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3177 let mut scopes = ScopeRecords::from(scopes);
3178 for (index, record) in scopes.records.iter_mut().enumerate() {
3179 if let Some(refusal) = Self::canonicalize_workdir(record) {
3180 scopes.refusals.insert(index, refusal);
3181 }
3182 }
3183 scopes
3184 }
3185
3186 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3187 let workdir = record.workdir.as_mut()?;
3188 let path = std::path::Path::new(workdir);
3189 if !path.is_absolute() {
3190 return Some((
3191 error_codes::INVALID_WORKDIR,
3192 "workdir must be absolute".to_string(),
3193 ));
3194 }
3195 match ProjectRootId::from_path_allowing_missing(path) {
3196 Ok(root) => match root.as_path().to_str() {
3197 Some(canonical) => {
3198 *workdir = canonical.to_string();
3199 None
3200 }
3201 None => Some((
3202 error_codes::INVALID_WORKDIR,
3203 "canonical workdir is not UTF-8".to_string(),
3204 )),
3205 },
3206 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3207 }
3208 }
3209
3210 fn handle_scope_sync(
3214 &self,
3215 connection_id: ConnectionId,
3216 frame: Frame,
3217 generation: u64,
3218 scopes: Vec<ScopeRecord>,
3219 ) -> Result<Vec<Frame>, RouterError> {
3220 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3221 }
3222
3223 fn handle_scope_change(
3224 &self,
3225 connection_id: ConnectionId,
3226 frame: Frame,
3227 generation: u64,
3228 scopes: Vec<ScopeRecord>,
3229 end: Option<Vec<ScopeEnd>>,
3230 ) -> Result<Vec<Frame>, RouterError> {
3231 let is_apply = end.is_some();
3232 let Some(registration) = self
3233 .registry
3234 .get_module_by_connection(connection_id)
3235 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3236 else {
3237 return Ok(vec![control_error_frame(
3238 &frame,
3239 "not_registered",
3240 "scope.sync and scope.apply require an active module registration owned by this connection",
3241 )?]);
3242 };
3243 let owner = registration.manifest.module_id;
3244 let scopes = Self::prepare_scope_records(scopes);
3245 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3246 let is_current_launch = |connection: ConnectionId| {
3247 self.hello_launch_nonces
3248 .lock()
3249 .unwrap_or_else(|poisoned| poisoned.into_inner())
3250 .presented(connection, current_nonce.as_deref())
3251 };
3252 let mut table = self
3257 .scopes
3258 .write()
3259 .unwrap_or_else(|poisoned| poisoned.into_inner());
3260 let outcome = match end {
3261 Some(end) => table.apply(
3262 &owner,
3263 connection_id,
3264 is_current_launch,
3265 generation,
3266 scopes,
3267 end,
3268 ),
3269 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3270 };
3271 let drained = match &outcome {
3272 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3273 Err(_) => Vec::new(),
3274 };
3275 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3276 drop(table);
3277 if has_deadlines {
3278 self.scope_deadline_added.notify_one();
3279 }
3280 match outcome {
3281 Ok(applied) => {
3282 let counts = ScopeOutcomeCounts::of(&applied.results);
3283 info!(
3284 owner = %owner,
3285 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3286 generation,
3287 records = applied.results.len(),
3288 created = counts.created,
3289 replaced = counts.replaced,
3290 updated = counts.updated,
3291 unchanged = counts.unchanged,
3292 refused = counts.refused,
3293 ended = applied.ended.len(),
3294 tag_changes = applied.tag_changes.len(),
3295 routes_closed = drained.len(),
3296 "scope change accepted"
3297 );
3298 for refused in applied
3304 .results
3305 .iter()
3306 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3307 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3308 {
3309 warn!(
3310 owner = %owner,
3311 generation,
3312 scope_ref = %refused.scope_ref,
3313 scope_epoch = refused.scope_epoch,
3314 code = refused.code.as_deref().unwrap_or(""),
3315 "scope record refused"
3316 );
3317 }
3318 self.close_scope_drained_routes(drained);
3319 let response = if is_apply {
3320 ModuleControlResponseToModule::ScopeApply {
3321 generation,
3322 results: applied.results,
3323 end_results: applied.end_results,
3324 ended: applied.ended,
3325 }
3326 } else {
3327 ModuleControlResponseToModule::ScopeSync {
3328 generation,
3329 results: applied.results,
3330 ended: applied.ended,
3331 }
3332 };
3333 Ok(vec![control_response_body_frame(
3334 &frame,
3335 &response,
3336 "scope change response",
3337 )?])
3338 }
3339 Err(refusal) => {
3340 info!(
3341 owner = %owner,
3342 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3343 generation,
3344 code = refusal.code,
3345 "scope change refused"
3346 );
3347 Ok(vec![control_error_frame(
3348 &frame,
3349 refusal.code,
3350 refusal.message,
3351 )?])
3352 }
3353 }
3354 }
3355
3356 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3362 if drained.is_empty() {
3363 return;
3364 }
3365 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3366 BTreeMap::new();
3367 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3368 for route in drained {
3369 warn!(
3370 module_id = %route.module_id,
3371 reason = ?route.reason,
3372 client_connection_id = route.client.connection_id.get(),
3373 route_channel = route.client.channel,
3374 "closing route because its scope changed"
3375 );
3376 pushes
3377 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3378 .or_insert_with(|| (route.reason, Vec::new()))
3379 .1
3380 .push(EndpointRoute {
3381 goodbye_target: route.client.clone(),
3382 principal: Principal::Unverified,
3383 bound_at: Instant::now(),
3384 draining: false,
3385 drain_reason: None,
3386 });
3387 goodbyes.push(route.module);
3388 goodbyes.push(route.client);
3389 }
3390 for ((module_id, _), (reason, routes)) in pushes {
3391 send_route_control_pushes(
3392 &self.forwarding,
3393 routes,
3394 ClientControlPush::RouteClosed {
3395 module_id,
3396 channels: Vec::new(),
3397 reason,
3398 drained: false,
3399 abandoned: 0,
3400 excluded_subscriptions: 0,
3401 terminal: Some(false),
3402 },
3403 );
3404 }
3405 self.emit_route_goodbyes(goodbyes);
3406 }
3407
3408 fn handle_scope_describe(
3411 &self,
3412 connection_id: ConnectionId,
3413 frame: Frame,
3414 owner: Principal,
3415 scope_ref: String,
3416 ) -> Result<Vec<Frame>, RouterError> {
3417 let registered = self
3418 .registry
3419 .get_module_by_connection(connection_id)
3420 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3421 let Some(registered) = registered else {
3422 return Ok(vec![control_error_frame(
3423 &frame,
3424 "not_registered",
3425 "scope.describe requires an active module registration owned by this connection",
3426 )?]);
3427 };
3428 let mut description = self
3429 .scopes
3430 .read()
3431 .unwrap_or_else(|poisoned| poisoned.into_inner())
3432 .describe(&owner, &scope_ref);
3433 if !registered
3436 .manifest
3437 .capabilities
3438 .as_ref()
3439 .is_some_and(|caps| {
3440 caps.provides
3441 .iter()
3442 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3443 })
3444 {
3445 if let Some(stamp) = description.stamp.as_mut() {
3446 stamp.attributes.on_behalf_of_agent = None;
3447 }
3448 }
3449 let owner_configured = match &owner {
3450 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3455 _ => false,
3456 };
3457 let response = ModuleControlResponseToModule::ScopeDescribe {
3458 status: description.status,
3459 scope_epoch: description.scope_epoch,
3460 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3461 owner_synced: description.owner_synced,
3462 owner_configured,
3463 scope: description.stamp,
3464 };
3465 Ok(vec![control_response_body_frame(
3466 &frame,
3467 &response,
3468 "ModuleControlResponseToModule::ScopeDescribe",
3469 )?])
3470 }
3471
3472 fn handle_catalog_update(
3473 &self,
3474 connection_id: ConnectionId,
3475 frame: Frame,
3476 provides: Vec<ProviderRole>,
3477 capabilities: Option<CapabilityDeclarations>,
3478 ready: Option<bool>,
3479 ) -> Result<Vec<Frame>, RouterError> {
3480 self.refresh_capability_requirements();
3481 let Some(registration) = self
3482 .registry
3483 .get_module_by_connection(connection_id)
3484 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3485 else {
3486 return Ok(vec![control_error_frame(
3487 &frame,
3488 "not_registered",
3489 "catalog.update requires an active module registration owned by this connection",
3490 )?]);
3491 };
3492
3493 if let Some(message) =
3494 catalog_update_frozen_field_message(®istration.manifest, &provides)
3495 {
3496 return Ok(vec![control_error_frame(
3497 &frame,
3498 "catalog_update_frozen_field",
3499 message,
3500 )?]);
3501 }
3502
3503 let mut candidate = registration.manifest.clone();
3504 candidate.provides = provides.clone();
3505 candidate.capabilities = capabilities
3506 .clone()
3507 .or_else(|| registration.manifest.capabilities.clone());
3508 if let Some(rich_ui) = &candidate.rich_ui {
3509 if let Err(err) = rich_ui.validate() {
3510 return Ok(vec![control_error_frame(
3511 &frame,
3512 "invalid_manifest",
3513 err.to_string(),
3514 )?]);
3515 }
3516 }
3517 if let Err(err) = candidate.validate_capability_grammar() {
3518 return Ok(vec![control_error_frame(
3519 &frame,
3520 "invalid_capability_grammar",
3521 err.to_string(),
3522 )?]);
3523 }
3524
3525 let mut conflicts = self
3528 .capability_evaluator
3529 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3530 if let Some(conflict) = conflicts.first() {
3531 let message = format!(
3532 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3533 conflict.capability, conflict.claimants[0], candidate.module_id
3534 );
3535 for conflict in &mut conflicts {
3536 conflict.source = DuplicateClaimSource::CatalogUpdate;
3537 }
3538 log_duplicate_claim_events(conflicts);
3539 return Ok(vec![control_error_frame(
3540 &frame,
3541 "reserved_capability",
3542 message,
3543 )?]);
3544 }
3545
3546 let updated = self
3547 .registry
3548 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3549 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3550 if updated.is_none() {
3551 return Ok(vec![control_error_frame(
3552 &frame,
3553 "not_registered",
3554 "catalog.update requires an active module registration owned by this connection",
3555 )?]);
3556 }
3557 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3558 log_duplicate_claim_events(
3559 self.capability_evaluator
3560 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3561 );
3562 }
3563 if capability_census_trigger(
3564 registration.manifest.capabilities.as_ref(),
3565 updated
3566 .as_ref()
3567 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3568 ) {
3569 self.enforce_capability_denies();
3570 }
3571 self.refresh_capability_requirements();
3572
3573 let response = ModuleControlResponseToModule::CatalogUpdate {};
3574 control_response_body_frame(
3575 &frame,
3576 &response,
3577 "ModuleControlResponseToModule::CatalogUpdate",
3578 )
3579 .map(|frame| vec![frame])
3580 }
3581
3582 pub(crate) fn with_activity_ledger_config(
3583 mut self,
3584 run_dir: &Path,
3585 incarnation: String,
3586 config: crate::activity::ActivityConfig,
3587 ) -> Self {
3588 self.activity = Some(crate::activity::ActivityLedger::open_config(
3589 run_dir.join("activity.jsonl"),
3590 incarnation,
3591 config,
3592 ));
3593 self.supervisor.set_activity(self.activity.clone());
3594 self
3595 }
3596
3597 pub(crate) async fn configure_activity(
3598 &self,
3599 config: crate::activity::ActivityConfig,
3600 ) -> Result<(), &'static str> {
3601 match &self.activity {
3602 Some(ledger) => ledger.configure(config).await,
3603 None => Ok(()),
3604 }
3605 }
3606
3607 fn cancel_activity_subscription(
3608 &self,
3609 ctx: &RouteCtx,
3610 frame: &Frame,
3611 ) -> Result<bool, RouterError> {
3612 let task = self
3613 .activity_streams
3614 .lock()
3615 .unwrap_or_else(|p| p.into_inner())
3616 .remove(&(ctx.connection_id, frame.header.corr));
3617 let Some(task) = task else {
3618 return Ok(false);
3619 };
3620 task.abort();
3621 let end = Frame::build_with_version(
3622 response_version(frame),
3623 FrameType::StreamEnd,
3624 control_flags(),
3625 0,
3626 0,
3627 frame.header.corr,
3628 Vec::new(),
3629 )
3630 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3631 let sink = ctx.egress.clone();
3632 tokio::spawn(async move {
3633 let _ = sink.send(end).await;
3634 });
3635 Ok(true)
3636 }
3637
3638 async fn handle_activity_read(
3639 &self,
3640 ctx: &RouteCtx,
3641 frame: Frame,
3642 ) -> Result<Vec<Frame>, RouterError> {
3643 use subc_control::{
3644 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3645 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3646 };
3647 #[derive(Deserialize)]
3648 struct Identity {
3649 #[serde(default)]
3650 consumer_identity: Option<ConsumerIdentity>,
3651 }
3652 let principal = match self
3656 .forwarding
3657 .module_id_for_connection(ctx.connection_id)
3658 .map_err(RouterError::Forwarding)?
3659 {
3660 Some(module_id) => Ok(Principal::Reserved { module_id }),
3661 None => {
3662 let identity: Identity = match serde_json::from_slice(&frame.body) {
3663 Ok(identity) => identity,
3664 Err(_) => {
3665 return Ok(vec![control_error_frame(
3666 &frame,
3667 ACTIVITY_MALFORMED,
3668 "malformed activity read",
3669 )?])
3670 }
3671 };
3672 self.route_open_principal(&frame, identity.consumer_identity)?
3673 }
3674 };
3675 let admitted = matches!(principal, Ok(Principal::Direct))
3676 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3677 if !admitted {
3678 return Ok(vec![control_error_frame(
3679 &frame,
3680 ACTIVITY_READ_FORBIDDEN,
3681 "activity reads require an operator or trusted mirror principal",
3682 )?]);
3683 }
3684 let Some(ledger) = &self.activity else {
3685 return Ok(vec![control_error_frame(
3686 &frame,
3687 ACTIVITY_UNAVAILABLE,
3688 "activity ledger unavailable",
3689 )?]);
3690 };
3691 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3692 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3693 {
3694 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3695 Ok(request) => ledger.query(request).await,
3696 Err(_) => Err(ACTIVITY_MALFORMED),
3697 };
3698 return match result {
3699 Ok(reply) => Ok(vec![control_response_body_frame(
3700 &frame,
3701 &reply,
3702 ops::ACTIVITY_QUERY,
3703 )?]),
3704 Err(code) => Ok(vec![control_error_frame(
3705 &frame,
3706 code,
3707 "activity query refused",
3708 )?]),
3709 };
3710 }
3711 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3712 Ok(request) => request,
3713 Err(_) => {
3714 return Ok(vec![control_error_frame(
3715 &frame,
3716 ACTIVITY_MALFORMED,
3717 "malformed activity subscription",
3718 )?])
3719 }
3720 };
3721 let mut subscription = match ledger.subscribe(request.since).await {
3722 Ok(subscription) => subscription,
3723 Err(code) => {
3724 return Ok(vec![control_error_frame(
3725 &frame,
3726 code,
3727 "activity subscription refused",
3728 )?])
3729 }
3730 };
3731 let key = (ctx.connection_id, frame.header.corr);
3732 let sink = ctx.egress.clone();
3733 let mut streams = self
3734 .activity_streams
3735 .lock()
3736 .unwrap_or_else(|p| p.into_inner());
3737 streams.retain(|_, task| !task.is_finished());
3738 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3739 return Ok(vec![control_error_frame(
3740 &frame,
3741 "duplicate_subscription",
3742 "activity subscription correlation id already in use",
3743 )?]);
3744 }
3745 let task = tokio::spawn(async move {
3746 loop {
3747 let entry = if let Some(entry) = subscription.replay.pop_front() {
3748 Some(entry)
3749 } else {
3750 subscription.receiver.recv().await
3751 };
3752 let Some(entry) = entry else {
3753 break;
3754 };
3755 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3756 return;
3757 };
3758 let Ok(next) = Frame::build_with_version(
3759 response_version(&frame),
3760 FrameType::StreamData,
3761 control_flags(),
3762 0,
3763 0,
3764 frame.header.corr,
3765 body,
3766 ) else {
3767 return;
3768 };
3769 if sink.send(next).await.is_err() {
3770 return;
3771 }
3772 }
3773 if let Some(code) = subscription.terminal_code() {
3776 if let Ok(end) =
3777 control_error_frame(&frame, code, "activity subscriber fell behind")
3778 {
3779 let _ = sink.send(end).await;
3780 }
3781 }
3782 });
3783 streams.insert(key, task.abort_handle());
3784 Ok(Vec::new())
3785 }
3786
3787 async fn handle_activity_record(
3788 &self,
3789 ctx: &RouteCtx,
3790 frame: Frame,
3791 ) -> Result<Vec<Frame>, RouterError> {
3792 use subc_control::{
3793 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3794 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3795 };
3796 let result = async {
3797 let module = self
3798 .forwarding
3799 .module_id_for_connection(ctx.connection_id)
3800 .ok()
3801 .flatten()
3802 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3803 let request: ActivityRecordRequest =
3804 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3805 crate::activity::validate(&module, &request)?;
3806 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3807 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3808 let at_ms = std::time::SystemTime::now()
3809 .duration_since(std::time::UNIX_EPOCH)
3810 .unwrap_or_default()
3811 .as_millis() as u64;
3812 ledger.record(module, request, actor, at_ms).await
3813 }
3814 .await;
3815 match result {
3816 Ok(entry) => Ok(vec![control_response_body_frame(
3817 &frame,
3818 &ActivityRecordResponse { seq: entry.seq },
3819 "activity.record",
3820 )?]),
3821 Err(code) => Ok(vec![control_error_frame(
3822 &frame,
3823 code,
3824 "activity recording refused",
3825 )?]),
3826 }
3827 }
3828
3829 fn activity_actor(
3830 &self,
3831 connection: ConnectionId,
3832 module: &str,
3833 on: Option<&subc_control::ActivityOn>,
3834 ) -> Result<subc_control::ActivityActor, &'static str> {
3835 use crate::forwarding::{DataRoute, DataRouteState};
3836 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3837 let principal = |id: &str| Principal::Reserved {
3838 module_id: id.to_owned(),
3839 };
3840 let mut actor = ActivityActor {
3841 principal: principal(module),
3842 scope_owner: None,
3843 agent_id: None,
3844 run_id: None,
3845 flow_id: None,
3846 project_root: None,
3847 workdir: None,
3848 project_id: None,
3849 };
3850 let scope = match on {
3851 None => return Ok(actor),
3852 Some(ActivityOn::Scope(scope)) => {
3853 let selector = subc_protocol::scope::ScopeSelector::new(
3854 principal(module),
3855 scope.scope_ref.clone(),
3856 Some(scope.epoch),
3857 );
3858 self.scopes
3859 .read()
3860 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3861 .attribute(&principal(module), module, &selector, true)
3862 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3863 }
3864 Some(ActivityOn::Route(route)) => {
3865 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3866 .forwarding
3867 .lookup_data_route(connection, route.channel, route.epoch)
3868 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3869 else {
3870 return Err(ACTIVITY_ACTOR_UNKNOWN);
3871 };
3872 if binding.module_id != module {
3873 return Err(ACTIVITY_ACTOR_UNKNOWN);
3874 }
3875 actor.principal = binding.principal.clone();
3876 actor.project_root = binding
3877 .project_root
3878 .as_ref()
3879 .map(|root| root.as_path().to_path_buf());
3880 let Some(bound) = &binding.scope else {
3881 return Ok(actor);
3882 };
3883 let owner = principal(&bound.owner);
3884 let selector = subc_protocol::scope::ScopeSelector::new(
3885 owner.clone(),
3886 bound.scope_ref.clone(),
3887 Some(bound.tag.scope_epoch),
3888 );
3889 let admitted = self
3890 .scopes
3891 .read()
3892 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3893 .attribute(&owner, module, &selector, true)
3894 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3895 if admitted.tag != bound.tag {
3896 return Err(ACTIVITY_ACTOR_UNKNOWN);
3897 }
3898 admitted
3899 }
3900 };
3901 actor.scope_owner = Some(scope.stamp.owner);
3902 actor.agent_id = scope.stamp.attributes.agent_id;
3903 actor.run_id = scope.stamp.attributes.run_id;
3904 actor.flow_id = scope.stamp.attributes.flow_id;
3905 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
3906 actor.project_id = scope.stamp.project_id;
3907 Ok(actor)
3908 }
3909
3910 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3911 self.refresh_capability_requirements();
3912 let mut counters = self.counters.snapshot();
3918 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3919 obj.insert(
3920 "activity_unparseable_lines".into(),
3921 ledger.read_errors().into(),
3922 );
3923 obj.insert(
3924 "activity_lifecycle_append_failures".into(),
3925 ledger.lifecycle_failures().into(),
3926 );
3927 obj.insert(
3928 "activity_started_digest_skips".into(),
3929 ledger.lifecycle_digest_skips().into(),
3930 );
3931 }
3932 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3933 self.forwarding.client_route_concentration(),
3934 counters.as_object_mut(),
3935 ) {
3936 obj.insert(
3937 "client_connections_with_routes".into(),
3938 connections_with_routes.into(),
3939 );
3940 obj.insert("max_routes_on_one_connection".into(), max.into());
3941 }
3942 if let (Some(open_breakers), Some(obj)) = (
3948 self.route_bind_breakers.open_snapshot(),
3949 counters.as_object_mut(),
3950 ) {
3951 obj.insert("route_bind_breakers_open".into(), open_breakers);
3952 }
3953 let response = ClientControlResponse::ServerDescribe {
3954 protocol_ver: PROTOCOL_VERSION,
3955 subc_ops: subc_ops(),
3956 capabilities: self.describe_capabilities(),
3957 connected_clients: self.connected_clients.count(),
3958 counters: Some(counters),
3959 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3960 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3961 capability_requirements: self.capability_requirement_statuses(),
3962 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3963 config_path: self
3964 .rescan
3965 .as_ref()
3966 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
3967 };
3968 Ok(vec![control_response_body_frame(
3969 &frame,
3970 &response,
3971 "ClientControlResponse::ServerDescribe",
3972 )?])
3973 }
3974
3975 fn handle_catalog_list(
3976 &self,
3977 frame: Frame,
3978 module_id: Option<String>,
3979 ) -> Result<Vec<Frame>, RouterError> {
3980 let (generation, modules) = self
3981 .registry
3982 .list_modules_with_nonce_verification()
3983 .map_err(|err| {
3984 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3985 })?;
3986 let entries = modules
3987 .into_iter()
3988 .filter(|(registration, _)| {
3989 module_id
3990 .as_deref()
3991 .map(|wanted| registration.manifest.module_id == wanted)
3992 .unwrap_or(true)
3993 })
3994 .map(|(registration, nonce_verified)| {
3995 let not_ready = self.not_ready_reason(®istration);
3996 let roles = registration.manifest.provides;
3997 CatalogEntry::new(
3998 registration.manifest.module_id,
3999 roles,
4000 registration.control_ops,
4001 )
4002 .with_ready(not_ready.is_none())
4003 .with_not_ready(not_ready)
4004 .with_module_version(Some(registration.manifest.module_version))
4005 .with_capabilities(registration.manifest.capabilities)
4006 .with_self_signals(registration.manifest.self_signals)
4007 .with_events(registration.manifest.events)
4008 .with_operation_delivery(registration.manifest.operation_delivery)
4009 .with_operation_payload(registration.manifest.operation_payload)
4010 .with_rich_ui(registration.manifest.rich_ui)
4011 .with_role_declarations(registration.manifest.role_declarations)
4012 .with_nonce_verified(nonce_verified)
4013 })
4014 .collect();
4015 let response = ClientControlResponse::CatalogList {
4016 generation,
4017 modules: entries,
4018 subc_ops: subc_ops(),
4019 };
4020 Ok(vec![control_response_body_frame(
4021 &frame,
4022 &response,
4023 "ClientControlResponse::CatalogList",
4024 )?])
4025 }
4026
4027 fn attest_route_open_consumer(
4032 &self,
4033 consumer_identity: Option<&ConsumerIdentity>,
4034 ) -> Option<Principal> {
4035 let Some(consumer_identity) = consumer_identity else {
4036 return Some(Principal::Direct);
4037 };
4038 self.supervisor
4039 .spawned_consumer_authorized(
4040 &consumer_identity.module_id,
4041 &consumer_identity.launch_nonce,
4042 )
4043 .then(|| Principal::Reserved {
4044 module_id: consumer_identity.module_id.clone(),
4045 })
4046 }
4047
4048 fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4052 let Ok(ClientControlRequest::RouteOpen {
4053 identity,
4054 consumer_identity,
4055 scope,
4056 ..
4057 }) = parse_client_control_request(&frame.body)
4058 else {
4059 return RouteOpenCaller::unknown();
4060 };
4061 let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4062 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4063 }
4064
4065 fn route_open_principal(
4066 &self,
4067 frame: &Frame,
4068 consumer_identity: Option<ConsumerIdentity>,
4069 ) -> Result<Result<Principal, Frame>, RouterError> {
4070 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4071 self.route_open_principal_attested(frame, consumer_identity, attested)
4072 }
4073
4074 fn route_open_principal_attested(
4078 &self,
4079 frame: &Frame,
4080 consumer_identity: Option<ConsumerIdentity>,
4081 attested: Option<Principal>,
4082 ) -> Result<Result<Principal, Frame>, RouterError> {
4083 if let Some(principal) = attested {
4084 return Ok(Ok(principal));
4085 }
4086 let Some(consumer_identity) = consumer_identity else {
4087 return Ok(Ok(Principal::Direct));
4088 };
4089
4090 Ok(Err(control_error_frame(
4091 frame,
4092 "bad_consumer_identity",
4093 format!(
4094 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4095 consumer_identity.module_id
4096 ),
4097 )?))
4098 }
4099
4100 #[allow(clippy::too_many_arguments)]
4107 fn route_open_refusal_frame(
4108 &self,
4109 ctx: &RouteCtx,
4110 caller: &RouteOpenCaller,
4111 frame: &Frame,
4112 module_id: &str,
4113 reason: &'static str,
4114 code: &'static str,
4115 message: impl Into<String>,
4116 ) -> Result<Frame, RouterError> {
4117 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4118 control_error_frame(frame, code, message.into())
4119 }
4120
4121 #[allow(clippy::too_many_arguments)]
4139 fn route_open_breaker_refusal_frame(
4140 &self,
4141 ctx: &RouteCtx,
4142 caller: &RouteOpenCaller,
4143 frame: &Frame,
4144 module_id: &str,
4145 consecutive_timeouts: u32,
4146 retry_in: Duration,
4147 probe_in_flight: bool,
4148 ) -> Result<Frame, RouterError> {
4149 self.counters
4150 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4151 debug!(
4152 target: "control",
4153 code = "module_timeout",
4154 module_id = ?module_id,
4155 connection_id = ctx.connection_id.get(),
4156 principal = %caller.principal,
4157 scope = %caller.scope,
4158 harness = %caller.harness,
4159 consecutive_timeouts,
4160 retry_in_ms = retry_in.as_millis() as u64,
4161 probe_in_flight,
4162 "route.open refused by open bind-relay breaker"
4163 );
4164 let detail = if probe_in_flight {
4171 "one test route is already being tried; retry once it settles".to_string()
4172 } else {
4173 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4174 };
4175 control_error_frame(
4176 frame,
4177 "module_timeout",
4178 format!(
4179 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4180 timed out in a row); {detail}; its established routes are unaffected"
4181 ),
4182 )
4183 }
4184
4185 fn observe_route_open_refusal(
4198 &self,
4199 ctx: &RouteCtx,
4200 caller: &RouteOpenCaller,
4201 module_id: &str,
4202 reason: &'static str,
4203 code: &'static str,
4204 ) {
4205 self.counters.increment_route_open_refused(code);
4206 info!(
4207 target: "control",
4208 code,
4209 reason,
4210 module_id = ?module_id,
4211 connection_id = ctx.connection_id.get(),
4212 principal = %caller.principal,
4213 scope = %caller.scope,
4214 harness = %caller.harness,
4215 "route.open refused"
4216 );
4217 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4218 self.route_outages.record_not_serving(module_id, reason);
4219 }
4220 }
4221
4222 fn observe_route_open_accept(
4267 &self,
4268 ctx: &RouteCtx,
4269 module_id: &str,
4270 principal: &str,
4271 scope: &str,
4272 ) {
4273 self.route_outages.record_accepted(module_id);
4274 self.counters.increment_route_open_accepted(principal);
4275 info!(
4276 target: "control",
4277 principal,
4278 module_id,
4279 connection_id = ctx.connection_id.get(),
4280 scope = %scope,
4281 "route.open accepted"
4282 );
4283 }
4284
4285 fn supervised_absent_route_open_refusal_frame(
4286 &self,
4287 ctx: &RouteCtx,
4288 caller: &RouteOpenCaller,
4289 frame: &Frame,
4290 module_id: &str,
4291 code: &'static str,
4292 status: &crate::supervise::ModuleStatus,
4293 ) -> Result<Frame, RouterError> {
4294 self.counters.increment_route_open_refused(code);
4295 info!(
4296 target: "control",
4297 code,
4298 reason = "supervised_not_registered",
4299 module_id = ?module_id,
4300 connection_id = ctx.connection_id.get(),
4301 principal = %caller.principal,
4302 scope = %caller.scope,
4303 harness = %caller.harness,
4304 state = %status.state,
4305 enabled = status.enabled,
4306 live = status.live,
4307 "route.open refused"
4308 );
4309 self.route_outages
4313 .record_not_serving(module_id, "supervised_not_registered");
4314 let message = if let Some(pid) = status.waiting_for_previous_pid {
4315 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4316 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4317 })
4318 } else {
4319 format!(
4320 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4321 status.state, status.enabled, status.live
4322 )
4323 };
4324 control_error_frame(frame, code, message)
4325 }
4326
4327 async fn handle_route_open(
4328 &self,
4329 ctx: &RouteCtx,
4330 frame: Frame,
4331 request: RouteOpenRequest,
4332 timings: &mut RouteOpenTimings,
4333 ) -> Result<Vec<Frame>, RouterError> {
4334 let RouteOpenRequest {
4335 target,
4336 mut identity,
4337 consumer_identity,
4338 consumer_capabilities,
4339 role_versions,
4340 admission_facts,
4341 scope,
4342 } = request;
4343 let target_module_id = target_module_id(&target).to_string();
4344 timings.module_id.clone_from(&target_module_id);
4345 if self
4346 .registry
4347 .get_module_by_connection(ctx.connection_id)
4348 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4349 .is_some()
4350 {
4351 return Ok(vec![control_error_frame(
4352 &frame,
4353 "invalid_request",
4354 "module connections cannot open client routes",
4355 )?]);
4356 }
4357 debug!(
4358 connection_id = ctx.connection_id.get(),
4359 corr = frame.header.corr,
4360 module_id = %target_module_id,
4361 "handling route.open"
4362 );
4363
4364 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4369 let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4370
4371 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4377 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4378 self.observe_route_open_refusal(
4379 ctx,
4380 &caller,
4381 &target_module_id,
4382 "invalid_role_versions",
4383 error_codes::INVALID_REQUEST,
4384 );
4385 return Ok(vec![control_error_body_frame(
4386 &frame,
4387 ErrorBody {
4388 code: error_codes::INVALID_REQUEST.to_string(),
4389 message: error.to_string(),
4390 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4391 },
4392 )?]);
4393 }
4394
4395 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4396 if status.waiting_for_previous_pid.is_some() {
4397 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4398 ctx,
4399 &caller,
4400 &frame,
4401 &target_module_id,
4402 "module_warming",
4403 &status,
4404 )?]);
4405 }
4406 }
4407
4408 let Some(registration) = self
4427 .registry
4428 .get_module(&target_module_id)
4429 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4430 else {
4431 if let Some((status, warming)) =
4432 self.supervisor_status(&target_module_id, frame.header.corr)?
4433 {
4434 if status.protocol == ModuleProtocol::None {
4441 return Ok(vec![self.route_open_refusal_frame(
4442 ctx,
4443 &caller,
4444 &frame,
4445 &target_module_id,
4446 "protocol_none",
4447 error_codes::MODULE_NO_PROTOCOL,
4448 format!(
4449 "module_id '{target_module_id}' is declared protocol: none; \
4450 it speaks no subc wire and serves no routes"
4451 ),
4452 )?]);
4453 }
4454 let code = if warming {
4455 "module_warming"
4456 } else {
4457 "target_unavailable"
4458 };
4459 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4460 ctx,
4461 &caller,
4462 &frame,
4463 &target_module_id,
4464 code,
4465 &status,
4466 )?]);
4467 }
4468 if let Some(removed_ago_ms) =
4469 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4470 {
4471 return Ok(vec![self.route_open_refusal_frame(
4472 ctx,
4473 &caller,
4474 &frame,
4475 &target_module_id,
4476 "removed",
4477 error_codes::MODULE_REMOVED,
4478 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4479 )?]);
4480 }
4481 return Ok(vec![self.route_open_refusal_frame(
4482 ctx,
4483 &caller,
4484 &frame,
4485 &target_module_id,
4486 "not_registered",
4487 error_codes::UNKNOWN_MODULE,
4488 format!("module_id '{target_module_id}' is not registered"),
4489 )?]);
4490 };
4491
4492 if !registration.ready {
4496 self.counters
4497 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4498 info!(
4499 target: "control",
4500 code = error_codes::MODULE_WARMING,
4501 module_id = ?target_module_id,
4502 connection_id = ctx.connection_id.get(),
4503 principal = %caller.principal,
4504 scope = %caller.scope,
4505 harness = %caller.harness,
4506 reason = "declared_not_ready",
4507 "route.open refused"
4508 );
4509 self.route_outages
4512 .record_not_serving(&target_module_id, "declared_not_ready");
4513 return Ok(vec![control_error_body_frame(
4514 &frame,
4515 ErrorBody {
4516 code: error_codes::MODULE_WARMING.to_string(),
4517 message: format!(
4518 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4519 ),
4520 detail: Some(serde_json::json!({
4521 "reason": "declared_not_ready"
4522 })),
4523 },
4524 )?]);
4525 }
4526
4527 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4546 self.counters
4547 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4548 info!(
4549 target: "control",
4550 code = error_codes::MODULE_WARMING,
4551 module_id = ?target_module_id,
4552 connection_id = ctx.connection_id.get(),
4553 principal = %caller.principal,
4554 scope = %caller.scope,
4555 harness = %caller.harness,
4556 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4557 capability = %capability,
4558 "route.open refused"
4559 );
4560 return Ok(vec![control_error_body_frame(
4561 &frame,
4562 ErrorBody {
4563 code: error_codes::MODULE_WARMING.to_string(),
4564 message: format!(
4565 "module_id '{target_module_id}' requires capability '{capability}', \
4566 which no registered module provides; retry"
4567 ),
4568 detail: Some(serde_json::json!({
4569 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4570 "capability": capability,
4571 })),
4572 },
4573 )?]);
4574 }
4575
4576 if !target_has_required_role(&target, ®istration.manifest.provides) {
4577 return Ok(vec![self.route_open_refusal_frame(
4578 ctx,
4579 &caller,
4580 &frame,
4581 &target_module_id,
4582 "role_not_provided",
4583 "target_unavailable",
4584 format!("module_id '{target_module_id}' does not provide the requested target"),
4585 )?]);
4586 }
4587
4588 if registration.state != ChannelState::Active {
4589 return Ok(vec![self.route_open_refusal_frame(
4590 ctx,
4591 &caller,
4592 &frame,
4593 &target_module_id,
4594 "registration_not_active",
4595 "target_unavailable",
4596 format!("module_id '{target_module_id}' is not active"),
4597 )?]);
4598 }
4599
4600 if self
4601 .forwarding
4602 .module_is_draining(&target_module_id)
4603 .map_err(RouterError::Forwarding)?
4604 {
4605 return Ok(vec![self.route_open_refusal_frame(
4606 ctx,
4607 &caller,
4608 &frame,
4609 &target_module_id,
4610 "reloading",
4611 "module_reloading",
4612 format!("module_id '{target_module_id}' is reloading"),
4613 )?]);
4614 }
4615
4616 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4617 process_liveness.process_live(&target_module_id) == Some(false)
4618 }) {
4619 if process_liveness.process_replacing(&target_module_id) {
4626 return Ok(vec![self.route_open_refusal_frame(
4627 ctx,
4628 &caller,
4629 &frame,
4630 &target_module_id,
4631 "reloading",
4632 "module_reloading",
4633 format!("module_id '{target_module_id}' is reloading"),
4634 )?]);
4635 }
4636 return Ok(vec![self.route_open_refusal_frame(
4637 ctx,
4638 &caller,
4639 &frame,
4640 &target_module_id,
4641 "supervisor_not_live",
4642 "target_unavailable",
4643 format!("module_id '{target_module_id}' is not live"),
4644 )?]);
4645 }
4646
4647 if !self
4648 .forwarding
4649 .has_live_module_connection(&target_module_id)
4650 .map_err(RouterError::Forwarding)?
4651 {
4652 return Ok(vec![self.route_open_refusal_frame(
4653 ctx,
4654 &caller,
4655 &frame,
4656 &target_module_id,
4657 "no_forwarding_connection",
4658 "target_unavailable",
4659 format!("module_id '{target_module_id}' has no live forwarding connection"),
4660 )?]);
4661 }
4662
4663 if let Some(error) =
4664 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4665 {
4666 self.observe_route_open_refusal(
4667 ctx,
4668 &caller,
4669 &target_module_id,
4670 "op_not_allowed",
4671 "op_not_allowed",
4672 );
4673 return Ok(vec![error]);
4674 }
4675
4676 let principal =
4677 match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
4678 Ok(principal) => principal,
4679 Err(error) => {
4680 self.observe_route_open_refusal(
4681 ctx,
4682 &caller,
4683 &target_module_id,
4684 "bad_consumer_identity",
4685 "bad_consumer_identity",
4686 );
4687 return Ok(vec![error]);
4688 }
4689 };
4690 timings.principal = principal_label(&principal);
4691
4692 if let Principal::Reserved {
4696 module_id: opening_module_id,
4697 } = &principal
4698 {
4699 if let Some(opening_registration) = self
4700 .registry
4701 .get_module(opening_module_id)
4702 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4703 {
4704 if let Some(capability) =
4705 denied_capability(&opening_registration.manifest, ®istration.manifest)
4706 {
4707 warn!(
4708 opening_module_id,
4709 target_module_id,
4710 capability,
4711 "refusing route.open because an attested capability deny edge matches"
4712 );
4713 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4714 &frame,
4715 &target_module_id,
4716 "capability_deny_edge",
4717 "capability_forbidden",
4718 format!(
4719 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4720 ),
4721 )?]);
4722 }
4723 }
4724 }
4725
4726 if admission_facts.is_some() {
4727 let carrier_matches = matches!(
4728 &principal,
4729 Principal::Reserved { module_id }
4730 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4731 );
4732 if !carrier_matches {
4733 return Ok(vec![self.route_open_refusal_frame(
4734 ctx,
4735 &caller,
4736 &frame,
4737 &target_module_id,
4738 "admission_facts_carrier_not_permitted",
4739 "admission_facts_not_permitted",
4740 "admission facts may only be carried by the configured reserved module",
4741 )?]);
4742 }
4743
4744 let target_allowed = self
4745 .admission_facts_targets
4746 .as_ref()
4747 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4748 if !target_allowed {
4749 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4750 &frame,
4751 &target_module_id,
4752 "admission_facts_target_not_listed",
4753 "admission_facts_target_not_allowed",
4754 format!(
4755 "admission facts are not permitted for target module_id '{target_module_id}'"
4756 ),
4757 )?]);
4758 }
4759
4760 }
4764
4765 let (bound_scope, mut scope_stamp) = match &scope {
4771 None => (None, None),
4772 Some(selector) => {
4773 let owner_configured = match &selector.owner {
4774 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4780 _ => false,
4781 };
4782 let mut scopes = self
4783 .scopes
4784 .write()
4785 .unwrap_or_else(|poisoned| poisoned.into_inner());
4786 let admitted = scopes
4787 .admit(&principal, &target_module_id, selector, owner_configured)
4788 .and_then(|admission| {
4789 crate::scopes::check_target_flow_support(
4790 &admission.stamp,
4791 &target_module_id,
4792 registration.manifest.capabilities.as_ref(),
4793 )?;
4794 crate::scopes::check_target_agent_run_support(
4795 &admission.stamp,
4796 &target_module_id,
4797 registration.manifest.capabilities.as_ref(),
4798 )?;
4799 let bound = scopes.reserve_bind(
4800 &admission,
4801 &principal,
4802 ctx.connection_id,
4803 selector.hold_presence,
4804 );
4805 Ok((admission, bound))
4806 });
4807 match admitted {
4808 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4809 Err(refusal) => {
4810 return Ok(vec![self.route_open_refusal_frame(
4811 ctx,
4812 &caller,
4813 &frame,
4814 &target_module_id,
4815 refusal.code,
4816 refusal.code,
4817 refusal.message,
4818 )?]);
4819 }
4820 }
4821 }
4822 };
4823 if let Some(stamp) = &scope_stamp {
4826 caller.scope = route_open_scope_label(stamp);
4827 }
4828
4829 timings.enter(RouteOpenPhase::ProjectRoot);
4851 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4852 Ok(project_root) => project_root,
4853 Err(err) => {
4854 return Ok(vec![control_error_frame(
4855 &frame,
4856 "invalid_project_root",
4857 err.to_string(),
4858 )?])
4859 }
4860 };
4861 timings.enter(RouteOpenPhase::Admission);
4862 identity.project_root = project_root.as_path().to_path_buf();
4863
4864 let _concurrency_guard = match self
4875 .route_bind_concurrency
4876 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4877 {
4878 Ok(guard) => guard,
4879 Err(in_flight) => {
4880 return Ok(vec![self.route_open_target_capacity_refusal(
4881 ctx,
4882 &caller,
4883 &frame,
4884 &target_module_id,
4885 in_flight,
4886 )?]);
4887 }
4888 };
4889
4890 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4893 RouteBindAdmission::Admitted { guard, probe } => {
4894 if probe {
4895 info!(
4896 module_id = %target_module_id,
4897 connection_id = ctx.connection_id.get(),
4898 "route.bind breaker half-open: admitting one probe"
4899 );
4900 }
4901 guard
4902 }
4903 RouteBindAdmission::Refused {
4904 consecutive_timeouts,
4905 retry_in,
4906 probe_in_flight,
4907 } => {
4908 return Ok(vec![self.route_open_breaker_refusal_frame(
4909 ctx,
4910 &caller,
4911 &frame,
4912 &target_module_id,
4913 consecutive_timeouts,
4914 retry_in,
4915 probe_in_flight,
4916 )?]);
4917 }
4918 };
4919
4920 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4925 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4926 timings.enter(RouteOpenPhase::EgressReservation);
4927 let pending = match self
4928 .forwarding
4929 .begin_route_bind_relay_for(
4930 ctx.connection_id,
4931 ctx.egress.clone(),
4932 response_version(&frame),
4933 frame.header.corr,
4934 &target_module_id,
4935 principal.clone(),
4936 bound_scope,
4937 Some(project_root),
4938 relay_deadline,
4939 )
4940 .await
4941 {
4942 Ok(pending) => pending,
4943 Err(err) => {
4944 return Ok(vec![self.route_open_refusal_frame(
4945 ctx,
4946 &caller,
4947 &frame,
4948 &target_module_id,
4949 "relay_reservation_failed",
4950 forwarding_error_code(&err),
4951 err.to_string(),
4952 )?])
4953 }
4954 };
4955 timings.enter(RouteOpenPhase::Admission);
4956 let crate::forwarding::PendingRouteBindRelay {
4957 endpoint,
4958 module_sink,
4959 negotiated_ver,
4960 client_channel,
4961 client_epoch,
4962 module_channel,
4963 module_epoch,
4964 corr: relay_corr,
4965 receiver,
4966 } = pending;
4967 let mut reservation =
4968 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4969
4970 let relay_registration = self
4975 .registry
4976 .get_module_by_connection(endpoint.connection_id)
4977 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4978 let relay_capabilities = relay_registration
4979 .as_ref()
4980 .and_then(|registration| registration.manifest.capabilities.as_ref());
4981 if let Some(selector) = &scope {
4982 if let Err(refusal) = self
4985 .scopes
4986 .read()
4987 .unwrap_or_else(|poisoned| poisoned.into_inner())
4988 .check_bind_proof(&principal, selector)
4989 {
4990 reservation.release_and_disarm();
4991 return Ok(vec![self.route_open_refusal_frame(
4992 ctx,
4993 &caller,
4994 &frame,
4995 &target_module_id,
4996 refusal.code,
4997 refusal.code,
4998 refusal.message,
4999 )?]);
5000 }
5001 }
5002 if let Some(stamp) = scope_stamp
5003 .as_ref()
5004 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5005 {
5006 if let Err(refusal) = crate::scopes::check_target_flow_support(
5007 stamp,
5008 &target_module_id,
5009 relay_capabilities,
5010 )
5011 .and_then(|()| {
5012 crate::scopes::check_target_agent_run_support(
5013 stamp,
5014 &target_module_id,
5015 relay_capabilities,
5016 )
5017 }) {
5018 reservation.release_and_disarm();
5019 return Ok(vec![self.route_open_refusal_frame(
5020 ctx,
5021 &caller,
5022 &frame,
5023 &target_module_id,
5024 refusal.code,
5025 refusal.code,
5026 refusal.message,
5027 )?]);
5028 }
5029 }
5030
5031 if !relay_capabilities.is_some_and(|caps| {
5034 caps.provides
5035 .iter()
5036 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5037 }) {
5038 if let Some(stamp) = scope_stamp.as_mut() {
5039 stamp.workdir = None;
5040 stamp.project_id = None;
5041 }
5042 }
5043 if !relay_capabilities.is_some_and(|caps| {
5044 caps.provides
5045 .iter()
5046 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5047 }) {
5048 if let Some(stamp) = scope_stamp.as_mut() {
5049 stamp.attributes.on_behalf_of_agent = None;
5050 }
5051 }
5052 let call_mode = if relay_capabilities
5053 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5054 {
5055 Some(derive_call_mode(scope_stamp.as_ref()))
5056 } else {
5057 if let Some(stamp) = scope_stamp.as_mut() {
5058 stamp.expires_at_ms = None;
5059 }
5060 None
5061 };
5062
5063 debug!(
5064 connection_id = ctx.connection_id.get(),
5065 client_channel,
5066 client_epoch,
5067 module_channel,
5068 module_epoch,
5069 "reserved route handle pair"
5070 );
5071 let route_principal_label = principal_label(&principal);
5074 let relay = ModuleControlRequest::RouteBind {
5075 route_channel: module_channel,
5076 epoch: module_epoch,
5077 target,
5078 identity,
5079 principal: Some(principal),
5080 consumer_capabilities,
5081 role_versions,
5082 admission_facts,
5083 scope: scope_stamp,
5084 call_mode,
5085 };
5086 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
5087 RouterError::backend(
5088 0,
5089 frame.header.corr,
5090 format!("failed to encode route.bind request: {err}"),
5091 )
5092 })?;
5093 let relay_frame = Frame::build_with_version(
5094 negotiated_ver,
5095 FrameType::Request,
5096 control_flags(),
5097 0,
5098 0,
5099 relay_corr,
5100 relay_body,
5101 )
5102 .map_err(RouterError::FrameBuild)?;
5103
5104 timings.enter(RouteOpenPhase::BindSend);
5105 let sent = module_sink.send(relay_frame).await;
5106 timings.enter(RouteOpenPhase::Admission);
5107 if let Err(err) = sent {
5108 reservation.release_and_disarm();
5109 return Ok(vec![self.route_open_refusal_frame(
5110 ctx,
5111 &caller,
5112 &frame,
5113 &target_module_id,
5114 "relay_send_failed",
5115 "target_unavailable",
5116 err.to_string(),
5117 )?]);
5118 }
5119
5120 #[cfg(test)]
5121 if let Some(hold) = &self.route_bind_relay_sent_hold {
5122 hold.wait().await;
5126 hold.wait().await;
5127 }
5128
5129 if self
5134 .forwarding
5135 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5136 .map_err(RouterError::Forwarding)?
5137 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5138 {
5139 self.send_abandoned_route_bind_goodbye(
5140 &module_sink,
5141 negotiated_ver,
5142 module_channel,
5143 module_epoch,
5144 );
5145 }
5146
5147 timings.enter(RouteOpenPhase::BindAckWait);
5148 let outcome = timeout_at(relay_deadline, receiver).await;
5149 timings.enter(RouteOpenPhase::Admission);
5150 match outcome {
5151 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5152 reservation.disarm();
5153 if breaker.record_accepted() {
5154 info!(
5155 module_id = %target_module_id,
5156 "route.bind breaker closed: the probe was accepted"
5157 );
5158 }
5159 self.observe_route_open_accept(
5160 ctx,
5161 &target_module_id,
5162 &route_principal_label,
5163 &caller.scope,
5164 );
5165 Ok(Vec::new())
5166 }
5167 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5168 reservation.release_and_disarm();
5169 breaker.record_inconclusive();
5173 let scope_code = match body.code.as_str() {
5177 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
5178 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
5179 _ => None,
5180 };
5181 if let Some(code) = scope_code {
5182 self.observe_route_open_refusal(
5183 ctx,
5184 &caller,
5185 &target_module_id,
5186 "scope_changed_before_commit",
5187 code,
5188 );
5189 return Ok(vec![control_error_body_frame(&frame, body)?]);
5190 }
5191 self.counters
5192 .increment_route_open_refused("module_rejected");
5193 info!(
5194 target: "control",
5195 code = "module_rejected",
5196 module_code = ?body.code,
5197 module_id = ?target_module_id,
5198 connection_id = ctx.connection_id.get(),
5199 principal = %caller.principal,
5200 scope = %caller.scope,
5201 harness = %caller.harness,
5202 "route.open refused"
5203 );
5204 Ok(vec![control_error_body_frame(&frame, body)?])
5205 }
5206 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5207 reservation.release_and_disarm();
5208 breaker.record_inconclusive();
5209 tracing::warn!(
5216 module_id = %target_module_id,
5217 "route.bind relay abandoned: {message}"
5218 );
5219 Ok(vec![self.route_open_refusal_frame(
5220 ctx,
5221 &caller,
5222 &frame,
5223 &target_module_id,
5224 "relay_abandoned",
5225 "target_unavailable",
5226 message,
5227 )?])
5228 }
5229 Ok(Err(_)) => {
5230 reservation.release_and_disarm();
5231 breaker.record_inconclusive();
5232 Ok(vec![self.route_open_refusal_frame(
5233 ctx,
5234 &caller,
5235 &frame,
5236 &target_module_id,
5237 "relay_waiter_canceled",
5238 "target_unavailable",
5239 "route.bind relay waiter was canceled before the module responded",
5240 )?])
5241 }
5242 Err(_) => {
5243 reservation.release_and_disarm();
5244 if let Some(opened) = breaker.record_timeout(
5248 self.route_bind_breaker_threshold,
5249 self.route_bind_breaker_cooldown,
5250 ) {
5251 warn!(
5252 module_id = %target_module_id,
5253 consecutive_timeouts = opened.consecutive_timeouts,
5254 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5255 reopened_after_probe = opened.reopened_after_probe,
5256 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5257 );
5258 }
5259 tracing::warn!(
5267 module_id = %target_module_id,
5268 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5269 "route.bind relay timed out: module did not ack within budget"
5270 );
5271 Ok(vec![self.route_open_refusal_frame(
5272 ctx,
5273 &caller,
5274 &frame,
5275 &target_module_id,
5276 "relay_timed_out",
5277 "module_timeout",
5278 format!(
5279 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5280 route_bind_relay_timeout
5281 ),
5282 )?])
5283 }
5284 }
5285 }
5286
5287 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5288 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5289 snapshot: self.supervisor.spawn_snapshot(),
5290 };
5291 Ok(vec![control_response_body_frame(
5292 &frame,
5293 &response,
5294 "ClientControlResponse::SupervisorSpawnSnapshot",
5295 )?])
5296 }
5297
5298 fn handle_supervisor_spawn_subscribe(
5299 &self,
5300 ctx: &RouteCtx,
5301 frame: Frame,
5302 since: Option<SpawnCursor>,
5303 ) -> Result<Vec<Frame>, RouterError> {
5304 match self.supervisor.subscribe_spawns(
5305 ctx.connection_id,
5306 frame.header.corr,
5307 response_version(&frame),
5308 since,
5309 ctx.egress.clone(),
5310 ) {
5311 Ok(()) => Ok(Vec::new()),
5312 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5313 Ok(vec![control_error_body_frame(
5314 &frame,
5315 ErrorBody {
5316 code: "spawn_cursor_incarnation_mismatch".to_string(),
5317 message: "spawn cursor belongs to a different daemon incarnation"
5318 .to_string(),
5319 detail: Some(serde_json::json!({
5320 "current_daemon_incarnation": current
5321 })),
5322 },
5323 )?])
5324 }
5325 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5326 &frame,
5327 ErrorBody {
5328 code: "spawn_cursor_too_old".to_string(),
5329 message: "spawn cursor predates the retained event ring".to_string(),
5330 detail: Some(serde_json::json!({
5331 "oldest_retained_cursor": oldest
5332 })),
5333 },
5334 )?]),
5335 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5336 0,
5337 frame.header.corr,
5338 format!("failed to open supervisor spawn subscription: {error}"),
5339 )),
5340 }
5341 }
5342
5343 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5344 let generation = self
5345 .registry
5346 .generation()
5347 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5348 let mut modules = Vec::new();
5349 for module in self.supervisor.list() {
5350 let status = module.status_for_control("list").map_err(|err| {
5351 RouterError::backend(
5352 0,
5353 frame.header.corr,
5354 format!("failed to read supervisor status: {err}"),
5355 )
5356 })?;
5357 let (configured, _) = module.configuration().map_err(|err| {
5358 RouterError::backend(
5359 0,
5360 frame.header.corr,
5361 format!("failed to read module configuration: {err}"),
5362 )
5363 })?;
5364 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
5365 RouterError::backend(
5366 0,
5367 frame.header.corr,
5368 format!("failed to read retry time: {err}"),
5369 )
5370 })?;
5371 let image = module.running_image_agreement().await;
5373 let resources = Some(module.child_resource_usage());
5376 let mut verdict =
5377 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5378 verdict.spec = Some(launch_spec_agreement(
5379 &configured,
5380 status.process_alive,
5381 status.launch_spec_digest.as_ref(),
5382 ));
5383 let pending_reload = Some(verdict);
5384 let configured_enabled = self
5385 .capability_evaluator
5386 .configured_enabled(&status.module_id);
5387 modules.push(
5390 SupervisorEntry::new(
5391 status.module_id,
5392 status.state.to_string(),
5393 status.enabled,
5394 status.live,
5395 status.health.status,
5396 )
5397 .with_configured_enabled(configured_enabled)
5398 .with_next_retry_at_ms(next_retry_at_ms)
5399 .with_launch_nonce_env(Some(!cfg!(unix)))
5400 .with_launch_nonce_source(status.launch_nonce_source)
5401 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5402 .with_protocol(status.protocol)
5403 .with_pending_reload(pending_reload)
5404 .with_last_probe_ms(status.health.last_probe_ms)
5405 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5406 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5407 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5408 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5409 .with_restart_count(Some(status.restart_count))
5410 .with_max_restarts(Some(status.max_restarts))
5411 .with_lifetime_restarts(Some(status.lifetime_restarts))
5412 .with_spawn_generation(Some(status.spawn_generation))
5413 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5414 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5415 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5416 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5417 .with_resources(resources),
5418 );
5419 }
5420 let response = ClientControlResponse::SupervisorList {
5421 generation,
5422 modules,
5423 };
5424 Ok(vec![control_response_body_frame(
5425 &frame,
5426 &response,
5427 "ClientControlResponse::SupervisorList",
5428 )?])
5429 }
5430
5431 fn handle_supervisor_stderr_tail(
5432 &self,
5433 frame: Frame,
5434 module_id: String,
5435 max_lines: Option<u32>,
5436 max_bytes: Option<u32>,
5437 ) -> Result<Vec<Frame>, RouterError> {
5438 let Some(module) = self.supervisor.get(&module_id) else {
5439 return Ok(vec![control_error_frame(
5440 &frame,
5441 "unknown_module",
5442 format!("module_id '{module_id}' is not supervised"),
5443 )?]);
5444 };
5445
5446 let snapshot = module.stderr_tail(
5447 max_lines.map(|value| value as usize),
5448 max_bytes.map(|value| value as usize),
5449 );
5450
5451 let response = ClientControlResponse::SupervisorStderrTail {
5452 module_id,
5453 tail: StderrTail {
5454 capture: match snapshot.capture {
5455 CaptureState::Captured => StderrCaptureState::Captured,
5456 CaptureState::Incomplete { reason } => {
5457 StderrCaptureState::Incomplete { reason }
5458 }
5459 CaptureState::NotCaptured { reason } => {
5460 StderrCaptureState::NotCaptured { reason }
5461 }
5462 },
5463 entries: snapshot
5464 .entries
5465 .into_iter()
5466 .map(|entry| match entry {
5467 TailEntry::Line {
5468 text,
5469 truncated,
5470 at_ms,
5471 } => StderrTailEntry::Line {
5472 text,
5473 truncated,
5474 at_ms,
5475 },
5476 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5477 })
5478 .collect(),
5479 dropped_lines: snapshot.dropped_lines,
5480 },
5481 };
5482 Ok(vec![control_response_body_frame(
5483 &frame,
5484 &response,
5485 "ClientControlResponse::SupervisorStderrTail",
5486 )?])
5487 }
5488
5489 async fn handle_supervisor_terminals(
5490 &self,
5491 frame: Frame,
5492 module_id: String,
5493 ) -> Result<Vec<Frame>, RouterError> {
5494 let Some(module) = self.supervisor.get(&module_id) else {
5495 return Ok(vec![control_error_frame(
5496 &frame,
5497 "unknown_module",
5498 format!("module_id '{module_id}' is not supervised"),
5499 )?]);
5500 };
5501
5502 let terminals = module
5505 .read_durable_terminal_history()
5506 .await
5507 .map_err(|error| {
5508 RouterError::backend(
5509 0,
5510 frame.header.corr,
5511 format!("failed to read terminal history: {error}"),
5512 )
5513 })?;
5514 let response = ClientControlResponse::SupervisorTerminals {
5515 module_id,
5516 terminals,
5517 };
5518 Ok(vec![control_response_body_frame(
5519 &frame,
5520 &response,
5521 "ClientControlResponse::SupervisorTerminals",
5522 )?])
5523 }
5524
5525 fn handle_supervisor_routes(
5526 &self,
5527 frame: Frame,
5528 module_id: Option<String>,
5529 ) -> Result<Vec<Frame>, RouterError> {
5530 let modules = self
5531 .forwarding
5532 .route_census(module_id.as_deref())
5533 .map_err(RouterError::Forwarding)?
5534 .into_iter()
5535 .map(|(module_id, routes)| SupervisorRouteModule {
5536 module_id,
5537 routes: routes
5538 .into_iter()
5539 .map(|route| SupervisorRoute {
5540 consumer: match route.principal {
5541 Principal::Reserved { module_id } => {
5542 SupervisorRouteConsumer::Reserved { module_id }
5543 }
5544 Principal::Direct | Principal::Unverified => {
5545 SupervisorRouteConsumer::Direct {
5546 connection_id: route.goodbye_target.connection_id.get(),
5547 }
5548 }
5549 },
5550 age_ms: Instant::now()
5551 .saturating_duration_since(route.bound_at)
5552 .as_millis()
5553 .try_into()
5554 .unwrap_or(u64::MAX),
5555 draining: route.draining,
5556 drain_reason: route.drain_reason,
5557 })
5558 .collect(),
5559 })
5560 .collect();
5561 let response = ClientControlResponse::SupervisorRoutes { modules };
5562 Ok(vec![control_response_body_frame(
5563 &frame,
5564 &response,
5565 "ClientControlResponse::SupervisorRoutes",
5566 )?])
5567 }
5568
5569 async fn handle_supervisor_provenance(
5570 &self,
5571 frame: Frame,
5572 module_id: Option<String>,
5573 ) -> Result<Vec<Frame>, RouterError> {
5574 let mut selected = if let Some(module_id) = module_id {
5575 let Some(module) = self.supervisor.get(&module_id) else {
5576 return Ok(vec![control_error_frame(
5577 &frame,
5578 "unknown_module",
5579 format!("module_id '{module_id}' is not supervised"),
5580 )?]);
5581 };
5582 vec![module]
5583 } else {
5584 self.supervisor.list()
5585 };
5586
5587 let mut modules = Vec::with_capacity(selected.len());
5588 for module in selected.drain(..) {
5589 let (status, observed_image) = module
5590 .status_and_running_image_agreement()
5591 .await
5592 .map_err(|err| {
5593 RouterError::backend(
5594 0,
5595 frame.header.corr,
5596 format!("failed to read supervisor status: {err}"),
5597 )
5598 })?;
5599 let module_declared = self
5600 .registry
5601 .get_module(&status.module_id)
5602 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5603 .and_then(|registration| registration.manifest.provenance)
5604 .map(|build| ModuleDeclaredProvenance::Reported { build })
5605 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5606 #[cfg(test)]
5607 let running_image = match &self.provenance_probe_override {
5608 Some(result) => result.clone(),
5609 None => observed_image,
5610 };
5611 #[cfg(not(test))]
5612 let running_image = observed_image;
5613 modules.push(SupervisorModuleProvenance {
5614 module_id: status.module_id,
5615 module_declared,
5616 daemon_observed: SupervisorObservedProcess {
5617 launch_nonce_source: status.launch_nonce_source,
5618 pid: status.pid,
5619 spawned_at_ms: status.spawned_at_ms,
5620 spawned_from: status.spawned_from,
5621 running_image,
5622 },
5623 });
5624 }
5625 let daemon = SupervisorDaemonProvenance {
5626 daemon_build: self.daemon_provenance.build.clone(),
5627 daemon_observed: DaemonObservedProcess {
5628 pid: self.daemon_provenance.pid,
5629 started_at_ms: self
5630 .daemon_provenance
5631 .start_clock
5632 .map(|clock| clock.started_at_ms())
5633 .or(self.daemon_provenance.started_at_ms),
5634 running_image: self
5635 .daemon_provenance
5636 .probe
5637 .observe(
5638 self.daemon_provenance.pid,
5639 self.daemon_provenance.executable_path.as_deref(),
5640 self.daemon_provenance.executable_identity.clone(),
5641 self.daemon_provenance.process_start_time,
5642 )
5643 .await,
5644 },
5645 };
5646 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5647 Ok(vec![control_response_body_frame(
5648 &frame,
5649 &response,
5650 "ClientControlResponse::SupervisorProvenance",
5651 )?])
5652 }
5653
5654 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5655 self.refresh_capability_requirements();
5656 let generation = self
5657 .registry
5658 .generation()
5659 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5660 let modules = self
5661 .supervisor
5662 .list()
5663 .into_iter()
5664 .map(|module| {
5665 let status = module.status_for_control("health").map_err(|err| {
5666 RouterError::backend(
5667 0,
5668 frame.header.corr,
5669 format!("failed to read supervisor health: {err}"),
5670 )
5671 })?;
5672 let module_id = status.module_id;
5673 let capability_detail = self
5674 .capability_evaluator
5675 .required_problem_detail(&module_id);
5676 Ok(SupervisorHealthEntry {
5677 module_id,
5678 status: status.health.status,
5679 detail: append_capability_problem_detail(
5680 status.health.detail,
5681 capability_detail,
5682 ),
5683 metrics: status.health.metrics,
5684 consecutive_failures: status.health.consecutive_failures,
5685 late_answer_count: status.health.late_answer_count,
5686 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5687 last_action: status.health.last_action,
5688 last_action_ms: status.health.last_action_ms,
5689 last_probe_ms: status.health.last_probe_ms,
5690 })
5691 })
5692 .collect::<Result<Vec<_>, RouterError>>()?;
5693 let response = ClientControlResponse::SupervisorHealth {
5694 generation,
5695 modules,
5696 };
5697 Ok(vec![control_response_body_frame(
5698 &frame,
5699 &response,
5700 "ClientControlResponse::SupervisorHealth",
5701 )?])
5702 }
5703
5704 async fn handle_supervisor_restart(
5705 &self,
5706 frame: Frame,
5707 module_id: String,
5708 drain_timeout_ms: Option<u64>,
5709 requester: Principal,
5710 ) -> Result<Vec<Frame>, RouterError> {
5711 let operation_lock = self.supervisor.operation_lock();
5712 let _operation_guard = operation_lock.lock().await;
5713 let Some(module) = self.supervisor.get(&module_id) else {
5714 return Ok(vec![control_error_frame(
5715 &frame,
5716 "unknown_module",
5717 format!("module_id '{module_id}' is not supervised"),
5718 )?]);
5719 };
5720
5721 self.route_outages.mark_operator_action(&module_id);
5722 if let Err(err) = module
5723 .restart_requested(drain_timeout_ms, Some(requester))
5724 .await
5725 {
5726 self.route_outages
5727 .operator_action_ended_unrefused(&module_id);
5728 let (code, message) = match err {
5729 crate::supervise::SuperviseError::Disabled { .. } => {
5730 ("module_disabled", err.to_string())
5731 }
5732 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5733 ("swap_in_progress", err.to_string())
5734 }
5735 _ => (
5736 "target_unavailable",
5737 format!("failed to restart module_id '{module_id}': {err}"),
5738 ),
5739 };
5740 return Ok(vec![control_error_frame(&frame, code, message)?]);
5741 }
5742
5743 let response = ClientControlResponse::SupervisorAck {
5744 module_id,
5745 applied: true,
5746 };
5747 Ok(vec![control_response_body_frame(
5748 &frame,
5749 &response,
5750 "ClientControlResponse::SupervisorAck",
5751 )?])
5752 }
5753
5754 async fn handle_supervisor_swap(
5758 &self,
5759 frame: Frame,
5760 module_id: String,
5761 ready_timeout_ms: Option<u64>,
5762 requester: Principal,
5763 ) -> Result<Vec<Frame>, RouterError> {
5764 let module = {
5771 let operation_lock = self.supervisor.operation_lock();
5772 let _operation_guard = operation_lock.lock().await;
5773 self.supervisor.get(&module_id)
5774 };
5775 let Some(module) = module else {
5776 return Ok(vec![control_error_frame(
5777 &frame,
5778 "unknown_module",
5779 format!("module_id '{module_id}' is not supervised"),
5780 )?]);
5781 };
5782
5783 self.route_outages.mark_operator_action(&module_id);
5784 if let Err(err) = module
5785 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5786 .await
5787 {
5788 self.route_outages
5789 .operator_action_ended_unrefused(&module_id);
5790 use crate::supervise::SuperviseError;
5791 let message = err.to_string();
5792 let error = match err {
5793 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5794 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5795 code: "swap_refused".to_string(),
5796 message,
5797 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5798 },
5799 SuperviseError::SwapFailed {
5800 arm,
5801 candidate_exit,
5802 ..
5803 } => ErrorBody {
5804 code: "swap_failed".to_string(),
5805 message,
5806 detail: Some(serde_json::json!({
5807 "arm": arm.as_str(),
5808 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5809 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5810 })),
5811 },
5812 _ => ErrorBody::new(
5813 "target_unavailable",
5814 format!("failed to swap module_id '{module_id}': {message}"),
5815 ),
5816 };
5817 return Ok(vec![control_error_body_frame(&frame, error)?]);
5818 }
5819 self.route_outages
5823 .operator_action_ended_unrefused(&module_id);
5824
5825 let response = ClientControlResponse::SupervisorAck {
5826 module_id,
5827 applied: true,
5828 };
5829 Ok(vec![control_response_body_frame(
5830 &frame,
5831 &response,
5832 "ClientControlResponse::SupervisorAck",
5833 )?])
5834 }
5835
5836 async fn handle_supervisor_reload(
5837 &self,
5838 frame: Frame,
5839 module_id: String,
5840 requester: Principal,
5841 ) -> Result<Vec<Frame>, RouterError> {
5842 let operation_lock = self.supervisor.operation_lock();
5843 let _operation_guard = operation_lock.lock().await;
5844 let Some(module) = self.supervisor.get(&module_id) else {
5845 return Ok(vec![control_error_frame(
5846 &frame,
5847 "unknown_module",
5848 format!("module_id '{module_id}' is not supervised"),
5849 )?]);
5850 };
5851
5852 self.route_outages.mark_operator_action(&module_id);
5853 if let Err(err) = module.reload_requested(Some(requester)).await {
5854 self.route_outages
5855 .operator_action_ended_unrefused(&module_id);
5856 let (code, message) = match err {
5857 crate::supervise::SuperviseError::Disabled { .. } => {
5858 ("module_disabled", err.to_string())
5859 }
5860 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5861 ("swap_in_progress", err.to_string())
5862 }
5863 _ => (
5864 "reload_failed",
5865 format!("failed to reload module_id '{module_id}': {err}"),
5866 ),
5867 };
5868 return Ok(vec![control_error_frame(&frame, code, message)?]);
5869 }
5870
5871 let response = ClientControlResponse::SupervisorAck {
5872 module_id,
5873 applied: true,
5874 };
5875 Ok(vec![control_response_body_frame(
5876 &frame,
5877 &response,
5878 "ClientControlResponse::SupervisorAck",
5879 )?])
5880 }
5881
5882 async fn handle_supervisor_rescan(
5886 &self,
5887 frame: Frame,
5888 mode: RescanMode,
5889 requester: Principal,
5890 ) -> Result<Vec<Frame>, RouterError> {
5891 let preview = mode.previews();
5892 let Some(context) = self.rescan.clone() else {
5893 return Ok(vec![control_error_frame(
5894 &frame,
5895 "rescan_unavailable",
5896 "the daemon was not started with a reloadable config path".to_string(),
5897 )?]);
5898 };
5899
5900 let operation_lock = self.supervisor.operation_lock();
5901 let _operation_guard = operation_lock.lock().await;
5902 let loaded = match self.load_rescan_config(&mode, &context) {
5906 Ok(config) => config,
5907 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5908 };
5909 let Some((config, activity_config)) = loaded else {
5917 return Ok(vec![control_error_frame(
5918 &frame,
5919 "invalid_daemon_config",
5920 format!(
5921 "daemon config not found at {}; refusing to rescan (an absent config would \
5922 retire every supervised module)",
5923 context.config_path.display()
5924 ),
5925 )?]);
5926 };
5927 let (
5928 configured_port,
5929 storage_config,
5930 admission_facts_carrier_module_id,
5931 admission_facts_targets,
5932 scope_authority_owners,
5933 privileged_operator_prompt,
5934 modules,
5935 reserved_capabilities,
5936 ) = (
5937 config.port,
5938 config.storage,
5939 config.admission_facts_carrier_module_id,
5940 config.admission_facts_targets,
5941 config.scope_authority_owners,
5942 config.privileged_operator_prompt,
5943 config.modules,
5944 config.reserved_capabilities,
5945 );
5946
5947 let mut restart_required = Vec::new();
5956 for section in RestartRequiredSection::ALL {
5957 let changed = match section {
5958 RestartRequiredSection::Port => configured_port != context.configured_port,
5959 RestartRequiredSection::Storage => storage_config != context.storage_config,
5960 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5961 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5962 }
5963 RestartRequiredSection::AdmissionFactsTargets => {
5964 admission_facts_targets != context.admission_facts_targets
5965 }
5966 RestartRequiredSection::ScopeAuthorityOwners => {
5967 scope_authority_owners != context.scope_authority_owners
5968 }
5969 RestartRequiredSection::PrivilegedOperatorPrompt => {
5970 privileged_operator_prompt != context.privileged_operator_prompt
5971 }
5972 };
5973 if changed {
5974 restart_required.push(section.label().to_string());
5975 }
5976 }
5977 if !restart_required.is_empty() {
5978 warn!(
5979 config_path = %context.config_path.display(),
5980 sections = %restart_required.join(", "),
5981 "daemon config changed outside the modules section; restart the daemon to apply those changes"
5982 );
5983 }
5984
5985 for configured in &modules {
5986 if let Err(err) = validate_spec(&configured.module_spec()) {
5987 return Ok(vec![control_error_frame(
5988 &frame,
5989 "invalid_daemon_config",
5990 format!("supervisor rescan rejected daemon config: {err}"),
5991 )?]);
5992 }
5993 }
5994
5995 let configured_capabilities = modules
5996 .iter()
5997 .map(|module| (module.module_id.clone(), module.enabled))
5998 .collect::<Vec<_>>();
5999 let preview_capability_warnings = if preview {
6000 let (_, registrations) = self.runtime_capability_snapshot()?;
6001 let current_modules = self
6002 .supervisor
6003 .list()
6004 .into_iter()
6005 .map(|module| module.module_id().to_string())
6006 .collect::<BTreeSet<_>>();
6007 let resulting_modules = configured_capabilities.clone();
6008 let removed = current_modules
6009 .into_iter()
6010 .filter(|module_id| {
6011 !resulting_modules
6012 .iter()
6013 .any(|(configured_id, _)| configured_id == module_id)
6014 })
6015 .collect::<Vec<_>>();
6016 let disabled = match &mode {
6019 RescanMode::Candidate(_) => resulting_modules
6020 .iter()
6021 .filter(|(_, enabled)| !enabled)
6022 .map(|(module_id, _)| module_id.clone())
6023 .collect::<Vec<_>>(),
6024 _ => Vec::new(),
6025 };
6026 self.capability_evaluator.preview_warnings(
6027 resulting_modules,
6028 &removed,
6029 &disabled,
6030 ®istrations,
6031 )
6032 } else {
6033 Vec::new()
6034 };
6035 let reconciled = match &mode {
6036 RescanMode::Candidate(candidate) => self
6037 .preview_candidate(&context, modules, candidate)
6038 .map(|result| (result, Vec::new())),
6039 _ => {
6040 self.reconcile_supervised_modules(
6041 &context.supervisor,
6042 modules,
6043 preview,
6044 requester.clone(),
6045 )
6046 .await
6047 }
6048 };
6049 let (result, changed) = match reconciled {
6050 Ok(result) => result,
6051 Err(message) => {
6052 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6053 }
6054 };
6055 if !preview {
6056 if let Some(ledger) = &self.activity {
6057 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6058 ledger.config_changed(&result.added, &result.removed, &changed, requester);
6059 }
6060 if let Err(code) = self.configure_activity(activity_config).await {
6061 warn!(
6062 code,
6063 "activity retention could not be applied during rescan"
6064 );
6065 }
6066 self.capability_evaluator
6067 .configure(configured_capabilities, reserved_capabilities);
6068 self.capability_evaluator.wake_deadline_loop();
6069 self.refresh_capability_requirements();
6070 }
6071 let mut result = result;
6072 result.restart_required = restart_required;
6073 result.capability_warnings = preview_capability_warnings;
6074 let response = ClientControlResponse::SupervisorRescan { result };
6075 Ok(vec![control_response_body_frame(
6076 &frame,
6077 &response,
6078 "ClientControlResponse::SupervisorRescan",
6079 )?])
6080 }
6081
6082 async fn handle_supervisor_release_reserved(
6083 &self,
6084 frame: Frame,
6085 module_id: String,
6086 ) -> Result<Vec<Frame>, RouterError> {
6087 let Some(context) = self.rescan.clone() else {
6088 return Ok(vec![control_error_frame(
6089 &frame,
6090 "release_unavailable",
6091 "reserved-id release requires a daemon started with a reloadable config path",
6092 )?]);
6093 };
6094 let operation_lock = self.supervisor.operation_lock();
6095 let _operation_guard = operation_lock.lock().await;
6096 let loaded = match crate::daemon_config::load(&context.config_path) {
6097 Ok(Some(config)) => config,
6098 Ok(None) => {
6099 return Ok(vec![control_error_frame(
6100 &frame,
6101 "invalid_daemon_config",
6102 format!(
6103 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6104 context.config_path.display()
6105 ),
6106 )?])
6107 }
6108 Err(err) => {
6109 return Ok(vec![control_error_frame(
6110 &frame,
6111 "invalid_daemon_config",
6112 format!("unable to verify reserved-id release against daemon config: {err}"),
6113 )?])
6114 }
6115 };
6116 if loaded
6117 .modules
6118 .iter()
6119 .any(|configured| configured.module_id == module_id)
6120 {
6121 return Ok(vec![control_error_frame(
6122 &frame,
6123 "reserved_module_configured",
6124 format!(
6125 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6126 ),
6127 )?]);
6128 }
6129 if !self.supervisor.release_retained_reserved_gate(&module_id) {
6130 return Ok(vec![control_error_frame(
6131 &frame,
6132 "reserved_gate_not_retained",
6133 format!(
6134 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6135 ),
6136 )?]);
6137 }
6138
6139 let response = ClientControlResponse::SupervisorAck {
6140 module_id,
6141 applied: true,
6142 };
6143 Ok(vec![control_response_body_frame(
6144 &frame,
6145 &response,
6146 "ClientControlResponse::SupervisorAck",
6147 )?])
6148 }
6149
6150 fn plan_rescan(
6154 &self,
6155 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6156 ) -> Result<RescanPlan, String> {
6157 let mut current = BTreeMap::new();
6158 for module in self.supervisor.list() {
6159 let (spec, health) = module.configuration().map_err(|err| {
6160 format!(
6161 "failed to read configuration for module_id '{}': {err}",
6162 module.module_id()
6163 )
6164 })?;
6165 let enabled = module
6166 .status()
6167 .map_err(|err| {
6168 format!(
6169 "failed to read status for module_id '{}': {err}",
6170 module.module_id()
6171 )
6172 })?
6173 .enabled;
6174 current.insert(
6175 module.module_id().to_string(),
6176 (module, spec, health, enabled),
6177 );
6178 }
6179 let configured = configured_modules
6180 .into_iter()
6181 .map(|module| (module.module_id.clone(), module))
6182 .collect::<BTreeMap<_, _>>();
6183
6184 let added = configured
6185 .keys()
6186 .filter(|module_id| !current.contains_key(*module_id))
6187 .cloned()
6188 .collect::<Vec<_>>();
6189 let removed = current
6190 .keys()
6191 .filter(|module_id| !configured.contains_key(*module_id))
6192 .cloned()
6193 .collect::<Vec<_>>();
6194 let mut changed_pending_reload = Vec::new();
6195 let mut configuration_changes = BTreeSet::new();
6196 let mut enabled_changes = BTreeSet::new();
6197 let mut unchanged = 0_u32;
6198
6199 for (module_id, configured_module) in &configured {
6200 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6201 else {
6202 continue;
6203 };
6204 let launch_changed = *current_spec != configured_module.module_spec();
6208 let configuration_changed =
6209 launch_changed || *current_health != configured_module.health;
6210 let enabled_changed = *current_enabled != configured_module.enabled;
6211 if configuration_changed {
6212 configuration_changes.insert(module_id.clone());
6213 }
6214 if launch_changed {
6215 changed_pending_reload.push(module_id.clone());
6216 }
6217 if enabled_changed {
6218 enabled_changes.insert(module_id.clone());
6219 }
6220 if !configuration_changed && !enabled_changed {
6221 unchanged = unchanged.saturating_add(1);
6222 }
6223 }
6224 Ok(RescanPlan {
6225 current,
6226 configured,
6227 added,
6228 removed,
6229 changed_pending_reload,
6230 configuration_changes,
6231 enabled_changes,
6232 unchanged,
6233 })
6234 }
6235
6236 async fn reconcile_supervised_modules(
6245 &self,
6246 supervisor: &Supervisor,
6247 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6248 preview: bool,
6249 requester: Principal,
6250 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6251 let plan = self.plan_rescan(configured_modules)?;
6252
6253 if preview {
6259 return Ok((plan.preview_result(), Vec::new()));
6260 }
6261 let changed = plan
6262 .configuration_changes
6263 .union(&plan.enabled_changes)
6264 .cloned()
6265 .collect::<Vec<_>>();
6266 let RescanPlan {
6267 current,
6268 configured,
6269 added,
6270 removed,
6271 changed_pending_reload,
6272 configuration_changes,
6273 enabled_changes,
6274 unchanged,
6275 } = plan;
6276 if !(added.is_empty()
6281 && removed.is_empty()
6282 && configuration_changes.is_empty()
6283 && enabled_changes.is_empty())
6284 {
6285 if let Some(context) = &self.rescan {
6286 context
6287 .generation
6288 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6289 }
6290 }
6291
6292 for module_id in &removed {
6293 let module = ¤t
6294 .get(module_id)
6295 .expect("removed module came from current supervisor state")
6296 .0;
6297 module
6298 .retire_requested(Some(requester.clone()))
6299 .await
6300 .map_err(|err| {
6301 format!("failed to retire module_id '{module_id}' during rescan: {err}")
6302 })?;
6303 self.supervisor.record_rescan_removal(module_id);
6337 self.supervisor.retire(module_id);
6338 self.route_outages.forget(module_id);
6339 }
6340
6341 for module_id in configured.keys() {
6342 let Some((module, _, _, _)) = current.get(module_id) else {
6343 continue;
6344 };
6345 let configured_module = configured
6346 .get(module_id)
6347 .expect("configured module id came from configured map");
6348 if configuration_changes.contains(module_id) {
6349 module
6350 .update_configuration(
6351 configured_module.module_spec(),
6352 configured_module.health.clone(),
6353 configured_module.drain_timeout_ms,
6354 )
6355 .await
6356 .map_err(|err| {
6357 format!(
6358 "failed to update module_id '{module_id}' configuration during rescan: {err}"
6359 )
6360 })?;
6361 }
6362 if enabled_changes.contains(module_id) {
6363 self.route_outages.mark_operator_action(module_id);
6366 module
6367 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6368 .await
6369 .map_err(|err| {
6370 self.route_outages.operator_action_ended_unrefused(module_id);
6371 format!(
6372 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6373 configured_module.enabled
6374 )
6375 })?;
6376 }
6377 }
6378
6379 for module_id in &added {
6380 let configured_module = configured
6381 .get(module_id)
6382 .expect("added module id came from configured map");
6383 supervisor
6384 .supervise_configured_requested(
6385 configured_module.module_spec(),
6386 configured_module.enabled,
6387 configured_module.health.clone(),
6388 configured_module.drain_timeout_ms,
6389 configured_module.restart,
6390 configured_module.register_timeout,
6391 Some(requester.clone()),
6392 )
6393 .map_err(|err| {
6394 format!("failed to add module_id '{module_id}' during rescan: {err}")
6395 })?;
6396 }
6397
6398 Ok((
6399 SupervisorRescanResult {
6400 added,
6401 removed,
6402 changed_pending_reload,
6403 enabled_changes: enabled_changes.iter().cloned().collect(),
6404 unchanged,
6405 preview: false,
6406 restart_required: Vec::new(),
6409 capability_warnings: Vec::new(),
6410 module_actions: Vec::new(),
6411 earlier_pending: Vec::new(),
6412 expect: None,
6413 },
6414 changed,
6415 ))
6416 }
6417
6418 async fn handle_supervisor_set_enabled(
6419 &self,
6420 frame: Frame,
6421 module_id: String,
6422 enabled: bool,
6423 requester: Principal,
6424 ) -> Result<Vec<Frame>, RouterError> {
6425 let operation_lock = self.supervisor.operation_lock();
6426 let _operation_guard = operation_lock.lock().await;
6427 let Some(module) = self.supervisor.get(&module_id) else {
6428 return Ok(vec![control_error_frame(
6429 &frame,
6430 "unknown_module",
6431 format!("module_id '{module_id}' is not supervised"),
6432 )?]);
6433 };
6434
6435 self.route_outages.mark_operator_action(&module_id);
6438 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6439 Ok(applied) => applied,
6440 Err(err) => {
6441 self.route_outages
6442 .operator_action_ended_unrefused(&module_id);
6443 return Ok(vec![control_error_frame(
6444 &frame,
6445 "target_unavailable",
6446 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6447 )?]);
6448 }
6449 };
6450 if !applied {
6451 self.route_outages
6454 .operator_action_ended_unrefused(&module_id);
6455 }
6456
6457 self.capability_evaluator.wake_deadline_loop();
6458 self.refresh_capability_requirements();
6459 let response = ClientControlResponse::SupervisorAck { module_id, applied };
6460 Ok(vec![control_response_body_frame(
6461 &frame,
6462 &response,
6463 "ClientControlResponse::SupervisorAck",
6464 )?])
6465 }
6466
6467 async fn handle_supervisor_health_probe(
6468 &self,
6469 frame: Frame,
6470 module_id: String,
6471 ) -> Result<Vec<Frame>, RouterError> {
6472 self.refresh_capability_requirements();
6473 let Some(registration) = self
6474 .registry
6475 .get_module(&module_id)
6476 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6477 else {
6478 return Ok(vec![control_error_frame(
6479 &frame,
6480 "unknown_module",
6481 format!("module_id '{module_id}' is not registered"),
6482 )?]);
6483 };
6484
6485 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6498 {
6499 return Ok(vec![control_error_frame(
6500 &frame,
6501 "health_not_advertised",
6502 format!("module_id '{module_id}' did not advertise health.check"),
6503 )?]);
6504 }
6505
6506 let deadline = Instant::now() + self.health_probe_timeout;
6507 let pending = match self.forwarding.begin_module_control_rpc_for(
6508 &module_id,
6509 MODULE_CONTROL_OP_HEALTH_CHECK,
6510 deadline,
6511 ) {
6512 Ok(pending) => pending,
6513 Err(err) => {
6514 return Ok(vec![control_error_frame(
6515 &frame,
6516 forwarding_error_code(&err),
6517 err.to_string(),
6518 )?])
6519 }
6520 };
6521
6522 let PendingModuleControlRpc {
6523 endpoint,
6524 module_sink,
6525 negotiated_ver,
6526 corr: probe_corr,
6527 receiver,
6528 } = pending;
6529 let mut guard =
6530 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6531 let probe_body =
6532 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6533 RouterError::backend(
6534 0,
6535 frame.header.corr,
6536 format!("failed to encode health.check request: {err}"),
6537 )
6538 })?;
6539 let probe_frame = Frame::build_with_version(
6540 negotiated_ver,
6541 FrameType::Request,
6542 control_flags(),
6543 0,
6544 0,
6545 probe_corr,
6546 probe_body,
6547 )
6548 .map_err(RouterError::FrameBuild)?;
6549
6550 match timeout_at(deadline, module_sink.send(probe_frame)).await {
6553 Ok(Ok(())) => {}
6554 Ok(Err(err)) => {
6555 return Ok(vec![control_error_frame(
6556 &frame,
6557 "target_unavailable",
6558 err.to_string(),
6559 )?]);
6560 }
6561 Err(_) => {
6562 return Ok(vec![control_error_frame(
6563 &frame,
6564 "module_timeout",
6565 format!(
6566 "module_id '{module_id}' did not answer health.check within {:?}",
6567 self.health_probe_timeout
6568 ),
6569 )?]);
6570 }
6571 }
6572
6573 match timeout_at(deadline, receiver).await {
6574 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6575 guard.disarm();
6576 let Some(report) = response.health_report() else {
6577 return Ok(vec![control_error_frame(
6578 &frame,
6579 "invalid_control_body",
6580 "health.check RPC returned a non-health response",
6581 )?]);
6582 };
6583 let HealthReport {
6588 status,
6589 detail,
6590 metrics,
6591 } = report;
6592 let capability_detail = self
6593 .capability_evaluator
6594 .required_problem_detail(&module_id);
6595 let response = ClientControlResponse::SupervisorHealthProbe {
6596 module_id,
6597 status,
6598 detail: append_capability_problem_detail(detail, capability_detail),
6599 metrics,
6600 };
6601 Ok(vec![control_response_body_frame(
6602 &frame,
6603 &response,
6604 "ClientControlResponse::SupervisorHealthProbe",
6605 )?])
6606 }
6607 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6608 guard.disarm();
6609 Ok(vec![control_error_body_frame(&frame, body)?])
6610 }
6611 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6612 guard.disarm();
6613 Ok(vec![control_error_frame(
6614 &frame,
6615 "target_unavailable",
6616 message,
6617 )?])
6618 }
6619 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6620 guard.disarm();
6621 Ok(vec![control_error_frame(
6622 &frame,
6623 "invalid_control_body",
6624 message,
6625 )?])
6626 }
6627 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6628 guard.disarm();
6629 Ok(vec![control_error_frame(
6630 &frame,
6631 "invalid_control_body",
6632 format!("expected module-control op '{expected}', got '{actual}'"),
6633 )?])
6634 }
6635 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6636 guard.disarm();
6637 Ok(vec![control_error_frame(
6638 &frame,
6639 "module_timeout",
6640 format!(
6641 "module_id '{module_id}' answered health.check after {:?}",
6642 self.health_probe_timeout
6643 ),
6644 )?])
6645 }
6646 Ok(Err(_)) => Ok(vec![control_error_frame(
6647 &frame,
6648 "target_unavailable",
6649 "health.check waiter was canceled before the module responded",
6650 )?]),
6651 Err(_) => Ok(vec![control_error_frame(
6652 &frame,
6653 "module_timeout",
6654 format!(
6655 "module_id '{module_id}' did not answer health.check within {:?}",
6656 self.health_probe_timeout
6657 ),
6658 )?]),
6659 }
6660 }
6661
6662 fn supervisor_status(
6663 &self,
6664 module_id: &str,
6665 corr: u64,
6666 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6667 self.supervisor
6668 .get(module_id)
6669 .map(|module| {
6670 let warming = module.is_warming_for_control("status").map_err(|err| {
6671 RouterError::backend(
6672 0,
6673 corr,
6674 format!(
6675 "failed to read supervisor warming state for module_id '{module_id}': {err}"
6676 ),
6677 )
6678 })?;
6679 module.status_for_control("status").map_err(|err| {
6680 RouterError::backend(
6681 0,
6682 corr,
6683 format!(
6684 "failed to read supervisor status for module_id '{module_id}': {err}"
6685 ),
6686 )
6687 }).map(|status| (status, warming))
6688 })
6689 .transpose()
6690 }
6691
6692 fn guard_module_control_op(
6693 &self,
6694 frame: &Frame,
6695 module_id: &str,
6696 op: &str,
6697 ) -> Result<Option<Frame>, RouterError> {
6698 if self.module_grants_op(module_id, op, frame.header.corr)? {
6699 return Ok(None);
6700 }
6701
6702 Ok(Some(control_error_frame(
6703 frame,
6704 "op_not_allowed",
6705 format!("module_id '{module_id}' did not grant control op '{op}'"),
6706 )?))
6707 }
6708
6709 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6710 let Some(registration) = self
6711 .registry
6712 .get_module(module_id)
6713 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6714 else {
6715 return Ok(false);
6716 };
6717 Ok(module_registration_grants_op(®istration.control_ops, op))
6718 }
6719
6720 fn handle_status_update(
6721 &self,
6722 endpoint: ModuleEndpointId,
6723 frame: Frame,
6724 ) -> Result<Vec<Frame>, RouterError> {
6725 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6726 Ok(update) => update,
6727 Err(err) => {
6728 if is_known_module_push_op(&frame.body) {
6733 return Ok(vec![control_error_frame(
6734 &frame,
6735 "invalid_control_body",
6736 format!("malformed module control push body: {err}"),
6737 )?]);
6738 }
6739 return Ok(Vec::new());
6740 }
6741 };
6742
6743 match update {
6744 ModuleControlPush::RouteStatus {
6745 route_channel,
6746 route_epoch,
6747 status,
6748 } => {
6749 self.forwarding
6750 .cache_status(endpoint, route_channel, route_epoch, status)
6751 .map_err(RouterError::Forwarding)?;
6752 }
6753 }
6754 Ok(Vec::new())
6755 }
6756
6757 fn handle_route_poll(
6758 &self,
6759 ctx: &RouteCtx,
6760 frame: Frame,
6761 route_channel: u16,
6762 route_epoch: u32,
6763 kind: PollKind,
6764 ) -> Result<Vec<Frame>, RouterError> {
6765 let snapshot = self
6766 .forwarding
6767 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6768 .map_err(RouterError::Forwarding)?;
6769 let response = match (kind, snapshot) {
6770 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6771 ClientControlResponse::RoutePoll {
6772 route_channel,
6773 route_epoch,
6774 status,
6775 live: None,
6776 }
6777 }
6778 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6779 route_channel,
6780 route_epoch,
6781 status: None,
6782 live: None,
6783 },
6784 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6785 let live = self
6811 .process_liveness
6812 .as_ref()
6813 .and_then(|source| source.process_live(&module_id))
6814 .unwrap_or(true);
6815 ClientControlResponse::RoutePoll {
6816 route_channel,
6817 route_epoch,
6818 status: None,
6819 live: Some(live),
6820 }
6821 }
6822 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6823 route_channel,
6824 route_epoch,
6825 status: None,
6826 live: Some(false),
6827 },
6828 };
6829
6830 Ok(vec![control_response_body_frame(
6831 &frame,
6832 &response,
6833 "ClientControlResponse::RoutePoll",
6834 )?])
6835 }
6836
6837 pub(crate) fn observe_module_control_completion(
6838 &self,
6839 completion: ModuleControlRpcCompletion,
6840 ) -> bool {
6841 match completion {
6842 ModuleControlRpcCompletion::Unknown => false,
6843 ModuleControlRpcCompletion::Settled => true,
6844 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6845 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6846 info!(
6847 module_id = %module_id,
6848 latency_ms,
6849 "late health.check answer proves the module is alive"
6850 );
6851 match self
6852 .supervisor
6853 .record_late_health_answer(&module_id, latency_ms)
6854 {
6855 Ok(true) => {}
6856 Ok(false) => debug!(
6857 module_id = %module_id,
6858 latency_ms,
6859 "late health.check answer has no active supervisor snapshot"
6860 ),
6861 Err(err) => warn!(
6862 module_id = %module_id,
6863 latency_ms,
6864 error = %err,
6865 "failed to record late health.check answer"
6866 ),
6867 }
6868 true
6869 }
6870 }
6871 }
6872
6873 fn refuse_to_end_module_connection_for_a_client(
6894 &self,
6895 module_connection_id: ConnectionId,
6896 corr: u64,
6897 err: ForwardingError,
6898 ) -> Result<(), RouterError> {
6899 if let ForwardingError::ConnectionClosing { connection_id } = err {
6900 if connection_id != module_connection_id {
6901 warn!(
6902 module_connection_id = module_connection_id.get(),
6903 client_connection_id = connection_id.get(),
6904 corr,
6905 "dropping a route.bind response for a closing client; the module connection keeps serving"
6906 );
6907 return Ok(());
6908 }
6909 }
6910 Err(RouterError::Forwarding(err))
6911 }
6912
6913 fn handle_module_relay_response(
6914 &self,
6915 connection_id: ConnectionId,
6916 frame: Frame,
6917 ) -> Result<Vec<Frame>, RouterError> {
6918 let mut secondary_error = None;
6919 let outcome = match frame.header.ty {
6920 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6921 Ok(probe) if probe.op == "route.bind" => {
6922 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6923 Ok(ModuleControlResponse::RouteBindAck {}) => {
6924 RouteBindRelayOutcome::Accepted
6925 }
6926 Ok(other) => {
6927 let message =
6928 format!("route.bind response carried unexpected body: {other:?}");
6929 secondary_error = Some(control_error_frame(
6930 &frame,
6931 "invalid_control_body",
6932 message.clone(),
6933 )?);
6934 RouteBindRelayOutcome::ModuleGone(message)
6935 }
6936 Err(err) => {
6937 let message = format!("malformed route.bind response body: {err}");
6938 secondary_error = Some(control_error_frame(
6939 &frame,
6940 "invalid_control_body",
6941 message.clone(),
6942 )?);
6943 RouteBindRelayOutcome::ModuleGone(message)
6944 }
6945 }
6946 }
6947 Ok(probe) => {
6948 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6949 {
6950 Ok(response) => ModuleControlRpcOutcome::Response(response),
6951 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6952 "malformed {} response body: {err}",
6953 probe.op
6954 )),
6955 };
6956 let completion = self
6957 .forwarding
6958 .complete_module_control_rpc(
6959 connection_id,
6960 frame.header.corr,
6961 Some(&probe.op),
6962 outcome,
6963 )
6964 .map_err(RouterError::Forwarding)?;
6965 if !self.observe_module_control_completion(completion) {
6966 debug!(
6967 connection_id = connection_id.get(),
6968 corr = frame.header.corr,
6969 op = %probe.op,
6970 "dropping late or unknown module-control RPC response"
6971 );
6972 }
6973 return Ok(Vec::new());
6974 }
6975 Err(err) => {
6976 if let Some(expected_op) = self
6977 .forwarding
6978 .pending_module_control_op(connection_id, frame.header.corr)
6979 .map_err(RouterError::Forwarding)?
6980 {
6981 let completion = self
6982 .forwarding
6983 .complete_module_control_rpc(
6984 connection_id,
6985 frame.header.corr,
6986 None,
6987 ModuleControlRpcOutcome::MalformedResponse(format!(
6988 "malformed {expected_op} response body: {err}"
6989 )),
6990 )
6991 .map_err(RouterError::Forwarding)?;
6992 if !self.observe_module_control_completion(completion) {
6993 debug!(
6994 connection_id = connection_id.get(),
6995 corr = frame.header.corr,
6996 "dropping late malformed module-control RPC response"
6997 );
6998 }
6999 return Ok(Vec::new());
7000 }
7001 let message = format!("malformed route.bind response body: {err}");
7002 secondary_error = Some(control_error_frame(
7003 &frame,
7004 "invalid_control_body",
7005 message.clone(),
7006 )?);
7007 RouteBindRelayOutcome::ModuleGone(message)
7008 }
7009 },
7010 FrameType::Error => {
7011 if self
7012 .forwarding
7013 .pending_module_control_op(connection_id, frame.header.corr)
7014 .map_err(RouterError::Forwarding)?
7015 .is_some()
7016 {
7017 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7018 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7019 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7020 "malformed module-control ERROR body: {err}"
7021 )),
7022 };
7023 let completion = self
7024 .forwarding
7025 .complete_module_control_rpc(
7026 connection_id,
7027 frame.header.corr,
7028 None,
7029 outcome,
7030 )
7031 .map_err(RouterError::Forwarding)?;
7032 if !self.observe_module_control_completion(completion) {
7033 debug!(
7034 connection_id = connection_id.get(),
7035 corr = frame.header.corr,
7036 "dropping late or unknown module-control RPC error"
7037 );
7038 }
7039 return Ok(Vec::new());
7040 }
7041 match serde_json::from_slice::<ErrorBody>(&frame.body) {
7042 Ok(body) => RouteBindRelayOutcome::Rejected(body),
7043 Err(err) => {
7044 let message = format!("malformed route.bind ERROR body: {err}");
7045 secondary_error = Some(control_error_frame(
7046 &frame,
7047 "invalid_control_body",
7048 message.clone(),
7049 )?);
7050 RouteBindRelayOutcome::ModuleGone(message)
7051 }
7052 }
7053 }
7054 ty => {
7055 return Ok(vec![control_error_frame(
7056 &frame,
7057 "unsupported_control_frame",
7058 format!("unsupported module channel-0 frame {ty:?}"),
7059 )?])
7060 }
7061 };
7062
7063 let settled =
7064 self.forwarding
7065 .complete_pending_relay(connection_id, frame.header.corr, outcome);
7066 let completion = match settled {
7067 Ok(completion) => completion,
7068 Err(err) => {
7069 self.refuse_to_end_module_connection_for_a_client(
7070 connection_id,
7071 frame.header.corr,
7072 err,
7073 )?;
7074 return Ok(secondary_error.into_iter().collect());
7075 }
7076 };
7077 if let Some(target) = completion.abandoned.as_ref() {
7078 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7079 }
7080 if !completion.settled {
7081 debug!(
7082 connection_id = connection_id.get(),
7083 corr = frame.header.corr,
7084 frame_type = ?frame.header.ty,
7085 "dropping late or unknown route.bind relay response"
7086 );
7087 }
7088 Ok(secondary_error.into_iter().collect())
7089 }
7090
7091 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7092 debug!(connection_id = connection_id.get(), "handling GOODBYE");
7093 self.cleanup_connection_with_end_reason(
7097 connection_id,
7098 RegistrationEndReason::ExplicitGoodbye,
7099 )
7100 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7101 Ok(Vec::new())
7102 }
7103}
7104
7105impl Default for ControlHandler {
7106 fn default() -> Self {
7107 Self::new(Arc::new(Registry::default()))
7108 }
7109}
7110
7111impl crate::supervise::SwapPromotionObserver for ControlHandler {
7112 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7113 self.apply_registration_capabilities(registration);
7114 }
7115}
7116
7117fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7118 CapabilityRequirementStatus {
7119 consumer: status.consumer,
7120 capability: status.capability,
7121 need: match status.need {
7122 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7123 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7124 },
7125 verdict: status.verdict.as_str().to_string(),
7126 episode_seq: status.episode_seq,
7127 config_satisfiable: status.config_satisfiable,
7128 runtime_available: status.runtime_available,
7129 detail: status.detail,
7130 }
7131}
7132
7133fn append_capability_problem_detail(
7134 detail: Option<String>,
7135 capability_detail: Option<String>,
7136) -> Option<String> {
7137 match (detail, capability_detail) {
7138 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7139 (Some(detail), None) => Some(detail),
7140 (None, Some(capability_detail)) => Some(capability_detail),
7141 (None, None) => None,
7142 }
7143}
7144
7145fn subc_ops() -> Vec<String> {
7146 SUBC_CONTROL_OPS
7147 .iter()
7148 .map(|op| (*op).to_string())
7149 .collect()
7150}
7151
7152fn module_subc_ops() -> Vec<String> {
7153 SUBC_CONTROL_OPS
7154 .iter()
7155 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7156 .map(|op| (*op).to_string())
7157 .collect()
7158}
7159
7160#[cfg(test)]
7161fn module_baseline_control_ops() -> Vec<String> {
7162 MODULE_BASELINE_CONTROL_OPS
7163 .iter()
7164 .map(|op| (*op).to_string())
7165 .collect()
7166}
7167
7168fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7169 let mut seen = HashSet::new();
7170 let mut effective = Vec::new();
7171 for op in MODULE_BASELINE_CONTROL_OPS {
7172 if seen.insert((*op).to_string()) {
7173 effective.push((*op).to_string());
7174 }
7175 }
7176 for op in declared.unwrap_or_default() {
7177 if seen.insert(op.clone()) {
7178 effective.push(op);
7179 }
7180 }
7181 effective
7182}
7183
7184fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7185 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7186}
7187
7188fn target_module_id(target: &RouteTarget) -> &str {
7189 match target {
7190 RouteTarget::ToolProvider { module_id }
7191 | RouteTarget::ManagementSurface { module_id }
7192 | RouteTarget::InternalService { module_id, .. } => module_id,
7193 }
7194}
7195
7196fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7197 roles.iter().any(|role| match (target, role) {
7198 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7199 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7200 (
7201 RouteTarget::InternalService { service_id, .. },
7202 ProviderRole::InternalService {
7203 service_id: provided,
7204 ..
7205 },
7206 ) => service_id == provided,
7207 _ => false,
7208 })
7209}
7210
7211fn is_routable_role(role: &ProviderRole) -> bool {
7212 matches!(
7213 role,
7214 ProviderRole::ToolProvider { .. }
7215 | ProviderRole::ManagementSurface { .. }
7216 | ProviderRole::InternalService { .. }
7217 )
7218}
7219
7220#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7221enum ControlRequestBodyError {
7222 UnknownOp,
7223 InvalidBody,
7224}
7225
7226#[derive(Debug, Deserialize)]
7227struct ControlOpProbe {
7228 op: String,
7229}
7230
7231const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7234
7235fn is_known_module_push_op(body: &[u8]) -> bool {
7236 serde_json::from_slice::<ControlOpProbe>(body)
7237 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7238 .unwrap_or(false)
7239}
7240
7241fn is_known_module_request_op(body: &[u8]) -> bool {
7242 serde_json::from_slice::<ControlOpProbe>(body)
7243 .map(|probe| is_module_to_subc_op(&probe.op))
7244 .unwrap_or(false)
7245}
7246
7247fn is_module_to_subc_op(op: &str) -> bool {
7248 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7249}
7250
7251fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7252 debug!(
7253 op = %op,
7254 connection_id = connection_id.get(),
7255 corr,
7256 "control dispatch"
7257 );
7258}
7259
7260fn log_slow_control_dispatch(
7261 dispatch_started_at: Option<StdInstant>,
7262 op: &'static str,
7263 connection_id: ConnectionId,
7264 corr: u64,
7265 route_open_timings: Option<&RouteOpenTimings>,
7266) {
7267 let Some(dispatch_started_at) = dispatch_started_at else {
7268 return;
7269 };
7270 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7272 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7273 if let Some(timings) = route_open_timings {
7274 warn!(
7275 op = %op,
7276 connection_id = connection_id.get(),
7277 corr,
7278 elapsed_ms = elapsed.as_millis() as u64,
7279 module_id = ?timings.module_id,
7280 principal = %timings.principal,
7281 admission_ms = timings.admission.as_millis() as u64,
7282 project_root_ms = timings.project_root.as_millis() as u64,
7283 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7284 bind_send_ms = timings.bind_send.as_millis() as u64,
7285 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7286 "slow control dispatch"
7287 );
7288 return;
7289 }
7290 warn!(
7291 op = %op,
7292 connection_id = connection_id.get(),
7293 corr,
7294 elapsed_ms = elapsed.as_millis() as u64,
7295 "slow control dispatch"
7296 );
7297 }
7298}
7299
7300#[cfg(test)]
7302type TestCallerLookup =
7303 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7304
7305fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7307 match request {
7308 ClientControlRequest::SupervisorRestart { module_id, .. } => {
7309 Some(PrivilegedOperation::Restart { module_id })
7310 }
7311 ClientControlRequest::SupervisorSwap { module_id, .. } => {
7312 Some(PrivilegedOperation::Swap { module_id })
7313 }
7314 ClientControlRequest::SupervisorReload { module_id } => {
7315 Some(PrivilegedOperation::Reload { module_id })
7316 }
7317 ClientControlRequest::SupervisorRescan { preview } => {
7318 if *preview {
7319 None
7320 } else {
7321 Some(PrivilegedOperation::Rescan)
7322 }
7323 }
7324 ClientControlRequest::SupervisorRescanChecked {
7329 preview,
7330 candidate,
7331 expect,
7332 } => (!*preview && candidate.is_none() && expect.is_some())
7333 .then_some(PrivilegedOperation::Rescan),
7334 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7335 Some(PrivilegedOperation::SetEnabled {
7336 module_id,
7337 enabled: *enabled,
7338 })
7339 }
7340 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7341 Some(PrivilegedOperation::ReleaseReserved { module_id })
7342 }
7343 ClientControlRequest::ServerDescribe {}
7344 | ClientControlRequest::CatalogList { .. }
7345 | ClientControlRequest::RouteOpen { .. }
7346 | ClientControlRequest::RoutePoll { .. }
7347 | ClientControlRequest::SupervisorList {}
7348 | ClientControlRequest::SupervisorHealth {}
7349 | ClientControlRequest::SupervisorHealthProbe { .. }
7350 | ClientControlRequest::SupervisorRoutes { .. }
7351 | ClientControlRequest::SupervisorProvenance { .. }
7352 | ClientControlRequest::SupervisorStderrTail { .. }
7353 | ClientControlRequest::SupervisorTerminals { .. }
7354 | ClientControlRequest::SupervisorSpawnSnapshot {}
7355 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7356 }
7357}
7358
7359fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7360 match request {
7361 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7362 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7363 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7364 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7365 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7366 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7367 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7368 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7369 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7370 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7371 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7372 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7373 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7374 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7375 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7376 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7377 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7378 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7379 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7380 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7381 }
7382}
7383
7384struct RouteOpenCaller {
7392 principal: String,
7396 scope: String,
7399 harness: String,
7402}
7403
7404impl RouteOpenCaller {
7405 fn new(
7406 principal: Option<&Principal>,
7407 scope: Option<&ScopeSelector>,
7408 identity: &BindIdentity,
7409 ) -> Self {
7410 Self {
7411 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
7412 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
7413 harness: bounded_log_text(&identity.harness),
7414 }
7415 }
7416
7417 fn unknown() -> Self {
7420 Self {
7421 principal: "unknown".to_string(),
7422 scope: "unknown".to_string(),
7423 harness: "unknown".to_string(),
7424 }
7425 }
7426}
7427
7428const MAX_LOGGED_TEXT_CHARS: usize = 64;
7431
7432fn bounded_log_text(value: &str) -> String {
7436 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
7437 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
7438 bounded.push('…');
7439 }
7440 format!("{bounded:?}")
7441}
7442
7443fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7449 use subc_protocol::scope::ScopeKind;
7450 let kind = match stamp.kind {
7451 ScopeKind::Head => "head",
7452 ScopeKind::Worker => "worker",
7453 ScopeKind::Ephemeral => "ephemeral",
7454 };
7455 format!(
7456 "{kind}/{}/{}@{}",
7457 principal_label(&stamp.owner),
7458 bounded_log_text(&stamp.scope_ref),
7459 stamp.scope_epoch
7460 )
7461}
7462
7463fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
7470 let epoch = selector
7471 .scope_epoch
7472 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
7473 format!(
7474 "requested/{}/{}@{epoch}",
7475 bounded_log_text(&principal_label(&selector.owner)),
7476 bounded_log_text(&selector.scope_ref),
7477 )
7478}
7479
7480fn principal_label(principal: &Principal) -> String {
7481 match principal {
7482 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7483 Principal::Direct => "direct".to_string(),
7484 other => format!("{other:?}"),
7485 }
7486}
7487
7488fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7489 match request {
7490 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7491 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7492 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7493 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7494 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7495 }
7496}
7497
7498fn parse_client_control_request(
7499 body: &[u8],
7500) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7501 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7502 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7503 Ok(probe)
7504 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7505 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7506 {
7507 ControlRequestBodyError::InvalidBody
7508 }
7509 Ok(_) => ControlRequestBodyError::UnknownOp,
7510 Err(_) => ControlRequestBodyError::InvalidBody,
7511 };
7512 (err, classification)
7513 })
7514}
7515
7516fn parse_module_control_request_from_module(
7517 body: &[u8],
7518) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7519 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7520 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7521 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7522 Ok(_) => ControlRequestBodyError::UnknownOp,
7523 Err(_) => ControlRequestBodyError::InvalidBody,
7524 };
7525 (err, classification)
7526 })
7527}
7528
7529#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7530enum ProviderRoleKind {
7531 ToolProvider,
7532 PipelineStage,
7533 ManagementSurface,
7534 InternalService,
7535}
7536
7537fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7538 match role {
7539 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7540 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7541 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7542 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7543 }
7544}
7545
7546fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7547 roles.iter().map(provider_role_kind).collect()
7548}
7549
7550const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7553
7554#[derive(Debug, Default, PartialEq, Eq)]
7556struct ScopeOutcomeCounts {
7557 created: usize,
7558 replaced: usize,
7559 updated: usize,
7560 unchanged: usize,
7561 refused: usize,
7562}
7563
7564impl ScopeOutcomeCounts {
7565 fn of(results: &[ScopeRecordResult]) -> Self {
7566 let mut counts = Self::default();
7567 for result in results {
7568 let slot = match result.outcome {
7569 ScopeRecordOutcome::Created => &mut counts.created,
7570 ScopeRecordOutcome::Replaced => &mut counts.replaced,
7571 ScopeRecordOutcome::Updated => &mut counts.updated,
7572 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7573 ScopeRecordOutcome::Refused => &mut counts.refused,
7574 };
7575 *slot += 1;
7576 }
7577 counts
7578 }
7579}
7580
7581#[cfg(test)]
7582mod scope_outcome_count_tests {
7583 use super::*;
7584
7585 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7586 ScopeRecordResult {
7587 scope_ref: "r".to_string(),
7588 scope_epoch: 1,
7589 outcome,
7590 code: None,
7591 message: None,
7592 version: None,
7593 parent_state: None,
7594 }
7595 }
7596
7597 #[test]
7600 fn every_outcome_is_counted_in_its_own_field() {
7601 let results = [
7602 result(ScopeRecordOutcome::Created),
7603 result(ScopeRecordOutcome::Created),
7604 result(ScopeRecordOutcome::Replaced),
7605 result(ScopeRecordOutcome::Updated),
7606 result(ScopeRecordOutcome::Unchanged),
7607 result(ScopeRecordOutcome::Refused),
7608 result(ScopeRecordOutcome::Refused),
7609 result(ScopeRecordOutcome::Refused),
7610 ];
7611 assert_eq!(
7612 ScopeOutcomeCounts::of(&results),
7613 ScopeOutcomeCounts {
7614 created: 2,
7615 replaced: 1,
7616 updated: 1,
7617 unchanged: 1,
7618 refused: 3,
7619 }
7620 );
7621 }
7622}
7623
7624fn capability_census_trigger(
7628 old: Option<&CapabilityDeclarations>,
7629 new: Option<&CapabilityDeclarations>,
7630) -> bool {
7631 let old_provides = old
7632 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7633 .unwrap_or_default();
7634 let old_denies = old
7635 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7636 .unwrap_or_default();
7637 let new = new.cloned().unwrap_or(CapabilityDeclarations {
7638 provides: Vec::new(),
7639 requires: Vec::new(),
7640 must_never_reach: Vec::new(),
7641 });
7642
7643 new.provides
7644 .iter()
7645 .any(|capability| !old_provides.contains(capability))
7646 || new
7647 .must_never_reach
7648 .iter()
7649 .any(|capability| !old_denies.contains(capability))
7650}
7651
7652fn denied_capability<'a>(
7655 opening_manifest: &'a ModuleManifest,
7656 target_manifest: &ModuleManifest,
7657) -> Option<&'a str> {
7658 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7659 let target_capabilities = target_manifest.capabilities.as_ref()?;
7660 opening_capabilities
7661 .must_never_reach
7662 .iter()
7663 .find(|denied| {
7664 target_capabilities
7665 .provides
7666 .iter()
7667 .any(|provided| provided == *denied)
7668 })
7669 .map(String::as_str)
7670}
7671
7672fn catalog_update_frozen_field_message(
7673 registered: &ModuleManifest,
7674 provides: &[ProviderRole],
7675) -> Option<String> {
7676 let old_has_provides = !registered.provides.is_empty();
7677 let new_has_provides = !provides.is_empty();
7678 if old_has_provides != new_has_provides {
7679 return Some(format!(
7680 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7681 registered.module_id
7682 ));
7683 }
7684
7685 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
7686 return Some(format!(
7687 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7688 registered.module_id
7689 ));
7690 }
7691
7692 let registered_concurrency = manifest_concurrency(registered);
7693 let mut candidate = registered.clone();
7694 candidate.provides = provides.to_vec();
7695 let candidate_concurrency = manifest_concurrency(&candidate);
7696 if candidate_concurrency != registered_concurrency {
7697 return Some(format!(
7698 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7699 registered.module_id, registered_concurrency, candidate_concurrency
7700 ));
7701 }
7702
7703 None
7706}
7707
7708fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7709 manifest.provides.iter().any(is_routable_role)
7710}
7711
7712fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7718 manifest
7719 .provides
7720 .iter()
7721 .find_map(|provider| match provider {
7722 ProviderRole::ToolProvider { concurrency, .. }
7723 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7724 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7725 })
7726 .unwrap_or(Concurrency::ModuleManaged)
7727}
7728
7729fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7736 let has_management_surface = manifest
7737 .provides
7738 .iter()
7739 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7740 if !has_management_surface {
7741 return false;
7742 }
7743 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7744 return false;
7745 };
7746 let Some(provides) = raw
7747 .get("manifest")
7748 .and_then(|manifest| manifest.get("provides"))
7749 .and_then(serde_json::Value::as_array)
7750 else {
7751 return false;
7752 };
7753 provides.iter().any(|role| {
7758 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7759 && role.get("concurrency").is_none()
7760 })
7761}
7762
7763fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7764 if peer_version != PROTOCOL_VERSION {
7765 return Err(format!(
7766 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7767 ));
7768 }
7769 Ok(PROTOCOL_VERSION)
7770}
7771
7772fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7773 Frame::build_with_version(
7774 response_version(frame),
7775 FrameType::Pong,
7776 frame.header.flags,
7777 0,
7778 0,
7779 frame.header.corr,
7780 Vec::new(),
7781 )
7782 .map_err(RouterError::FrameBuild)
7783}
7784
7785pub(crate) fn control_error_frame(
7786 frame: &Frame,
7787 code: &'static str,
7788 message: impl Into<String>,
7789) -> Result<Frame, RouterError> {
7790 control_error_body_frame(
7791 frame,
7792 ErrorBody {
7793 code: code.to_string(),
7794 message: message.into(),
7795 detail: None,
7796 },
7797 )
7798}
7799
7800fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7801 let body = serde_json::to_vec(&error).map_err(|err| {
7802 RouterError::backend(
7803 0,
7804 frame.header.corr,
7805 format!("failed to encode control ERROR: {err}"),
7806 )
7807 })?;
7808
7809 Frame::build_with_version(
7810 response_version(frame),
7811 FrameType::Error,
7812 control_flags(),
7813 0,
7814 0,
7815 frame.header.corr,
7816 body,
7817 )
7818 .map_err(RouterError::FrameBuild)
7819}
7820
7821fn control_response_body_frame<T: Serialize>(
7822 frame: &Frame,
7823 reply: &T,
7824 label: &'static str,
7825) -> Result<Frame, RouterError> {
7826 let body = serde_json::to_vec(reply).map_err(|err| {
7827 RouterError::backend(
7828 0,
7829 frame.header.corr,
7830 format!("failed to encode {label}: {err}"),
7831 )
7832 })?;
7833
7834 Frame::build_with_version(
7835 response_version(frame),
7836 FrameType::Response,
7837 control_flags(),
7838 0,
7839 0,
7840 frame.header.corr,
7841 body,
7842 )
7843 .map_err(RouterError::FrameBuild)
7844}
7845
7846fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7876 match err {
7877 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7878 ForwardingError::NoModuleConnection => "target_unavailable",
7879 ForwardingError::ModuleReloading { .. } => "module_reloading",
7880 ForwardingError::ClientRouteChannelExhausted { .. }
7881 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7882 ForwardingError::StaleModuleEndpoint
7883 | ForwardingError::UnknownReservation { .. }
7884 | ForwardingError::ConnectionClosing { .. }
7885 | ForwardingError::ClientEgressClosed { .. }
7886 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7887 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7890 ForwardingError::RelayCorrelationExhausted
7891 | ForwardingError::RouteOpenBuild(_)
7892 | ForwardingError::Poisoned => "forwarding_error",
7893 }
7894}
7895
7896fn response_version(frame: &Frame) -> u8 {
7897 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7898 frame.header.ver
7899 } else {
7900 PROTOCOL_VERSION
7901 }
7902}
7903
7904fn control_flags() -> Flags {
7905 Flags::new(false, Priority::Passive, false)
7906}
7907
7908fn send_goodbye_target_best_effort(
7913 counters: &DaemonCounters,
7914 target: &GoodbyeTarget,
7915 context: &'static str,
7916) {
7917 let Ok(frame) = Frame::build_with_version(
7918 target.negotiated_ver,
7919 FrameType::Goodbye,
7920 control_flags(),
7921 target.channel,
7922 target.epoch,
7923 0,
7924 Vec::new(),
7925 ) else {
7926 return;
7927 };
7928 crate::forwarding::send_module_route_goodbye(
7929 counters,
7930 &target.sink,
7931 frame,
7932 target.module_id.as_deref(),
7933 context,
7934 );
7935}
7936
7937pub(crate) fn send_route_control_pushes(
7938 forwarding: &ForwardingTable,
7939 routes: Vec<EndpointRoute>,
7940 push: ClientControlPush,
7941) {
7942 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7943 for route in routes {
7944 let target = route.goodbye_target;
7945 if let Some((existing, channels)) = targets
7946 .iter_mut()
7947 .find(|(existing, _)| existing.connection_id == target.connection_id)
7948 {
7949 debug_assert_eq!(
7950 existing.negotiated_ver, target.negotiated_ver,
7951 "one connection cannot negotiate multiple frame versions"
7952 );
7953 if !channels.contains(&target.channel) {
7954 channels.push(target.channel);
7955 }
7956 continue;
7957 }
7958 let channel = target.channel;
7959 targets.push((target, vec![channel]));
7960 }
7961 for (target, mut channels) in targets {
7962 channels.sort_unstable();
7963 let mut push = push.clone();
7964 match &mut push {
7965 ClientControlPush::RouteClosing {
7966 channels: covered, ..
7967 }
7968 | ClientControlPush::RouteClosed {
7969 channels: covered, ..
7970 } => *covered = channels,
7971 }
7972 let body = match serde_json::to_vec(&push) {
7973 Ok(body) => body,
7974 Err(err) => {
7975 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7976 continue;
7977 }
7978 };
7979 let frame = match Frame::build_with_version(
7980 target.negotiated_ver,
7981 FrameType::Push,
7982 control_flags(),
7983 0,
7984 0,
7985 0,
7986 body.clone(),
7987 ) {
7988 Ok(frame) => frame,
7989 Err(err) => {
7990 warn!(
7991 route_channel = target.channel,
7992 error = %err,
7993 "failed to build route lifecycle control PUSH frame"
7994 );
7995 continue;
7996 }
7997 };
7998 if let Err(err) = target.sink.try_send(frame) {
7999 if target.close_on_delivery_failure() {
8000 warn!(
8001 target_connection_id = target.connection_id.get(),
8002 route_channel = target.channel,
8003 error = %err,
8004 "route lifecycle control PUSH was not delivered to client; closing target connection"
8005 );
8006 let _ = forwarding.escalate_client_delivery_failure(
8007 target.connection_id,
8008 target.channel,
8009 target.epoch,
8010 CloseReason::new(
8011 "route_lifecycle_push_delivery_failed",
8012 format!(
8013 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8014 target.channel
8015 ),
8016 ),
8017 crate::forwarding::UndeliveredFrame {
8018 module_id: target.module_id.as_deref(),
8019 sink: &target.sink,
8020 },
8021 );
8022 }
8023 }
8024 }
8025}
8026
8027#[cfg(test)]
8028mod tests {
8029 mod activity_read {
8030 include!("activity_read_tests.rs");
8031 }
8032
8033 mod lifecycle_activity {
8034 include!("control_lifecycle_tests.rs");
8035 }
8036 #[cfg(unix)]
8037 #[tokio::test]
8038 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8039 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8040 let path = dir.join("subc.jsonc");
8041 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8042 "program":"/bin/sleep","args":["60"],"protocol":"none",
8043 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8044 }}}).to_string()).unwrap();
8045 let mut configured = crate::daemon_config::load(&path)
8046 .unwrap()
8047 .unwrap()
8048 .modules
8049 .pop()
8050 .unwrap();
8051 let registry = std::sync::Arc::new(crate::Registry::default());
8052 let handle = crate::SupervisorHandle::new();
8053 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8054 .with_handle(handle.clone());
8055 let module = supervisor
8056 .supervise_configured_with_health(
8057 configured.module_spec(),
8058 true,
8059 configured.health.clone(),
8060 None,
8061 configured.restart,
8062 )
8063 .unwrap();
8064 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8065 let before = module.status().unwrap().pid;
8066 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8067 configured.health.cadence = std::time::Duration::from_secs(3600);
8068 let health_only = handler
8069 .reconcile_supervised_modules(
8070 &supervisor,
8071 vec![configured.clone()],
8072 false,
8073 Principal::Direct,
8074 )
8075 .await
8076 .unwrap();
8077 assert!(
8078 health_only.0.changed_pending_reload.is_empty(),
8079 "health policy is already applied live"
8080 );
8081 assert_eq!(module.status().unwrap().pid, before);
8082 assert_eq!(
8083 module.configuration().unwrap().1.http,
8084 configured.health.http
8085 );
8086 configured.args = vec!["61".into()];
8087 let launch = handler
8088 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8089 .await
8090 .unwrap();
8091 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8092 assert_eq!(
8093 module.status().unwrap().pid,
8094 before,
8095 "a launch edit is stored until reload"
8096 );
8097 module.drain().await.unwrap();
8098 }
8099 use cortexkit_test_support::ScratchDir;
8100 use std::{
8101 collections::BTreeMap,
8102 fmt,
8103 path::PathBuf,
8104 sync::{Arc, Mutex},
8105 time::Duration,
8106 };
8107
8108 use serde_json::{json, Value};
8109 use subc_protocol::{
8110 manifest::{
8111 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8112 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8113 },
8114 session::HealthStatus,
8115 FrameType,
8116 };
8117
8118 use super::*;
8119 use crate::{
8120 forwarding::{DataRoute, DataRouteState},
8121 registry::ChannelState,
8122 router::FrameSink,
8123 stderr_tail::DEFAULT_MAX_LINE_BYTES,
8124 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8125 RouteCtx, Router,
8126 };
8127 use tokio::{
8128 sync::mpsc,
8129 time::{sleep, Instant},
8130 };
8131 use tracing::{
8132 field::{Field, Visit},
8133 Event, Subscriber,
8134 };
8135 use tracing_subscriber::{layer::Context, prelude::*, Layer};
8136
8137 fn fake_aft_stub_path() -> PathBuf {
8156 let mut path = std::env::current_exe().expect("current_exe available in tests");
8157 path.pop(); path.pop(); path.push(if cfg!(windows) {
8160 "fake-aft-stub.exe"
8161 } else {
8162 "fake-aft-stub"
8163 });
8164 assert!(
8165 path.exists(),
8166 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8167 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8168 path.display()
8169 );
8170 path
8171 }
8172
8173 fn client_retries(code: &str) -> bool {
8178 subc_protocol::error_codes::is_retryable_route_open(code)
8179 }
8180
8181 #[test]
8192 fn retryability_of_forwarding_codes_matches_the_failure() {
8193 let transient = [
8196 ForwardingError::NoModuleConnection,
8197 ForwardingError::ModuleReloading {
8198 module_id: "m".into(),
8199 },
8200 ForwardingError::StaleModuleEndpoint,
8201 ForwardingError::UnknownReservation {
8202 client_channel: 1,
8203 module_channel: 1,
8204 },
8205 ForwardingError::ConnectionClosing {
8206 connection_id: ConnectionId::new(1),
8207 },
8208 ForwardingError::ClientEgressClosed {
8209 connection_id: ConnectionId::new(1),
8210 },
8211 ForwardingError::ModuleEgressUnavailable {
8212 connection_id: ConnectionId::new(1),
8213 },
8214 ];
8215 for err in transient {
8216 let code = forwarding_error_code(&err);
8217 assert!(
8218 client_retries(code),
8219 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8220 );
8221 }
8222
8223 let permanent = [
8228 ForwardingError::ConnectionRoleConflict {
8229 connection_id: ConnectionId::new(1),
8230 },
8231 ForwardingError::ClientRouteChannelExhausted {
8232 connection_id: ConnectionId::new(1),
8233 },
8234 ForwardingError::ModuleRouteChannelExhausted {
8235 endpoint: ModuleEndpointId {
8236 connection_id: ConnectionId::new(1),
8237 generation: 1,
8238 },
8239 },
8240 ForwardingError::RelayCorrelationExhausted,
8241 ForwardingError::RouteOpenBuild("x".into()),
8242 ForwardingError::Poisoned,
8243 ];
8244 for err in permanent {
8245 let code = forwarding_error_code(&err);
8246 assert!(
8247 !client_retries(code),
8248 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8249 );
8250 }
8251 }
8252
8253 #[tokio::test]
8281 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8282 let handler = ControlHandler::default();
8283 let frame =
8284 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8285
8286 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8290 assert_eq!(
8291 stamped,
8292 Principal::Direct,
8293 "a caller that proved nothing must not be stamped as a supervised module"
8294 );
8295
8296 let forged = handler
8301 .route_open_principal(
8302 &frame,
8303 Some(ConsumerIdentity {
8304 module_id: "aft".to_string(),
8305 launch_nonce: "not-a-real-nonce".to_string(),
8306 }),
8307 )
8308 .unwrap();
8309 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8310 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8311 }
8312
8313 #[test]
8330 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8331 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8332 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8333 let ClientControlRequest::RouteOpen {
8334 consumer_identity, ..
8335 } = parsed
8336 else {
8337 panic!("route.open body must parse as RouteOpen");
8338 };
8339 assert_eq!(
8340 consumer_identity,
8341 Some(ConsumerIdentity {
8342 module_id: "aft".to_string(),
8343 launch_nonce: "n".to_string(),
8344 }),
8345 "the wire field name must reach the value route_open_principal reads"
8346 );
8347 }
8348
8349 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8350 ModuleManifest::builder(module_id, "0.1.0")
8351 .protocol_ver(protocol_ver)
8352 .provides(vec![ProviderRole::ToolProvider {
8353 tools: vec![Tool {
8354 name: "read".to_string(),
8355 description: None,
8356 execution_mode: ExecutionMode::Pure,
8357 schema: json!({"type": "object"}),
8358 }],
8359 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8360 concurrency: Concurrency::ModuleManaged,
8361 emits_push: true,
8362 sub_supervises: true,
8363 }])
8364 .build()
8365 }
8366
8367 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8368 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8369 }
8370
8371 fn hello_frame_with_control_ops(
8372 module_id: &str,
8373 protocol_ver: u8,
8374 corr: u64,
8375 control_ops: Option<Vec<String>>,
8376 ) -> Frame {
8377 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8378 }
8379
8380 fn hello_frame_with_nonce(
8381 module_id: &str,
8382 protocol_ver: u8,
8383 corr: u64,
8384 launch_nonce: Option<&str>,
8385 ) -> Frame {
8386 hello_frame_full(
8387 module_id,
8388 protocol_ver,
8389 corr,
8390 None,
8391 launch_nonce.map(ToOwned::to_owned),
8392 )
8393 }
8394
8395 fn hello_frame_full(
8396 module_id: &str,
8397 protocol_ver: u8,
8398 corr: u64,
8399 control_ops: Option<Vec<String>>,
8400 launch_nonce: Option<String>,
8401 ) -> Frame {
8402 let body = serde_json::to_vec(&ModuleHelloBody {
8403 manifest: manifest(module_id, protocol_ver),
8404 protocol_ver,
8405 control_ops,
8406 launch_nonce,
8407 })
8408 .unwrap();
8409 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8410 }
8411
8412 fn hello_frame_with_rich_ui(
8413 module_id: &str,
8414 protocol_ver: u8,
8415 corr: u64,
8416 launch_nonce: Option<&str>,
8417 rich_ui: Value,
8418 ) -> Frame {
8419 let mut body = serde_json::to_value(ModuleHelloBody {
8420 manifest: manifest(module_id, protocol_ver),
8421 protocol_ver,
8422 control_ops: None,
8423 launch_nonce: launch_nonce.map(str::to_string),
8424 })
8425 .unwrap();
8426 body["manifest"]["rich_ui"] = rich_ui;
8427 Frame::build(
8428 FrameType::Hello,
8429 control_flags(),
8430 0,
8431 0,
8432 corr,
8433 serde_json::to_vec(&body).unwrap(),
8434 )
8435 .unwrap()
8436 }
8437
8438 fn non_routable_hello_frame_with_control_ops(
8439 module_id: &str,
8440 corr: u64,
8441 control_ops: Option<Vec<String>>,
8442 ) -> Frame {
8443 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8444 manifest.provides.clear();
8445 let body = serde_json::to_vec(&ModuleHelloBody {
8446 manifest,
8447 protocol_ver: PROTOCOL_VERSION,
8448 control_ops,
8449 launch_nonce: None,
8450 })
8451 .unwrap();
8452 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8453 }
8454
8455 fn capability_grammar_hello_frame(
8456 capabilities: Value,
8457 runtime_computed: Option<Value>,
8458 corr: u64,
8459 ) -> Frame {
8460 let mut body = serde_json::to_value(ModuleHelloBody {
8461 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8462 protocol_ver: PROTOCOL_VERSION,
8463 control_ops: None,
8464 launch_nonce: None,
8465 })
8466 .expect("HELLO body serializes");
8467 body["manifest"]["capabilities"] = capabilities;
8468 if let Some(runtime_computed) = runtime_computed {
8469 body["runtime_computed"] = runtime_computed;
8470 }
8471 Frame::build(
8472 FrameType::Hello,
8473 control_flags(),
8474 0,
8475 0,
8476 corr,
8477 serde_json::to_vec(&body).expect("HELLO body reserializes"),
8478 )
8479 .expect("HELLO frame builds")
8480 }
8481
8482 fn channel_request(channel: u16, corr: u64) -> Frame {
8483 Frame::build(
8484 FrameType::Request,
8485 Flags::new(true, Priority::Interactive, false),
8486 channel,
8487 0,
8488 corr,
8489 b"opaque".to_vec(),
8490 )
8491 .unwrap()
8492 }
8493
8494 fn route_ctx(
8495 connection_id: ConnectionId,
8496 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8497 let (tx, rx) = mpsc::channel(8);
8498 (
8499 RouteCtx {
8500 connection_id,
8501 egress: FrameSink::new(tx),
8502 },
8503 rx,
8504 )
8505 }
8506
8507 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8508 serde_json::from_slice(&frame.body).unwrap()
8509 }
8510
8511 async fn hello_via_sink(
8516 handler: &ControlHandler,
8517 ctx: &RouteCtx,
8518 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8519 hello: Frame,
8520 ) -> Frame {
8521 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8522 assert!(
8523 replies.is_empty(),
8524 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8525 );
8526 let ack = rx
8527 .try_recv()
8528 .expect("HELLO_ACK is queued on the module sink")
8529 .frame;
8530 assert_eq!(ack.header.ty, FrameType::HelloAck);
8531 ack
8532 }
8533
8534 fn parse_error(frame: &Frame) -> Value {
8535 serde_json::from_slice(&frame.body).unwrap()
8536 }
8537
8538 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8539 serde_json::from_slice(&frame.body).unwrap()
8540 }
8541
8542 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8543 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8544 route_channel,
8545 route_epoch: 0,
8546 kind,
8547 })
8548 .unwrap();
8549 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8550 }
8551
8552 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8553 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8554 module_id: module_id.to_string(),
8555 })
8556 .unwrap();
8557 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8558 }
8559
8560 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8561 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8562 }
8563
8564 fn route_open_frame_with_consumer_capabilities(
8565 corr: u64,
8566 module_id: &str,
8567 project_root: ScratchDir,
8568 consumer_capabilities: Option<Vec<String>>,
8569 ) -> Frame {
8570 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8571 target: RouteTarget::ToolProvider {
8572 module_id: module_id.to_string(),
8573 },
8574 identity: BindIdentity::new(
8575 project_root.path().to_path_buf(),
8576 "unit".to_string(),
8577 "session".to_string(),
8578 ),
8579 consumer_identity: None,
8580 consumer_capabilities,
8581 role_versions: None,
8582 admission_facts: None,
8583 scope: None,
8584 })
8585 .unwrap();
8586 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8587 }
8588
8589 fn route_open_frame_with_role_versions(
8590 corr: u64,
8591 module_id: &str,
8592 project_root: ScratchDir,
8593 role_versions: Option<BTreeMap<String, String>>,
8594 ) -> Frame {
8595 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8596 target: RouteTarget::ToolProvider {
8597 module_id: module_id.to_string(),
8598 },
8599 identity: BindIdentity::new(
8600 project_root.path().to_path_buf(),
8601 "unit".to_string(),
8602 format!("session-{corr}"),
8603 ),
8604 consumer_identity: None,
8605 consumer_capabilities: None,
8606 role_versions,
8607 admission_facts: None,
8608 scope: None,
8609 })
8610 .unwrap();
8611 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8612 }
8613
8614 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8615 entries
8616 .iter()
8617 .map(|(role, version)| (role.to_string(), version.to_string()))
8618 .collect()
8619 }
8620
8621 fn route_open_frame_with_admission_facts(
8622 corr: u64,
8623 module_id: &str,
8624 project_root: ScratchDir,
8625 consumer_identity: Option<subc_control::ConsumerIdentity>,
8626 facts: Option<Value>,
8627 ) -> Frame {
8628 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8629 target: RouteTarget::ToolProvider {
8630 module_id: module_id.to_string(),
8631 },
8632 identity: BindIdentity::new(
8633 project_root.path().to_path_buf(),
8634 "unit".to_string(),
8635 format!("session-{corr}"),
8636 ),
8637 consumer_identity,
8638 consumer_capabilities: None,
8639 role_versions: None,
8640 admission_facts: facts,
8641 scope: None,
8642 })
8643 .unwrap();
8644 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8645 }
8646
8647 #[derive(Clone, Default)]
8648 struct EventCapture {
8649 events: Arc<Mutex<Vec<CapturedEvent>>>,
8650 }
8651
8652 #[derive(Clone, Debug)]
8653 struct CapturedEvent {
8654 target: String,
8655 level: tracing::Level,
8656 fields: BTreeMap<String, String>,
8657 }
8658
8659 impl EventCapture {
8660 fn events(&self) -> Vec<CapturedEvent> {
8661 self.events.lock().unwrap().clone()
8662 }
8663 }
8664
8665 impl<S> Layer<S> for EventCapture
8666 where
8667 S: Subscriber,
8668 {
8669 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8670 let mut visitor = EventFieldVisitor::default();
8671 event.record(&mut visitor);
8672 self.events.lock().unwrap().push(CapturedEvent {
8673 target: event.metadata().target().to_string(),
8674 level: *event.metadata().level(),
8675 fields: visitor.fields,
8676 });
8677 }
8678 }
8679
8680 #[derive(Default)]
8681 struct EventFieldVisitor {
8682 fields: BTreeMap<String, String>,
8683 }
8684
8685 impl Visit for EventFieldVisitor {
8686 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8687 self.fields
8688 .insert(field.name().to_string(), format!("{value:?}"));
8689 }
8690 }
8691
8692 fn health_response(corr: u64, status: HealthStatus) -> Frame {
8693 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8694 status,
8695 detail: Some("warming".to_string()),
8696 metrics: Some(json!({"queue_depth": 3})),
8697 })
8698 .unwrap();
8699 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8700 }
8701
8702 fn route_bind_ack(corr: u64) -> Frame {
8703 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8704 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8705 }
8706
8707 fn unique_project_root(label: &str) -> ScratchDir {
8708 ScratchDir::new(label)
8709 }
8710
8711 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8712 match parse_route_poll(frame) {
8713 ClientControlResponse::RoutePoll {
8714 status: None,
8715 live: Some(live),
8716 ..
8717 } => assert_eq!(live, expected_live),
8718 other => panic!("unexpected route.poll response: {other:?}"),
8719 }
8720 }
8721
8722 fn bind_liveness_route(
8723 registry: &Registry,
8724 forwarding: &ForwardingTable,
8725 module_id: &str,
8726 ) -> (RouteCtx, u16, u32) {
8727 let module_connection = ConnectionId::new(101);
8728 let client_connection = ConnectionId::new(202);
8729 let registration = registry
8730 .register_with_control_ops(
8731 manifest(module_id, PROTOCOL_VERSION),
8732 PROTOCOL_VERSION,
8733 module_connection,
8734 module_baseline_control_ops(),
8735 )
8736 .unwrap();
8737 let (module_tx, _module_rx) = mpsc::channel(8);
8738 let endpoint = forwarding
8739 .register_module_connection(
8740 module_connection,
8741 module_id.to_string(),
8742 PROTOCOL_VERSION,
8743 manifest_concurrency(®istration.manifest),
8744 FrameSink::new(module_tx),
8745 )
8746 .unwrap();
8747 let (client_ctx, _client_rx) = route_ctx(client_connection);
8748 let pending = forwarding
8749 .begin_route_bind_relay_for_test(
8750 client_connection,
8751 client_ctx.egress.clone(),
8752 1,
8753 module_id,
8754 )
8755 .unwrap();
8756 assert_eq!(pending.endpoint, endpoint);
8757 let route_channel = pending.client_channel;
8758 let route_epoch = pending.client_epoch;
8759 forwarding
8760 .complete_pending_relay(
8761 module_connection,
8762 pending.corr,
8763 RouteBindRelayOutcome::Accepted,
8764 )
8765 .unwrap();
8766 (client_ctx, route_channel, route_epoch)
8767 }
8768
8769 struct FakeProcessLiveness {
8770 live: Option<bool>,
8771 }
8772
8773 impl ModuleProcessLiveness for FakeProcessLiveness {
8774 fn process_live(&self, _module_id: &str) -> Option<bool> {
8775 self.live
8776 }
8777 }
8778
8779 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8780 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8781 {
8782 let registry = Arc::new(Registry::default());
8783 let supervisor_handle = SupervisorHandle::new();
8784 let supervisor = Supervisor::new_for_test(
8785 Arc::clone(®istry),
8786 RestartPolicy::new(1, Duration::from_millis(10)),
8787 )
8788 .with_handle(supervisor_handle.clone());
8789 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8790 let module = supervisor
8791 .spawn(ModuleSpec {
8792 module_id: "stderr-tail-wire".to_string(),
8793 program: fake_aft_stub_path(),
8794 args: Vec::new(),
8795 env: vec![
8796 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8797 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8798 ],
8799 reserved: false,
8800 reserved_prefixes: Vec::new(),
8801 protocol: ModuleProtocol::Subc,
8802 overlap: Default::default(),
8803 })
8804 .unwrap();
8805
8806 let deadline = Instant::now() + Duration::from_secs(5);
8807 loop {
8808 let tail = module.stderr_tail(None, None);
8809 if tail
8810 .entries
8811 .iter()
8812 .any(|entry| matches!(entry, TailEntry::ProcessStart))
8813 && tail.entries.iter().any(|entry| {
8814 matches!(
8815 entry,
8816 TailEntry::Line {
8817 truncated: true,
8818 ..
8819 }
8820 )
8821 })
8822 {
8823 break;
8824 }
8825 assert!(
8826 Instant::now() < deadline,
8827 "module did not produce a truncated line and restart boundary: {tail:?}"
8828 );
8829 sleep(Duration::from_millis(10)).await;
8830 }
8831
8832 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8833 let request = ClientControlRequest::SupervisorStderrTail {
8834 module_id: "stderr-tail-wire".to_string(),
8835 max_lines: None,
8836 max_bytes: None,
8837 };
8838 let frame = Frame::build(
8839 FrameType::Request,
8840 control_flags(),
8841 0,
8842 0,
8843 1,
8844 serde_json::to_vec(&request).unwrap(),
8845 )
8846 .unwrap();
8847 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8848 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8849 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8850 serde_json::from_slice(&responses[0].body).unwrap()
8851 else {
8852 panic!("expected supervisor.stderr_tail response");
8853 };
8854
8855 assert!(
8856 tail.entries
8857 .iter()
8858 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8859 "the control response lost the restart boundary"
8860 );
8861 let Some(StderrTailEntry::Line {
8862 text,
8863 truncated,
8864 at_ms,
8865 }) = tail.entries.iter().find(|entry| {
8866 matches!(
8867 entry,
8868 StderrTailEntry::Line {
8869 truncated: true,
8870 ..
8871 }
8872 )
8873 })
8874 else {
8875 panic!("the control response lost the truncated line");
8876 };
8877 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8878 assert!(*truncated);
8879 assert!(
8880 at_ms.is_some(),
8881 "the control response lost the line's capture time"
8882 );
8883 }
8884
8885 #[tokio::test(flavor = "current_thread")]
8890 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8891 let dir = ScratchDir::new("terminals-off-worker");
8892 let journal_path = dir.join("terminals.jsonl");
8893 let registry = Arc::new(Registry::default());
8894 let supervisor_handle = SupervisorHandle::new();
8895 let supervisor =
8896 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8897 .with_handle(supervisor_handle.clone())
8898 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8899 let module = supervisor
8900 .spawn(ModuleSpec {
8901 module_id: "terminal-off-worker".to_string(),
8902 program: fake_aft_stub_path(),
8903 args: Vec::new(),
8904 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8905 reserved: false,
8906 reserved_prefixes: Vec::new(),
8907 protocol: ModuleProtocol::Subc,
8908 overlap: Default::default(),
8909 })
8910 .unwrap();
8911 let deadline = Instant::now() + Duration::from_secs(5);
8912 while module.terminal_history().entries.len() != 2 {
8913 assert!(Instant::now() < deadline, "module did not record two exits");
8914 sleep(Duration::from_millis(10)).await;
8915 }
8916
8917 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8918 let handler =
8919 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
8920 let frame = Frame::build(
8921 FrameType::Request,
8922 control_flags(),
8923 0,
8924 0,
8925 1,
8926 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8927 module_id: "terminal-off-worker".to_string(),
8928 })
8929 .unwrap(),
8930 )
8931 .unwrap();
8932 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8933 let spawned_at = std::time::Instant::now();
8934 let read = tokio::spawn({
8935 let handler = Arc::clone(&handler);
8936 async move { handler.handle_control_frame(&ctx, frame).await }
8937 });
8938 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8941 .await
8942 .unwrap()
8943 .expect("the history read reached its pause");
8944 let elapsed = spawned_at.elapsed();
8945 assert!(
8946 elapsed < Duration::from_secs(2) && !read.is_finished(),
8947 "this task could not run while the history read was paused \
8948 (resumed after {elapsed:?}, read finished: {})",
8949 read.is_finished()
8950 );
8951
8952 drop(release);
8953 let responses = read.await.unwrap().unwrap();
8954 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8955 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8956 panic!("expected supervisor.terminals response");
8957 };
8958 assert_eq!(terminals.entries.len(), 2);
8959 assert_eq!(terminals.journal_skipped_lines, 0);
8960 assert_eq!(terminals.journal_read_errors, 0);
8961 }
8962
8963 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8964 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8965 let registry = Arc::new(Registry::default());
8966 let supervisor_handle = SupervisorHandle::new();
8967 let supervisor =
8968 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8969 .with_handle(supervisor_handle.clone());
8970 let module = supervisor
8971 .spawn(ModuleSpec {
8972 module_id: "terminal-golden".to_string(),
8973 program: fake_aft_stub_path(),
8974 args: Vec::new(),
8975 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8976 reserved: false,
8977 reserved_prefixes: Vec::new(),
8978 protocol: ModuleProtocol::Subc,
8979 overlap: Default::default(),
8980 })
8981 .unwrap();
8982
8983 let deadline = Instant::now() + Duration::from_secs(5);
8984 while module.terminal_history().entries.len() != 2 {
8985 assert!(
8986 Instant::now() < deadline,
8987 "module did not retain two terminal exits: {:?}",
8988 module.terminal_history()
8989 );
8990 sleep(Duration::from_millis(10)).await;
8991 }
8992
8993 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8994 let request = ClientControlRequest::SupervisorTerminals {
8995 module_id: "terminal-golden".to_string(),
8996 };
8997 let frame = Frame::build(
8998 FrameType::Request,
8999 control_flags(),
9000 0,
9001 0,
9002 1,
9003 serde_json::to_vec(&request).unwrap(),
9004 )
9005 .unwrap();
9006 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9007 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9008 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9009 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9010 panic!("expected supervisor.terminals response");
9011 };
9012 assert_eq!(terminals.entries.len(), 2);
9013 assert_eq!(terminals.dropped, 0);
9014
9015 let mut rendered = serde_json::to_value(response).unwrap();
9016 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9019 for (index, entry) in rendered["entries"]
9020 .as_array_mut()
9021 .expect("terminal response entries array")
9022 .iter_mut()
9023 .enumerate()
9024 {
9025 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9026 }
9027
9028 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9029 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9030 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9031 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9032 std::fs::write(&golden_path, &serialized).unwrap();
9033 }
9034 let expected: Value =
9035 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9036 assert_eq!(rendered, expected);
9037 }
9038
9039 #[test]
9040 fn hello_registers_manifest_and_returns_ack() {
9041 let registry = Arc::new(Registry::default());
9042 let handler = ControlHandler::new(Arc::clone(®istry));
9043 let conn = ConnectionId::new(1);
9044
9045 let responses = handler
9046 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9047 .unwrap();
9048
9049 assert_eq!(responses.len(), 1);
9050 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9051 assert_eq!(responses[0].header.channel, 0);
9052 assert_eq!(responses[0].header.corr, 7);
9053 let ack = parse_ack(&responses[0]);
9054 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9055 assert!(ack
9056 .subc_capabilities
9057 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9058 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9059 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9060 assert!(ack
9061 .subc_ops
9062 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9063 assert!(ack
9064 .subc_ops
9065 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9066
9067 let registration = registry.get_module("aft").unwrap().unwrap();
9068 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9069 assert_eq!(registration.state, ChannelState::Active);
9070 assert_eq!(registration.connection_id, conn);
9071 assert_eq!(registration.control_ops, module_baseline_control_ops());
9072 }
9073
9074 #[test]
9075 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9076 let invalid_identifiers = [
9077 ("case_change", "credentials-Provider/v1"),
9078 ("leading_zero", "credentials-provider/v01"),
9079 ("trailing_hyphen", "credentials-provider-/v1"),
9080 ("consecutive_hyphens", "credentials--provider/v1"),
9081 ("uppercase", "Credentials-provider/v1"),
9082 ("missing_v", "credentials-provider/1"),
9083 ("whitespace", "credentials provider/v1"),
9084 ("zero_version", "credentials-provider/v0"),
9085 ("out_of_range_version", "credentials-provider/v4294967296"),
9086 (
9087 "overlength_name",
9088 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9089 ),
9090 ];
9091 let mut cases = invalid_identifiers
9092 .into_iter()
9093 .map(|(name, identifier)| {
9094 (
9095 format!("identifier_{name}"),
9096 "capabilities.provides[0]".to_string(),
9097 identifier.to_string(),
9098 json!({ "provides": [identifier] }),
9099 None,
9100 )
9101 })
9102 .collect::<Vec<_>>();
9103 cases.extend([
9104 (
9105 "unknown_need".to_string(),
9106 "capabilities.requires[0].need".to_string(),
9107 "deferred".to_string(),
9108 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9109 None,
9110 ),
9111 (
9112 "duplicate_provides".to_string(),
9113 "capabilities.provides[1]".to_string(),
9114 "credentials-provider/v1".to_string(),
9115 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9116 None,
9117 ),
9118 (
9119 "duplicate_must_never_reach".to_string(),
9120 "capabilities.must_never_reach[1]".to_string(),
9121 "credentials-provider/v1".to_string(),
9122 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9123 None,
9124 ),
9125 (
9126 "duplicate_requires_same_need".to_string(),
9127 "capabilities.requires[1]".to_string(),
9128 "credentials-provider/v1".to_string(),
9129 json!({ "requires": [
9130 { "capability": "credentials-provider/v1", "need": "required" },
9131 { "capability": "credentials-provider/v1", "need": "required" }
9132 ] }),
9133 None,
9134 ),
9135 (
9136 "duplicate_requires_conflicting_need".to_string(),
9137 "capabilities.requires[1]".to_string(),
9138 "credentials-provider/v1".to_string(),
9139 json!({ "requires": [
9140 { "capability": "credentials-provider/v1", "need": "required" },
9141 { "capability": "credentials-provider/v1", "need": "optional" }
9142 ] }),
9143 None,
9144 ),
9145 (
9146 "capabilities_root_pointer".to_string(),
9147 "runtime_computed[0]".to_string(),
9148 "/capabilities".to_string(),
9149 json!({}),
9150 Some(json!(["/capabilities"])),
9151 ),
9152 (
9153 "capabilities_descendant_pointer".to_string(),
9154 "runtime_computed[0]".to_string(),
9155 "/capabilities/provides".to_string(),
9156 json!({}),
9157 Some(json!(["/capabilities/provides"])),
9158 ),
9159 (
9160 "malformed_pointer_without_leading_slash".to_string(),
9161 "runtime_computed[0]".to_string(),
9162 "capabilities".to_string(),
9163 json!({}),
9164 Some(json!(["capabilities"])),
9165 ),
9166 (
9167 "malformed_pointer_escape".to_string(),
9168 "runtime_computed[0]".to_string(),
9169 "/roles/~2/tools".to_string(),
9170 json!({}),
9171 Some(json!(["/roles/~2/tools"])),
9172 ),
9173 (
9174 "unknown_capabilities_field".to_string(),
9175 "capabilities.future".to_string(),
9176 "<array>".to_string(),
9177 json!({ "future": [] }),
9178 None,
9179 ),
9180 ]);
9181
9182 for (index, (name, field, value, capabilities, runtime_computed)) in
9183 cases.into_iter().enumerate()
9184 {
9185 let registry = Arc::new(Registry::default());
9186 let handler = ControlHandler::new(Arc::clone(®istry));
9187 let response = handler
9188 .handle_control(
9189 ConnectionId::new((index + 1) as u64),
9190 capability_grammar_hello_frame(
9191 capabilities,
9192 runtime_computed,
9193 index as u64 + 1,
9194 ),
9195 )
9196 .expect("invalid HELLO returns a refusal");
9197
9198 assert_eq!(response.len(), 1, "{name} must emit one refusal");
9199 let error = parse_error(&response[0]);
9200 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9201 let message = error["message"]
9202 .as_str()
9203 .expect("error message is a string");
9204 assert!(
9205 message.contains(&field),
9206 "{name}: field missing from {message}"
9207 );
9208 assert!(
9209 message.contains(&value),
9210 "{name}: value missing from {message}"
9211 );
9212 assert_eq!(
9213 registry
9214 .active_registration_count()
9215 .expect("registry reads"),
9216 0,
9217 "{name}: refused HELLO must not create a catalog entry"
9218 );
9219 }
9220 }
9221
9222 #[test]
9223 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9224 let registry = Arc::new(Registry::default());
9225 let handler = ControlHandler::new(Arc::clone(®istry));
9226 let capabilities = json!({
9227 "provides": ["credentials-provider/v1"],
9228 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9229 "must_never_reach": ["federation-transport/v1"]
9230 });
9231 let response = handler
9232 .handle_control(
9233 ConnectionId::new(99),
9234 capability_grammar_hello_frame(
9235 capabilities.clone(),
9236 Some(json!(["/roles/0/tools"])),
9237 99,
9238 ),
9239 )
9240 .expect("valid HELLO registers");
9241 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9242
9243 let request = Frame::build(
9244 FrameType::Request,
9245 control_flags(),
9246 0,
9247 0,
9248 100,
9249 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9250 .expect("catalog request serializes"),
9251 )
9252 .expect("catalog request frame builds");
9253 let response = handler
9254 .handle_catalog_list(request, None)
9255 .expect("catalog list succeeds");
9256 let ClientControlResponse::CatalogList { modules, .. } =
9257 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9258 else {
9259 panic!("catalog request must return catalog.list");
9260 };
9261 assert_eq!(modules.len(), 1);
9262 assert_eq!(
9263 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9264 capabilities
9265 );
9266 }
9267
9268 #[test]
9269 fn catalog_list_mirrors_management_operation_description() {
9270 let registry = Arc::new(Registry::default());
9271 let handler = ControlHandler::new(Arc::clone(®istry));
9272 let description = "List managed records and return their identifiers and metadata.";
9273 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9274 manifest.provides = vec![ProviderRole::ManagementSurface {
9275 operations: vec![ManagementOperation {
9276 name: "records.list".to_string(),
9277 kind: ManagementOperationKind::Query,
9278 description: Some(description.to_string()),
9279 }],
9280 config_schema: json!({"type": "object"}),
9281 observability: vec![ObservabilitySurface {
9282 name: "records.stats".to_string(),
9283 kind: ObservabilityKind::Snapshot,
9284 }],
9285 identity_scope: vec![IdentityScope::Project],
9286 concurrency: Concurrency::ModuleManaged,
9287 }];
9288 registry
9289 .register_with_control_ops(
9290 manifest,
9291 PROTOCOL_VERSION,
9292 ConnectionId::new(99),
9293 Vec::new(),
9294 )
9295 .expect("described management manifest registers");
9296
9297 let request = Frame::build(
9298 FrameType::Request,
9299 control_flags(),
9300 0,
9301 0,
9302 100,
9303 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9304 .expect("catalog request serializes"),
9305 )
9306 .expect("catalog request frame builds");
9307 let response = handler
9308 .handle_catalog_list(request, None)
9309 .expect("catalog list succeeds");
9310 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9311 assert_eq!(
9312 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9313 "catalog.list must preserve the declared operation description verbatim"
9314 );
9315 }
9316
9317 #[test]
9318 fn catalog_list_mirrors_registered_operation_delivery() {
9319 let registry = Arc::new(Registry::default());
9320 let handler = ControlHandler::new(Arc::clone(®istry));
9321 let delivery =
9322 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9323 let mut body = serde_json::to_value(ModuleHelloBody {
9324 manifest: manifest("watch-management", PROTOCOL_VERSION),
9325 protocol_ver: PROTOCOL_VERSION,
9326 control_ops: None,
9327 launch_nonce: None,
9328 })
9329 .unwrap();
9330 body["manifest"]["provides"] = json!([{
9331 "role": "management_surface",
9332 "operations": [
9333 {"name": "status", "kind": "query"},
9334 {"name": "future", "kind": "query"},
9335 {"name": "reset", "kind": "mutate"}
9336 ],
9337 "config_schema": {}, "observability": [], "identity_scope": []
9338 }]);
9339 body["manifest"]["operation_delivery"] = delivery.clone();
9340 let hello = Frame::build(
9341 FrameType::Hello,
9342 control_flags(),
9343 0,
9344 0,
9345 99,
9346 serde_json::to_vec(&body).unwrap(),
9347 )
9348 .unwrap();
9349 let response = handler
9350 .handle_control(ConnectionId::new(99), hello)
9351 .unwrap();
9352 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9353
9354 let (_, registrations) = registry.list_modules().unwrap();
9355 let registered = ®istrations[0].manifest;
9356 assert!(registered.is_watch_operation("status"));
9357 assert!(!registered.is_watch_operation("future"));
9358 assert!(!registered.is_watch_operation("reset"));
9359 assert!(!registered.is_watch_operation("missing"));
9360
9361 let request = Frame::build(
9362 FrameType::Request,
9363 control_flags(),
9364 0,
9365 0,
9366 100,
9367 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9368 )
9369 .unwrap();
9370 let response = handler.handle_catalog_list(request, None).unwrap();
9371 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9372 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9373 let ClientControlResponse::CatalogList { modules, .. } =
9374 serde_json::from_slice(&response[0].body).unwrap()
9375 else {
9376 panic!("expected catalog.list");
9377 };
9378 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9379 }
9380
9381 #[test]
9382 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
9383 let registry = Arc::new(Registry::default());
9384 let supervisor = SupervisorHandle::new();
9385 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9386 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9387 let descriptor = json!({
9388 "schema": "ckios.richui.manifest/1",
9389 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9390 "bytes": 4096,
9391 "chunks": 4
9392 });
9393 let connection = ConnectionId::new(201);
9394 let hello = handler
9395 .handle_control(
9396 connection,
9397 hello_frame_with_rich_ui(
9398 "vault",
9399 PROTOCOL_VERSION,
9400 201,
9401 Some("the-real-nonce"),
9402 descriptor.clone(),
9403 ),
9404 )
9405 .unwrap();
9406 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9407
9408 let request =
9409 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
9410 let provides = registry
9411 .get_module_by_connection(connection)
9412 .unwrap()
9413 .unwrap()
9414 .manifest
9415 .provides;
9416 handler
9417 .handle_catalog_update(connection, request, provides, None, Some(false))
9418 .unwrap();
9419
9420 let request = Frame::build(
9421 FrameType::Request,
9422 control_flags(),
9423 0,
9424 0,
9425 203,
9426 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9427 )
9428 .unwrap();
9429 let response = handler.handle_catalog_list(request, None).unwrap();
9430 let ClientControlResponse::CatalogList { modules, .. } =
9431 serde_json::from_slice(&response[0].body).unwrap()
9432 else {
9433 panic!("expected catalog.list");
9434 };
9435 assert_eq!(
9436 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
9437 descriptor
9438 );
9439 assert!(modules[0].nonce_verified);
9440 }
9441
9442 fn hello_with_role_declarations(declarations: Value) -> Frame {
9443 let mut body = serde_json::to_value(ModuleHelloBody {
9444 manifest: manifest("role-test", PROTOCOL_VERSION),
9445 protocol_ver: PROTOCOL_VERSION,
9446 control_ops: None,
9447 launch_nonce: None,
9448 })
9449 .unwrap();
9450 body["manifest"]["role_declarations"] = declarations;
9451 Frame::build(
9452 FrameType::Hello,
9453 control_flags(),
9454 0,
9455 0,
9456 301,
9457 serde_json::to_vec(&body).unwrap(),
9458 )
9459 .unwrap()
9460 }
9461
9462 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
9463 let request = Frame::build(
9464 FrameType::Request,
9465 control_flags(),
9466 0,
9467 0,
9468 302,
9469 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9470 )
9471 .unwrap();
9472 let response = handler.handle_catalog_list(request, None).unwrap();
9473 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
9474 let ClientControlResponse::CatalogList { mut modules, .. } =
9475 serde_json::from_slice(&response[0].body).unwrap()
9476 else {
9477 panic!("expected catalog.list");
9478 };
9479 (raw["modules"][0].clone(), modules.remove(0))
9480 }
9481
9482 #[test]
9483 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
9484 let registry = Arc::new(Registry::default());
9485 let handler = ControlHandler::new(Arc::clone(®istry));
9486 let declarations = json!({
9487 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
9488 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
9489 });
9490 let connection = ConnectionId::new(301);
9491 let response = handler
9492 .handle_control(
9493 connection,
9494 hello_with_role_declarations(declarations.clone()),
9495 )
9496 .unwrap();
9497 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9498 let registered = registry
9499 .get_module_by_connection(connection)
9500 .unwrap()
9501 .unwrap()
9502 .manifest;
9503 assert_eq!(
9507 serde_json::to_value(®istered.role_declarations).unwrap(),
9508 declarations
9509 );
9510 let (raw, entry) = role_declarations_catalog(&handler);
9511 assert_eq!(raw["role_declarations"], declarations);
9512 assert_eq!(entry.role_declarations, registered.role_declarations);
9513
9514 let request =
9515 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
9516 handler
9517 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
9518 .unwrap();
9519 let (raw, entry) = role_declarations_catalog(&handler);
9520 assert_eq!(raw["role_declarations"], declarations);
9521 assert_eq!(entry.role_declarations, registered.role_declarations);
9522 }
9523
9524 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
9525 let registry = Arc::new(Registry::default());
9526 let handler = ControlHandler::new(Arc::clone(®istry));
9527 let capture = EventCapture::default();
9528 let _subscriber =
9529 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9530 let response = handler
9531 .handle_control(
9532 ConnectionId::new(301),
9533 hello_with_role_declarations(declarations),
9534 )
9535 .unwrap();
9536 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9537 assert!(registry
9538 .get_module_by_connection(ConnectionId::new(301))
9539 .unwrap()
9540 .unwrap()
9541 .manifest
9542 .role_declarations
9543 .is_empty());
9544 let (raw, entry) = role_declarations_catalog(&handler);
9545 assert!(raw.get("role_declarations").is_none());
9546 assert!(entry.role_declarations.is_empty());
9547 let warnings: Vec<_> = capture
9548 .events()
9549 .into_iter()
9550 .filter(|event| {
9551 event.fields.get("message").map(String::as_str)
9552 == Some("dropping whole role_declarations map from HELLO")
9553 })
9554 .collect();
9555 assert_eq!(warnings.len(), 1);
9556 assert_eq!(warnings[0].level, tracing::Level::WARN);
9557 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
9558 assert_eq!(warnings[0].fields["reason"], reason);
9559 }
9560
9561 #[test]
9562 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
9563 for value in [
9564 json!(null),
9565 json!(false),
9566 json!(0),
9567 json!("opaque"),
9568 json!([]),
9569 ] {
9570 assert_hello_drops_role_declarations_with_warning(
9571 json!({"routing/v1": {}, "broken/v1": value}),
9572 "role_declarations values must be objects",
9573 );
9574 }
9575 assert_hello_drops_role_declarations_with_warning(
9576 json!([]),
9577 "role_declarations must be an object",
9578 );
9579 }
9580
9581 #[test]
9582 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
9583 assert_hello_drops_role_declarations_with_warning(
9584 json!({"routing/v1": {}, "bad key": {}}),
9585 "role_declarations key must use capability-name grammar",
9586 );
9587 }
9588
9589 #[test]
9590 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
9591 assert_hello_drops_role_declarations_with_warning(
9592 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
9593 "role_declarations exceeds 16384 compact JSON bytes",
9594 );
9595 }
9596
9597 #[test]
9598 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
9599 let registry = Arc::new(Registry::default());
9600 let handler = ControlHandler::new(registry.clone());
9601 let descriptor = json!({
9602 "schema": "ckios.richui.manifest/1",
9603 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9604 "bytes": 128,
9605 "chunks": 2
9606 });
9607 let malformed = [
9608 ("schema", json!("")),
9609 ("schema", json!("a".repeat(65))),
9610 ("schema", json!("Bad schema")),
9611 (
9612 "manifest_digest",
9613 json!(format!("sha256:{}", "A".repeat(64))),
9614 ),
9615 ("manifest_digest", json!("sha256:abc")),
9616 ("bytes", json!(0)),
9617 ("bytes", json!(16_777_217)),
9618 ("bytes", json!("128")),
9619 ("chunks", json!(0)),
9620 ("chunks", json!(1025)),
9621 ("chunks", json!("2")),
9622 ];
9623 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
9624 let mut invalid = descriptor.clone();
9625 invalid[field] = invalid_value;
9626 let response = handler
9627 .handle_control(
9628 ConnectionId::new(210 + index as u64),
9629 hello_frame_with_rich_ui(
9630 &format!("bad-rich-ui-{index}"),
9631 PROTOCOL_VERSION,
9632 210 + index as u64,
9633 None,
9634 invalid,
9635 ),
9636 )
9637 .unwrap();
9638 assert_eq!(response[0].header.ty, FrameType::Error);
9639 let error = parse_error(&response[0]);
9640 assert_eq!(error["code"], "invalid_manifest");
9641 assert!(error["message"]
9642 .as_str()
9643 .unwrap()
9644 .contains(&format!("rich_ui.{field}")));
9645 assert!(registry
9646 .get_module(&format!("bad-rich-ui-{index}"))
9647 .unwrap()
9648 .is_none());
9649 }
9650 }
9651
9652 #[test]
9653 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
9654 let registry = Arc::new(Registry::default());
9655 let handler = ControlHandler::new(Arc::clone(®istry));
9656 let connection = ConnectionId::new(219);
9657 registry
9658 .register_with_control_ops(
9659 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
9660 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
9661 "bad schema",
9662 format!("sha256:{}", "a".repeat(64)),
9663 128,
9664 2,
9665 )))
9666 .build(),
9667 PROTOCOL_VERSION,
9668 connection,
9669 Vec::new(),
9670 )
9671 .unwrap();
9672 let request =
9673 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
9674 let response = handler
9675 .handle_catalog_update(connection, request, Vec::new(), None, None)
9676 .unwrap();
9677 assert_eq!(response[0].header.ty, FrameType::Error);
9678 let error = parse_error(&response[0]);
9679 assert_eq!(error["code"], "invalid_manifest");
9680 assert!(error["message"]
9681 .as_str()
9682 .unwrap()
9683 .contains("rich_ui.schema"));
9684 }
9685
9686 #[test]
9687 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
9688 let registry = Arc::new(Registry::default());
9689 let supervisor = SupervisorHandle::new();
9690 supervisor.open_swap("provider", "candidate-nonce".to_string());
9691 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9692 let connection = ConnectionId::new(218);
9693 let hello = handler
9694 .handle_control(
9695 connection,
9696 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
9697 )
9698 .unwrap();
9699 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9700 assert!(registry.list_modules().unwrap().1.is_empty());
9701
9702 registry.promote_candidate("provider").unwrap().unwrap();
9703 let request = Frame::build(
9704 FrameType::Request,
9705 control_flags(),
9706 0,
9707 0,
9708 219,
9709 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9710 )
9711 .unwrap();
9712 let response = handler.handle_catalog_list(request, None).unwrap();
9713 let ClientControlResponse::CatalogList { modules, .. } =
9714 serde_json::from_slice(&response[0].body).unwrap()
9715 else {
9716 panic!("expected catalog.list");
9717 };
9718 assert_eq!(modules.len(), 1);
9719 assert!(modules[0].nonce_verified);
9720 }
9721
9722 #[test]
9723 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
9724 let registry = Arc::new(Registry::default());
9725 let handler = ControlHandler::new(Arc::clone(®istry));
9726 let hello = Frame::build(
9727 FrameType::Hello,
9728 control_flags(),
9729 0,
9730 0,
9731 220,
9732 serde_json::to_vec(&ModuleHelloBody {
9733 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
9734 protocol_ver: PROTOCOL_VERSION,
9735 control_ops: None,
9736 launch_nonce: Some("unverified".to_string()),
9737 })
9738 .unwrap(),
9739 )
9740 .unwrap();
9741 assert_eq!(
9742 handler
9743 .handle_control(ConnectionId::new(220), hello)
9744 .unwrap()[0]
9745 .header
9746 .ty,
9747 FrameType::HelloAck
9748 );
9749 let request = Frame::build(
9750 FrameType::Request,
9751 control_flags(),
9752 0,
9753 0,
9754 221,
9755 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9756 )
9757 .unwrap();
9758 let response = handler.handle_catalog_list(request, None).unwrap();
9759 let ClientControlResponse::CatalogList { modules, .. } =
9760 serde_json::from_slice(&response[0].body).unwrap()
9761 else {
9762 panic!("expected catalog.list");
9763 };
9764 assert_eq!(
9765 serde_json::to_vec(&modules[0]).unwrap(),
9766 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
9767 );
9768 assert!(!modules[0].nonce_verified);
9769 }
9770
9771 #[test]
9772 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
9773 let registry = Arc::new(Registry::default());
9774 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9775 [("vault".to_string(), true), ("squatter".to_string(), true)],
9776 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9777 );
9778 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
9779 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9780 provides: vec!["credentials-provider/v1".to_string()],
9781 requires: Vec::new(),
9782 must_never_reach: Vec::new(),
9783 });
9784 let frame = Frame::build(
9785 FrameType::Hello,
9786 control_flags(),
9787 0,
9788 0,
9789 77,
9790 serde_json::to_vec(&ModuleHelloBody {
9791 manifest: squatter,
9792 protocol_ver: PROTOCOL_VERSION,
9793 control_ops: None,
9794 launch_nonce: None,
9795 })
9796 .expect("HELLO serializes"),
9797 )
9798 .expect("HELLO frame builds");
9799 let response = handler
9800 .handle_control(ConnectionId::new(77), frame)
9801 .expect("reserved claim receives a typed refusal");
9802 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9803 assert_eq!(
9804 registry
9805 .active_registration_count()
9806 .expect("registry reads"),
9807 0,
9808 "a reserved capability refusal must not leave a catalog entry"
9809 );
9810 }
9811
9812 #[test]
9813 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
9814 for settlement in ["timeout", "inconclusive", "drop"] {
9815 let breakers = RouteBindBreakers::default();
9816 let RouteBindAdmission::Admitted {
9817 guard: mut old,
9818 probe: false,
9819 } = breakers.admit("prov")
9820 else {
9821 panic!("ordinary relay admitted")
9822 };
9823 let RouteBindAdmission::Admitted {
9824 guard: mut opener, ..
9825 } = breakers.admit("prov")
9826 else {
9827 panic!("second relay admitted")
9828 };
9829 assert!(
9830 !opener
9831 .record_timeout(1, Duration::ZERO)
9832 .unwrap()
9833 .reopened_after_probe
9834 );
9835 let RouteBindAdmission::Admitted {
9836 guard: mut probe,
9837 probe: true,
9838 } = breakers.admit("prov")
9839 else {
9840 panic!("one cooldown probe admitted")
9841 };
9842 match settlement {
9843 "timeout" => assert!(
9844 !old.record_timeout(1, Duration::ZERO)
9845 .unwrap()
9846 .reopened_after_probe
9847 ),
9848 "inconclusive" => old.record_inconclusive(),
9849 "drop" => drop(old),
9850 _ => unreachable!(),
9851 }
9852 assert!(
9853 matches!(
9854 breakers.admit("prov"),
9855 RouteBindAdmission::Refused {
9856 probe_in_flight: true,
9857 ..
9858 }
9859 ),
9860 "{settlement} of a pre-open relay cannot release the real probe"
9861 );
9862 assert!(
9863 probe
9864 .record_timeout(1, Duration::ZERO)
9865 .unwrap()
9866 .reopened_after_probe
9867 );
9868 assert!(matches!(
9869 breakers.admit("prov"),
9870 RouteBindAdmission::Admitted { probe: true, .. }
9871 ));
9872 }
9873 let breakers = RouteBindBreakers::default();
9874 let admit = || match breakers.admit("prov") {
9875 RouteBindAdmission::Admitted { guard, .. } => guard,
9876 _ => panic!("relay admitted"),
9877 };
9878 admit().record_timeout(1, Duration::ZERO);
9879 let mut old_probe = admit();
9880 breakers.reset_for_new_module_connection("prov");
9881 admit().record_timeout(1, Duration::ZERO);
9882 let _new_probe = admit();
9883 old_probe.record_inconclusive();
9884 assert!(matches!(
9885 breakers.admit("prov"),
9886 RouteBindAdmission::Refused {
9887 probe_in_flight: true,
9888 ..
9889 }
9890 ));
9891 }
9892
9893 #[tokio::test]
9894 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
9895 for candidate in [false, true] {
9896 let registry = Arc::new(Registry::default());
9897 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9898 [("vault".to_string(), true), ("squatter".to_string(), true)],
9899 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9900 );
9901 let conn = ConnectionId::new(77);
9902 let (ctx, mut rx) = route_ctx(conn);
9903 let initial = capability_manifest("squatter", &[], &[]);
9904 if candidate {
9905 registry
9906 .register_candidate_with_control_ops(
9907 initial.clone(),
9908 PROTOCOL_VERSION,
9909 conn,
9910 module_baseline_control_ops(),
9911 )
9912 .unwrap();
9913 handler
9914 .forwarding
9915 .register_candidate_module_connection(
9916 conn,
9917 "squatter".to_string(),
9918 PROTOCOL_VERSION,
9919 manifest_concurrency(&initial),
9920 ctx.egress.clone(),
9921 )
9922 .unwrap();
9923 } else {
9924 hello_via_sink(
9925 &handler,
9926 &ctx,
9927 &mut rx,
9928 hello_frame_with_manifest(initial.clone(), 1),
9929 )
9930 .await;
9931 }
9932 let response = handler
9933 .handle_control_frame(
9934 &ctx,
9935 catalog_update_with_capabilities_frame(
9936 2,
9937 capability_manifest("squatter", &["credentials-provider/v1"], &[])
9938 .capabilities
9939 .unwrap(),
9940 ),
9941 )
9942 .await
9943 .unwrap();
9944 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9945 assert_eq!(
9946 registry
9947 .get_module_by_connection(conn)
9948 .unwrap()
9949 .unwrap()
9950 .manifest,
9951 initial
9952 );
9953 }
9954 }
9955
9956 #[test]
9957 fn server_describe_surfaces_required_capability_verdict_fields() {
9958 let registry = Arc::new(Registry::default());
9959 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9960 [
9961 ("consumer".to_string(), true),
9962 ("provider".to_string(), false),
9963 ],
9964 BTreeMap::new(),
9965 );
9966 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9967 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9968 provides: Vec::new(),
9969 requires: vec![subc_protocol::manifest::CapabilityRequirement {
9970 capability: "credentials-provider/v1".to_string(),
9971 need: subc_protocol::manifest::CapabilityNeed::Required,
9972 }],
9973 must_never_reach: Vec::new(),
9974 });
9975 let hello = Frame::build(
9976 FrameType::Hello,
9977 control_flags(),
9978 0,
9979 0,
9980 78,
9981 serde_json::to_vec(&ModuleHelloBody {
9982 manifest: consumer,
9983 protocol_ver: PROTOCOL_VERSION,
9984 control_ops: None,
9985 launch_nonce: None,
9986 })
9987 .expect("HELLO serializes"),
9988 )
9989 .expect("HELLO frame builds");
9990 handler
9991 .handle_control(ConnectionId::new(78), hello)
9992 .expect("consumer registers");
9993 let describe = Frame::build(
9994 FrameType::Request,
9995 control_flags(),
9996 0,
9997 0,
9998 79,
9999 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
10000 .expect("request serializes"),
10001 )
10002 .expect("describe frame builds");
10003 let response = handler
10004 .handle_server_describe(describe)
10005 .expect("server.describe succeeds");
10006 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
10007 let requirement = &rendered["capability_requirements"][0];
10008 assert_eq!(requirement["consumer"], "consumer");
10009 assert_eq!(requirement["verdict"], "never_provided");
10010 assert_eq!(requirement["episode_seq"], 1);
10011 assert_eq!(requirement["config_satisfiable"], false);
10012 assert_eq!(requirement["runtime_available"], false);
10013 assert!(requirement["detail"]
10014 .as_str()
10015 .expect("detail string")
10016 .contains("credentials-provider/v1"));
10017 }
10018
10019 fn deregister_required_provider(
10025 daemon_shutdown_begun: bool,
10026 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
10027 const CAPABILITY: &str = "project-identity/v1";
10028 let registry = Arc::new(Registry::default());
10029 let daemon_shutdown = DaemonShutdownFlag::default();
10030 let handler = ControlHandler::new(Arc::clone(®istry))
10031 .with_capability_config(
10032 [
10033 ("consumer".to_string(), true),
10034 ("provider".to_string(), true),
10035 ],
10036 BTreeMap::new(),
10037 )
10038 .with_daemon_shutdown_flag(daemon_shutdown.clone());
10039 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10040 consumer.capabilities = Some(CapabilityDeclarations {
10041 provides: Vec::new(),
10042 requires: vec![subc_protocol::manifest::CapabilityRequirement {
10043 capability: CAPABILITY.to_string(),
10044 need: subc_protocol::manifest::CapabilityNeed::Required,
10045 }],
10046 must_never_reach: Vec::new(),
10047 });
10048 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
10049 let provider_conn = ConnectionId::new(182);
10050 for (conn, manifest, corr) in [
10051 (ConnectionId::new(181), consumer, 181),
10052 (provider_conn, provider, 182),
10053 ] {
10054 let hello = handler
10055 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
10056 .expect("module registers");
10057 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10058 }
10059 assert_eq!(
10060 handler.capability_evaluator.verdict("consumer", CAPABILITY),
10061 Some(CapabilityVerdict::Provided),
10062 "the registered provider must satisfy the requirement first"
10063 );
10064 if daemon_shutdown_begun {
10065 daemon_shutdown.set_for_test();
10066 }
10067
10068 let capture = EventCapture::default();
10069 let _subscriber =
10070 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10071 handler
10072 .cleanup_connection(provider_conn)
10073 .expect("provider deregisters");
10074 let events = capture
10075 .events()
10076 .into_iter()
10077 .filter(|event| {
10078 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
10079 })
10080 .collect();
10081 (
10082 events,
10083 handler.capability_evaluator.verdict("consumer", CAPABILITY),
10084 )
10085 }
10086
10087 #[test]
10088 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
10089 let (events, verdict) = deregister_required_provider(true);
10090 assert_eq!(
10091 verdict,
10092 Some(CapabilityVerdict::NeverProvided),
10093 "shutdown must not change the verdict route.open refusals read"
10094 );
10095 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10096 let event = &events[0];
10097 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
10098 assert_eq!(
10099 event.fields.get("cause").map(String::as_str),
10100 Some("\"daemon_shutdown\""),
10101 "{event:?}"
10102 );
10103 assert_eq!(
10104 event.fields.get("verdict").map(String::as_str),
10105 Some("\"never_provided\""),
10106 "{event:?}"
10107 );
10108 }
10109
10110 #[test]
10111 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
10112 let (events, verdict) = deregister_required_provider(false);
10113 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
10114 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10115 let event = &events[0];
10116 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
10117 assert_eq!(event.fields.get("cause"), None, "{event:?}");
10118 assert_eq!(
10119 event.fields.get("verdict").map(String::as_str),
10120 Some("\"never_provided\""),
10121 "{event:?}"
10122 );
10123 }
10124
10125 #[test]
10126 fn catalog_list_omits_capabilities_for_legacy_manifest() {
10127 let registry = Arc::new(Registry::default());
10128 let handler = ControlHandler::new(Arc::clone(®istry));
10129 let hello = handler
10130 .handle_control(
10131 ConnectionId::new(101),
10132 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
10133 )
10134 .expect("legacy HELLO registers");
10135 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10136
10137 let request = Frame::build(
10138 FrameType::Request,
10139 control_flags(),
10140 0,
10141 0,
10142 102,
10143 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10144 .expect("catalog request serializes"),
10145 )
10146 .expect("catalog request frame builds");
10147 let response = handler
10148 .handle_catalog_list(request, None)
10149 .expect("catalog list succeeds");
10150 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10151 assert!(
10152 body["modules"][0].get("capabilities").is_none(),
10153 "legacy manifest must retain an absent capabilities field on catalog.list"
10154 );
10155 }
10156
10157 #[test]
10158 fn hello_ack_omits_storage_when_no_storage_config() {
10159 let registry = Arc::new(Registry::default());
10160 let handler = ControlHandler::new(Arc::clone(®istry));
10161 let responses = handler
10162 .handle_control(
10163 ConnectionId::new(1),
10164 hello_frame("aft", PROTOCOL_VERSION, 7),
10165 )
10166 .unwrap();
10167 let ack = parse_ack(&responses[0]);
10168 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
10169 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
10170 }
10171
10172 #[tokio::test]
10173 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
10174 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
10175 let registry = Arc::new(Registry::default());
10176 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
10177 let responses = handler
10178 .handle_control(
10179 ConnectionId::new(1),
10180 hello_frame("aft", PROTOCOL_VERSION, 7),
10181 )
10182 .unwrap();
10183 let ack = parse_ack(&responses[0]);
10184 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
10185
10186 let described = handler
10187 .handle_control_frame(
10188 &route_ctx(ConnectionId::new(2)).0,
10189 Frame::build(
10190 FrameType::Request,
10191 control_flags(),
10192 0,
10193 0,
10194 9,
10195 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10196 )
10197 .unwrap(),
10198 )
10199 .await
10200 .unwrap();
10201 let ClientControlResponse::ServerDescribe { machine_id, .. } =
10202 serde_json::from_slice(&described[0].body).unwrap()
10203 else {
10204 panic!("server.describe answered with another shape");
10205 };
10206 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
10207 }
10208
10209 #[test]
10210 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
10211 let registry = Arc::new(Registry::default());
10214 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
10215 crate::daemon_config::StorageConfig::Sqlite {
10216 data_home: std::path::PathBuf::from("/data"),
10217 },
10218 ));
10219
10220 let responses = handler
10221 .handle_control(
10222 ConnectionId::new(1),
10223 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
10224 )
10225 .unwrap();
10226 let ack = parse_ack(&responses[0]);
10227 assert_eq!(
10228 ack.storage,
10229 Some(serde_json::json!({
10230 "module_id": "alfonso-routing",
10231 "storage_namespace": "default",
10232 "isolation": { "kind": "module" },
10233 "backend": {
10234 "backend": "sqlite",
10235 "path": "/data/cortexkit/alfonso-routing/store.db"
10236 }
10237 })),
10238 "the delivered descriptor is the module's own sqlite store path"
10239 );
10240 }
10241
10242 #[test]
10243 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
10244 let registry = Arc::new(Registry::default());
10245 let handler = ControlHandler::new(Arc::clone(®istry));
10246 let conn = ConnectionId::new(1);
10247 let responses = handler
10248 .handle_control(
10249 conn,
10250 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10251 )
10252 .unwrap();
10253 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
10254 let registration = registry.get_module("aft").unwrap().unwrap();
10255 assert_eq!(registration.control_ops, module_baseline_control_ops());
10256
10257 let frame =
10258 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
10259 assert!(handler
10260 .guard_module_control_op(&frame, "aft", "route.bind")
10261 .unwrap()
10262 .is_none());
10263 let error = handler
10264 .guard_module_control_op(&frame, "aft", "test.synthetic")
10265 .unwrap()
10266 .expect("synthetic ungranted op should be rejected");
10267 assert_eq!(error.header.ty, FrameType::Error);
10268 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
10269 }
10270
10271 #[test]
10272 fn hello_control_ops_some_adds_optional_grants() {
10273 let registry = Arc::new(Registry::default());
10274 let handler = ControlHandler::new(Arc::clone(®istry));
10275 handler
10276 .handle_control(
10277 ConnectionId::new(1),
10278 hello_frame_with_control_ops(
10279 "aft",
10280 PROTOCOL_VERSION,
10281 7,
10282 Some(vec![
10283 "future.synthetic".to_string(),
10284 "route.bind".to_string(),
10285 ]),
10286 ),
10287 )
10288 .unwrap();
10289 let registration = registry.get_module("aft").unwrap().unwrap();
10290 assert_eq!(
10291 registration.control_ops,
10292 vec![
10293 "route.bind".to_string(),
10294 "route.status".to_string(),
10295 "future.synthetic".to_string(),
10296 ]
10297 );
10298 let frame =
10299 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
10300 assert!(handler
10301 .guard_module_control_op(&frame, "aft", "future.synthetic")
10302 .unwrap()
10303 .is_none());
10304 }
10305
10306 #[tokio::test]
10307 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
10308 let registry = Arc::new(Registry::default());
10309 let forwarding = Arc::new(ForwardingTable::default());
10310 let handler =
10311 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10312 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
10313 hello_via_sink(
10314 &handler,
10315 &module_ctx,
10316 &mut module_rx,
10317 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10318 )
10319 .await;
10320
10321 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
10322 let responses = handler
10323 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
10324 .await
10325 .unwrap();
10326 assert_eq!(responses.len(), 1);
10327 assert_eq!(responses[0].header.ty, FrameType::Error);
10328 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
10329 assert!(module_rx.try_recv().is_err());
10330 }
10331
10332 #[tokio::test]
10333 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
10334 let registry = Arc::new(Registry::default());
10335 let forwarding = Arc::new(ForwardingTable::default());
10336 let handler =
10337 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10338 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
10339 hello_via_sink(
10340 &handler,
10341 &module_ctx,
10342 &mut module_rx,
10343 hello_frame_with_control_ops(
10344 "aft",
10345 PROTOCOL_VERSION,
10346 7,
10347 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10348 ),
10349 )
10350 .await;
10351
10352 let project_root = unique_project_root("demux");
10353 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
10354 let route_handler = handler.clone();
10355 let route_task = tokio::spawn(async move {
10356 route_handler
10357 .handle_control_frame(
10358 &route_client_ctx,
10359 route_open_frame(100, "aft", project_root),
10360 )
10361 .await
10362 .unwrap()
10363 });
10364 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10365 .await
10366 .unwrap()
10367 .unwrap();
10368 assert!(matches!(
10369 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
10370 ModuleControlRequest::RouteBind { .. }
10371 ));
10372
10373 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
10374 let health_handler = handler.clone();
10375 let health_task = tokio::spawn(async move {
10376 health_handler
10377 .handle_control_frame(
10378 &health_client_ctx,
10379 supervisor_health_probe_frame(101, "aft"),
10380 )
10381 .await
10382 .unwrap()
10383 });
10384 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10385 .await
10386 .unwrap()
10387 .unwrap();
10388 assert_eq!(
10389 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10390 ModuleControlRequest::HealthCheck {}
10391 );
10392
10393 handler
10394 .handle_control_frame(
10395 &module_ctx,
10396 health_response(health_frame.header.corr, HealthStatus::Degraded),
10397 )
10398 .await
10399 .unwrap();
10400 let health_response = health_task.await.unwrap();
10401 assert_eq!(health_response.len(), 1);
10402 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10403 ClientControlResponse::SupervisorHealthProbe {
10404 module_id,
10405 status,
10406 detail,
10407 metrics,
10408 } => {
10409 assert_eq!(module_id, "aft");
10410 assert_eq!(status, HealthStatus::Degraded);
10411 assert_eq!(detail.as_deref(), Some("warming"));
10412 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
10413 }
10414 other => panic!("unexpected health response: {other:?}"),
10415 }
10416
10417 handler
10418 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10419 .await
10420 .unwrap();
10421 let route_response = route_task.await.unwrap();
10422 assert!(route_response.is_empty());
10423 let published = route_client_rx.recv().await.unwrap();
10424 assert!(matches!(
10425 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10426 ClientControlResponse::RouteOpen { .. }
10427 ));
10428 }
10429
10430 async fn relay_route_open(
10435 handler: &ControlHandler,
10436 client_connection: ConnectionId,
10437 client_egress: &FrameSink,
10438 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10439 corr: u64,
10440 module_id: &str,
10441 project_root_label: &str,
10442 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
10443 let ctx = RouteCtx {
10444 connection_id: client_connection,
10445 egress: client_egress.clone(),
10446 };
10447 let handler = handler.clone();
10448 let project_root = unique_project_root(project_root_label);
10449 let module_id = module_id.to_string();
10450 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
10451 let task = tokio::spawn(async move {
10452 let _guard = tracing::dispatcher::set_default(&dispatch);
10453 handler
10454 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
10455 .await
10456 .unwrap()
10457 });
10458 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
10459 .await
10460 .expect("module receives the relayed route.bind")
10461 .expect("module egress is open");
10462 (task, bind.frame)
10463 }
10464
10465 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
10466 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
10467 ModuleControlRequest::RouteBind {
10468 route_channel,
10469 epoch,
10470 ..
10471 } => (route_channel, epoch),
10472 other => panic!("expected a route.bind request, got {other:?}"),
10473 }
10474 }
10475
10476 fn published_route(frame: &Frame) -> (u16, u32) {
10477 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
10478 ClientControlResponse::RouteOpen {
10479 route_channel,
10480 route_epoch,
10481 } => (route_channel, route_epoch),
10482 other => panic!("expected a route.open response, got {other:?}"),
10483 }
10484 }
10485
10486 #[tokio::test]
10506 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
10507 let registry = Arc::new(Registry::default());
10508 let forwarding = Arc::new(ForwardingTable::default());
10509 let handler =
10510 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10511
10512 let module_connection = ConnectionId::new(30);
10513 let (module_ctx, mut module_rx) = route_ctx(module_connection);
10514 hello_via_sink(
10515 &handler,
10516 &module_ctx,
10517 &mut module_rx,
10518 hello_frame("aft", PROTOCOL_VERSION, 7),
10519 )
10520 .await;
10521
10522 let dying_client = ConnectionId::new(31);
10523 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
10524
10525 let (first_task, first_bind) = relay_route_open(
10528 &handler,
10529 dying_client,
10530 &dying_ctx.egress,
10531 &mut module_rx,
10532 100,
10533 "aft",
10534 "closing-first",
10535 )
10536 .await;
10537 handler
10538 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
10539 .await
10540 .unwrap();
10541 assert!(first_task.await.unwrap().is_empty());
10542 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
10543
10544 let (second_task, second_bind) = relay_route_open(
10546 &handler,
10547 dying_client,
10548 &dying_ctx.egress,
10549 &mut module_rx,
10550 101,
10551 "aft",
10552 "closing-second",
10553 )
10554 .await;
10555 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
10556
10557 assert!(forwarding
10560 .escalate_client_delivery_failure(
10561 dying_client,
10562 first_channel,
10563 first_epoch,
10564 CloseReason::new(
10565 "module_to_client_delivery_failed",
10566 "client egress refused a module frame",
10567 ),
10568 crate::forwarding::UndeliveredFrame {
10569 module_id: None,
10570 sink: &dying_ctx.egress,
10571 },
10572 )
10573 .unwrap());
10574 assert!(!dying_ctx.egress.is_closed());
10575
10576 let ack = handler
10578 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
10579 .await;
10580 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10581 if module_loop_error.is_some() {
10582 handler.cleanup_connection(module_connection).unwrap();
10586 }
10587 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10591 .await
10592 .ok()
10593 .flatten();
10594
10595 assert!(
10597 registry
10598 .get_module_by_connection(module_connection)
10599 .unwrap()
10600 .is_some(),
10601 "one client's closing connection ended the shared module connection: \
10602 {module_loop_error:?}"
10603 );
10604 let cotenant = ConnectionId::new(32);
10606 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10607 let (cotenant_task, cotenant_bind) = relay_route_open(
10608 &handler,
10609 cotenant,
10610 &cotenant_ctx.egress,
10611 &mut module_rx,
10612 102,
10613 "aft",
10614 "closing-cotenant",
10615 )
10616 .await;
10617 handler
10618 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
10619 .await
10620 .unwrap();
10621 assert!(cotenant_task.await.unwrap().is_empty());
10622 let (cotenant_channel, cotenant_epoch) =
10623 published_route(&cotenant_rx.recv().await.unwrap());
10624 assert!(matches!(
10625 forwarding
10626 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10627 .unwrap(),
10628 DataRoute::Client(DataRouteState::Bound(_))
10629 ));
10630
10631 let goodbye = post_ack_module_frame
10634 .expect("module receives a GOODBYE for the abandoned route channel");
10635 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10636 assert_eq!(goodbye.header.channel, abandoned_channel);
10637 assert_eq!(goodbye.header.epoch, abandoned_epoch);
10638
10639 assert!(dying_rx.try_recv().is_err());
10643 let second_response = second_task.await.unwrap();
10644 assert_eq!(second_response.len(), 1);
10645 assert_eq!(
10646 parse_error(&second_response[0])["code"],
10647 "target_unavailable"
10648 );
10649 }
10650
10651 async fn catalog_snapshot(
10654 handler: &ControlHandler,
10655 client_ctx: &RouteCtx,
10656 corr: u64,
10657 ) -> (u64, Vec<(String, bool)>) {
10658 let body =
10659 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
10660 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
10661 let reply = handler
10662 .handle_control_frame(client_ctx, request)
10663 .await
10664 .unwrap()
10665 .pop()
10666 .unwrap();
10667 let ClientControlResponse::CatalogList {
10668 generation,
10669 modules,
10670 ..
10671 } = serde_json::from_slice(&reply.body).unwrap()
10672 else {
10673 panic!("not a catalog.list reply");
10674 };
10675 (
10676 generation,
10677 modules
10678 .into_iter()
10679 .map(|entry| (entry.module_id, entry.ready))
10680 .collect(),
10681 )
10682 }
10683
10684 fn hello_held_before_forwarding() -> (
10690 ControlHandler,
10691 Arc<std::sync::Barrier>,
10692 std::thread::JoinHandle<Vec<Frame>>,
10693 RouteCtx,
10694 mpsc::Receiver<crate::router::OutboundFrame>,
10695 ) {
10696 let hold = Arc::new(std::sync::Barrier::new(2));
10697 let handler = ControlHandler::with_forwarding(
10698 Arc::new(Registry::default()),
10699 Arc::new(ForwardingTable::default()),
10700 )
10701 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
10702 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10703 let hello = {
10704 let handler = handler.clone();
10705 let module_ctx = module_ctx.clone();
10706 std::thread::spawn(move || {
10707 handler
10708 .handle_hello(
10709 module_ctx.connection_id,
10710 Some(module_ctx.egress.clone()),
10711 hello_frame("aft", PROTOCOL_VERSION, 7),
10712 )
10713 .unwrap()
10714 })
10715 };
10716 (handler, hold, hello, module_ctx, module_rx)
10717 }
10718
10719 #[tokio::test]
10728 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
10729 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
10730 let client = ConnectionId::new(41);
10731 let (client_ctx, mut client_rx) = route_ctx(client);
10732
10733 hold.wait();
10734 assert!(handler.registry.get_module("aft").unwrap().is_some());
10736 assert!(!handler
10737 .forwarding
10738 .has_live_module_connection("aft")
10739 .unwrap());
10740 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
10741 assert!(
10742 listed.is_empty(),
10743 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
10744 );
10745 let refused = handler
10746 .handle_control_frame(
10747 &client_ctx,
10748 route_open_frame(301, "aft", unique_project_root("listed-window")),
10749 )
10750 .await
10751 .unwrap();
10752 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
10753 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
10754 let duplicate = handler
10755 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
10756 .await
10757 .unwrap();
10758 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10759
10760 hold.wait();
10761 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10762 let ack = module_rx.try_recv().unwrap().frame;
10763 assert_eq!(ack.header.ty, FrameType::HelloAck);
10764 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10765 assert_eq!(listed, vec![("aft".to_string(), true)]);
10766
10767 let (open_task, bind) = relay_route_open(
10768 &handler,
10769 client,
10770 &client_ctx.egress,
10771 &mut module_rx,
10772 303,
10773 "aft",
10774 "listed-after",
10775 )
10776 .await;
10777 assert!(handler
10778 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10779 .await
10780 .unwrap()
10781 .is_empty());
10782 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
10783 published_route(&client_rx.try_recv().unwrap().frame);
10784 }
10785
10786 #[tokio::test]
10793 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
10794 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
10795 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10796 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10797
10798 hold.wait();
10799 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
10800 assert!(listed.is_empty());
10801 assert_eq!(
10802 during, before,
10803 "the generation moved while the module was not listed"
10804 );
10805
10806 hold.wait();
10807 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10808 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10809 assert_eq!(listed, vec![("aft".to_string(), true)]);
10810 assert_eq!(
10811 after,
10812 before + 1,
10813 "the module appeared without exactly one generation bump"
10814 );
10815 }
10816
10817 #[tokio::test]
10822 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
10823 let registry = Arc::new(Registry::default());
10824 let handler = ControlHandler::with_forwarding(
10825 Arc::clone(®istry),
10826 Arc::new(ForwardingTable::default()),
10827 );
10828 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10829 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10830 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10831 drop(module_rx);
10832
10833 let replies = handler
10834 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
10835 .await
10836 .unwrap();
10837 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
10838 assert!(registry.get_module("aft").unwrap().is_none());
10839 assert!(registry
10840 .get_module_by_connection(module_ctx.connection_id)
10841 .unwrap()
10842 .is_none());
10843 assert_eq!(
10844 catalog_snapshot(&handler, &client_ctx, 301).await,
10845 (before, Vec::new())
10846 );
10847 }
10848
10849 async fn route_open_held_after_bind_send() -> (
10853 ControlHandler,
10854 Arc<tokio::sync::Barrier>,
10855 RouteCtx,
10856 mpsc::Receiver<crate::router::OutboundFrame>,
10857 ) {
10858 let hold = Arc::new(tokio::sync::Barrier::new(2));
10859 let handler = ControlHandler::with_forwarding(
10860 Arc::new(Registry::default()),
10861 Arc::new(ForwardingTable::default()),
10862 )
10863 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
10864 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10865 hello_via_sink(
10866 &handler,
10867 &module_ctx,
10868 &mut module_rx,
10869 hello_frame("aft", PROTOCOL_VERSION, 7),
10870 )
10871 .await;
10872 (handler, hold, module_ctx, module_rx)
10873 }
10874
10875 #[tokio::test]
10882 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
10883 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10884 let client = ConnectionId::new(41);
10885 let (client_ctx, mut client_rx) = route_ctx(client);
10886
10887 let (open_task, bind) = relay_route_open(
10888 &handler,
10889 client,
10890 &client_ctx.egress,
10891 &mut module_rx,
10892 200,
10893 "aft",
10894 "acked-before-mark",
10895 )
10896 .await;
10897 let (module_channel, module_epoch) = route_bind_channel(&bind);
10898 hold.wait().await;
10900
10901 assert!(handler
10903 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10904 .await
10905 .unwrap()
10906 .is_empty());
10907 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
10908
10909 hold.wait().await;
10910 assert!(
10911 open_task.await.unwrap().is_empty(),
10912 "route.open answers accepted"
10913 );
10914
10915 match module_rx.try_recv() {
10918 Err(mpsc::error::TryRecvError::Empty) => {}
10919 Ok(outbound) => panic!(
10920 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
10921 outbound.frame.header.ty,
10922 outbound.frame.header.channel,
10923 outbound.frame.header.epoch
10924 ),
10925 Err(err) => panic!("module egress closed: {err}"),
10926 }
10927
10928 let router = Router::with_control_handler(Arc::new(handler.clone()));
10931 let request = Frame::build(
10932 FrameType::Request,
10933 Flags::new(true, Priority::Interactive, false),
10934 client_channel,
10935 client_epoch,
10936 201,
10937 b"first request".to_vec(),
10938 )
10939 .unwrap();
10940 router
10941 .route_for_connection(&client_ctx, request)
10942 .await
10943 .unwrap();
10944 let delivered = module_rx.try_recv().unwrap().frame;
10945 assert_eq!(delivered.header.ty, FrameType::Request);
10946 assert_eq!(
10947 (delivered.header.channel, delivered.header.epoch),
10948 (module_channel, module_epoch)
10949 );
10950 assert_eq!(delivered.body, b"first request");
10951 }
10952
10953 #[tokio::test]
10959 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
10960 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10961 let client = ConnectionId::new(42);
10962 let (client_ctx, _client_rx) = route_ctx(client);
10963
10964 let (open_task, bind) = relay_route_open(
10965 &handler,
10966 client,
10967 &client_ctx.egress,
10968 &mut module_rx,
10969 300,
10970 "aft",
10971 "released-before-mark",
10972 )
10973 .await;
10974 let (module_channel, module_epoch) = route_bind_channel(&bind);
10975 hold.wait().await;
10976
10977 handler.cleanup_connection(client).unwrap();
10978 assert!(
10979 module_rx.try_recv().is_err(),
10980 "client teardown must not GOODBYE a bind route.open is still queueing"
10981 );
10982
10983 hold.wait().await;
10984 let response = open_task.await.unwrap();
10985 assert_eq!(response.len(), 1);
10986 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10987
10988 let goodbye = module_rx
10989 .try_recv()
10990 .expect("module is told to drop the binding the daemon abandoned")
10991 .frame;
10992 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10993 assert_eq!(
10994 (goodbye.header.channel, goodbye.header.epoch),
10995 (module_channel, module_epoch)
10996 );
10997 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10998 }
10999
11000 #[tokio::test]
11005 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
11006 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11007 let client = ConnectionId::new(43);
11008 let (client_ctx, client_rx) = route_ctx(client);
11009
11010 let (open_task, bind) = relay_route_open(
11011 &handler,
11012 client,
11013 &client_ctx.egress,
11014 &mut module_rx,
11015 400,
11016 "aft",
11017 "gone-client-before-mark",
11018 )
11019 .await;
11020 let (module_channel, module_epoch) = route_bind_channel(&bind);
11021 hold.wait().await;
11022
11023 drop(client_rx);
11024 assert!(client_ctx.egress.is_closed());
11025 handler
11026 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11027 .await
11028 .unwrap();
11029
11030 hold.wait().await;
11031 let response = open_task.await.unwrap();
11032 assert_eq!(response.len(), 1);
11033 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11034
11035 let goodbye = module_rx
11036 .try_recv()
11037 .expect("module is told to drop the binding it acked")
11038 .frame;
11039 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11040 assert_eq!(
11041 (goodbye.header.channel, goodbye.header.epoch),
11042 (module_channel, module_epoch)
11043 );
11044 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11045 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11046 }
11047
11048 #[test]
11055 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
11056 let handler = ControlHandler::default();
11057 let module_connection = ConnectionId::new(30);
11058 let client_connection = ConnectionId::new(31);
11059
11060 handler
11061 .refuse_to_end_module_connection_for_a_client(
11062 module_connection,
11063 77,
11064 ForwardingError::ConnectionClosing {
11065 connection_id: client_connection,
11066 },
11067 )
11068 .expect("a closing client must never end the module connection");
11069
11070 assert!(matches!(
11071 handler.refuse_to_end_module_connection_for_a_client(
11072 module_connection,
11073 78,
11074 ForwardingError::ConnectionClosing {
11075 connection_id: module_connection,
11076 },
11077 ),
11078 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
11079 connection_id
11080 })) if connection_id == module_connection
11081 ));
11082 assert!(matches!(
11083 handler.refuse_to_end_module_connection_for_a_client(
11084 module_connection,
11085 79,
11086 ForwardingError::Poisoned,
11087 ),
11088 Err(RouterError::Forwarding(ForwardingError::Poisoned))
11089 ));
11090 assert!(matches!(
11091 handler.refuse_to_end_module_connection_for_a_client(
11092 module_connection,
11093 80,
11094 ForwardingError::StaleModuleEndpoint,
11095 ),
11096 Err(RouterError::Forwarding(
11097 ForwardingError::StaleModuleEndpoint
11098 ))
11099 ));
11100 }
11101
11102 #[tokio::test]
11115 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
11116 let registry = Arc::new(Registry::default());
11117 let forwarding = Arc::new(ForwardingTable::default());
11118 let supervisor = SupervisorHandle::new();
11119 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11120 let handler =
11121 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11122 .with_supervisor(supervisor);
11123
11124 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11125 hello_via_sink(
11126 &handler,
11127 &target_ctx,
11128 &mut target_rx,
11129 hello_frame("target", PROTOCOL_VERSION, 1),
11130 )
11131 .await;
11132
11133 let wrong_nonce = handler
11137 .handle_control_frame(
11138 &route_ctx(ConnectionId::new(91)).0,
11139 route_open_frame_with_admission_facts(
11140 20,
11141 "target",
11142 unique_project_root("admission-facts"),
11143 Some(subc_control::ConsumerIdentity {
11144 module_id: "fed".to_string(),
11145 launch_nonce: "not-the-real-nonce".to_string(),
11146 }),
11147 None,
11148 ),
11149 )
11150 .await
11151 .unwrap();
11152 assert_eq!(
11153 parse_error(&wrong_nonce[0])["code"],
11154 "bad_consumer_identity",
11155 "a mismatched launch nonce must be refused, not stamped Reserved"
11156 );
11157
11158 let never_spawned = handler
11162 .handle_control_frame(
11163 &route_ctx(ConnectionId::new(92)).0,
11164 route_open_frame_with_admission_facts(
11165 21,
11166 "target",
11167 unique_project_root("admission-facts"),
11168 Some(subc_control::ConsumerIdentity {
11169 module_id: "never-spawned".to_string(),
11170 launch_nonce: "any-nonce".to_string(),
11171 }),
11172 None,
11173 ),
11174 )
11175 .await
11176 .unwrap();
11177 assert_eq!(
11178 parse_error(&never_spawned[0])["code"],
11179 "bad_consumer_identity",
11180 "an unspawned module_id must be refused rather than accepted for lack of a record"
11181 );
11182 }
11183
11184 #[tokio::test]
11207 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
11208 let registry = Arc::new(Registry::default());
11209 let forwarding = Arc::new(ForwardingTable::default());
11210 let supervisor = SupervisorHandle::new();
11211 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11212 let handler =
11213 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11214 .with_supervisor(supervisor);
11215
11216 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
11217 hello_via_sink(
11218 &handler,
11219 &target_ctx,
11220 &mut target_rx,
11221 hello_frame("target", PROTOCOL_VERSION, 1),
11222 )
11223 .await;
11224
11225 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
11226 let route_handler = handler.clone();
11227 let route_task = tokio::spawn(async move {
11228 route_handler
11229 .handle_control_frame(
11230 &client_ctx,
11231 route_open_frame_with_admission_facts(
11232 30,
11233 "target",
11234 unique_project_root("admission-facts"),
11235 Some(subc_control::ConsumerIdentity {
11236 module_id: "fed".to_string(),
11237 launch_nonce: "fed-nonce".to_string(),
11238 }),
11239 None,
11240 ),
11241 )
11242 .await
11243 .unwrap()
11244 });
11245
11246 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11253 .await
11254 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
11255 .expect("module control channel closed before route.bind");
11256 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11257 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11258 panic!("expected route.bind")
11259 };
11260 assert_eq!(
11261 principal,
11262 Some(Principal::Reserved {
11263 module_id: "fed".to_string()
11264 }),
11265 "a correctly attested consumer must be stamped Reserved for its own id"
11266 );
11267
11268 handler
11269 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11270 .await
11271 .unwrap();
11272 assert!(route_task.await.unwrap().is_empty());
11273 assert!(
11274 matches!(
11275 serde_json::from_slice::<ClientControlResponse>(
11276 &client_rx.recv().await.unwrap().body
11277 )
11278 .unwrap(),
11279 ClientControlResponse::RouteOpen { .. }
11280 ),
11281 "the route must actually open, not merely avoid an error"
11282 );
11283 }
11284
11285 #[tokio::test(start_paused = true)]
11286 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
11287 let registry = Arc::new(Registry::default());
11288 let forwarding = Arc::new(ForwardingTable::default());
11289 let supervisor = SupervisorHandle::new();
11290 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11291 let handler =
11292 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11293 .with_supervisor(supervisor);
11294
11295 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
11296 hello_via_sink(
11297 &handler,
11298 &target_ctx,
11299 &mut target_rx,
11300 hello_frame("target", PROTOCOL_VERSION, 1),
11301 )
11302 .await;
11303
11304 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
11305 let direct_handler = handler.clone();
11306 let direct_open = tokio::spawn(async move {
11307 direct_handler
11308 .handle_control_frame(
11309 &direct_ctx,
11310 route_open_frame(2, "target", unique_project_root("route-census-direct")),
11311 )
11312 .await
11313 .unwrap()
11314 });
11315 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11316 .await
11317 .expect("no direct route.bind within 5s")
11318 .expect("target control channel closed before direct route.bind");
11319 handler
11320 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
11321 .await
11322 .unwrap();
11323 assert!(direct_open.await.unwrap().is_empty());
11324 let _ = direct_rx.recv().await.unwrap();
11325
11326 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
11327 let reserved_handler = handler.clone();
11328 let reserved_open = tokio::spawn(async move {
11329 reserved_handler
11330 .handle_control_frame(
11331 &reserved_ctx,
11332 route_open_frame_with_admission_facts(
11333 3,
11334 "target",
11335 unique_project_root("admission-facts"),
11336 Some(ConsumerIdentity {
11337 module_id: "fed".to_string(),
11338 launch_nonce: "fed-nonce".to_string(),
11339 }),
11340 None,
11341 ),
11342 )
11343 .await
11344 .unwrap()
11345 });
11346 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11347 .await
11348 .expect("no reserved route.bind within 5s")
11349 .expect("target control channel closed before reserved route.bind");
11350 handler
11351 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
11352 .await
11353 .unwrap();
11354 assert!(reserved_open.await.unwrap().is_empty());
11355 let _ = reserved_rx.recv().await.unwrap();
11356
11357 forwarding
11358 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
11359 .unwrap();
11360 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
11361 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
11362 module_id: Some("target".to_string()),
11363 })
11364 .unwrap();
11365 let census_frame =
11366 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
11367 let response = handler
11368 .handle_control_frame(&census_ctx, census_frame)
11369 .await
11370 .unwrap()
11371 .pop()
11372 .unwrap();
11373 let actual: Value = serde_json::from_slice(&response.body).unwrap();
11374 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
11375 assert!(matches!(
11376 decoded,
11377 ClientControlResponse::SupervisorRoutes { .. }
11378 ));
11379 let routes = actual["modules"][0]["routes"].as_array().unwrap();
11380 assert_eq!(routes.len(), 2);
11381 assert!(routes.iter().all(|route| route["draining"] == true));
11382 assert!(
11385 routes.iter().all(|route| route["drain_reason"] == "reload"),
11386 "draining routes must name the drain's reason: {routes:?}"
11387 );
11388 assert!(routes.iter().any(|route| {
11389 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
11390 }));
11391 assert!(routes.iter().any(|route| {
11392 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
11393 }));
11394
11395 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11396 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
11397 if std::env::var_os("UPDATE_GOLDEN").is_some() {
11398 std::fs::write(
11399 &golden_path,
11400 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
11401 )
11402 .unwrap();
11403 }
11404 let expected: Value =
11405 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
11406 assert_eq!(actual, expected);
11407 }
11408
11409 async fn query_live_roots(
11410 handler: &ControlHandler,
11411 module_ctx: &RouteCtx,
11412 ) -> ModuleControlResponseToModule {
11413 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
11414 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
11415 let response = handler
11416 .handle_control_frame(module_ctx, frame)
11417 .await
11418 .unwrap()
11419 .pop()
11420 .unwrap();
11421 serde_json::from_slice(&response.body).unwrap()
11422 }
11423
11424 #[tokio::test(start_paused = true)]
11425 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
11426 let registry = Arc::new(Registry::default());
11427 let forwarding = Arc::new(ForwardingTable::default());
11428 let handler = ControlHandler::with_forwarding(registry, forwarding);
11429 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
11430 hello_via_sink(
11431 &handler,
11432 &target_ctx,
11433 &mut target_rx,
11434 hello_frame("target", PROTOCOL_VERSION, 1),
11435 )
11436 .await;
11437 let root = unique_project_root("live-roots-known");
11438 let path = ProjectRootId::from_path_allowing_missing(root.path())
11439 .unwrap()
11440 .as_path()
11441 .to_path_buf();
11442 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
11443 let open_handler = handler.clone();
11444 let opened = tokio::spawn(async move {
11445 open_handler
11446 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
11447 .await
11448 .unwrap()
11449 });
11450 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11451 .await
11452 .unwrap()
11453 .unwrap();
11454 handler
11455 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
11456 .await
11457 .unwrap();
11458 assert!(opened.await.unwrap().is_empty());
11459 let _ = client_rx.recv().await.unwrap();
11460
11461 let root = unique_project_root("live-roots-pending");
11462 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
11463 .unwrap()
11464 .as_path()
11465 .to_path_buf();
11466 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
11467 let open_handler = handler.clone();
11468 let pending = tokio::spawn(async move {
11469 open_handler
11470 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
11471 .await
11472 .unwrap()
11473 });
11474 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11475 .await
11476 .unwrap()
11477 .unwrap();
11478 let actual = query_live_roots(&handler, &target_ctx).await;
11479 let ModuleControlResponseToModule::LiveRoots {
11480 roots,
11481 unknown_root_bindings,
11482 total_bindings,
11483 } = actual
11484 else {
11485 panic!("expected live roots")
11486 };
11487 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
11488 assert_eq!(unknown_root_bindings, 0);
11489 assert_eq!(
11490 roots.len(),
11491 2,
11492 "root-known arm must retain each canonical root"
11493 );
11494 assert_eq!(
11495 total_bindings,
11496 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11497 );
11498 let counts = roots
11499 .iter()
11500 .map(|root| (root.project_root.clone(), root.bound, root.pending))
11501 .collect::<Vec<_>>();
11502 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
11503 expected.sort_by(|a, b| a.0.cmp(&b.0));
11504 assert_eq!(
11505 counts, expected,
11506 "roots must sort by path and count pending separately"
11507 );
11508 handler
11509 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
11510 .await
11511 .unwrap();
11512 assert!(pending.await.unwrap().is_empty());
11513 }
11514
11515 #[tokio::test(start_paused = true)]
11516 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
11517 let forwarding = Arc::new(ForwardingTable::default());
11518 let handler =
11519 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11520 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
11521 hello_via_sink(
11522 &handler,
11523 &target_ctx,
11524 &mut target_rx,
11525 hello_frame("target", PROTOCOL_VERSION, 1),
11526 )
11527 .await;
11528 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
11529 let pending = forwarding
11530 .begin_route_bind_relay_for_test(
11531 client_ctx.connection_id,
11532 client_ctx.egress.clone(),
11533 2,
11534 "target",
11535 )
11536 .unwrap();
11537 forwarding
11538 .complete_pending_relay(
11539 target_ctx.connection_id,
11540 pending.corr,
11541 RouteBindRelayOutcome::Accepted,
11542 )
11543 .unwrap();
11544 let actual = query_live_roots(&handler, &target_ctx).await;
11545 let ModuleControlResponseToModule::LiveRoots {
11546 roots,
11547 unknown_root_bindings,
11548 total_bindings,
11549 } = actual
11550 else {
11551 panic!("expected live roots")
11552 };
11553 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
11554 assert_eq!(
11555 unknown_root_bindings, 1,
11556 "unknown-root arm must not read as no bindings"
11557 );
11558 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
11559 assert_eq!(
11560 total_bindings,
11561 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11562 );
11563 }
11564
11565 #[tokio::test(start_paused = true)]
11573 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
11574 let forwarding = Arc::new(ForwardingTable::default());
11575 let handler =
11576 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11577 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
11578 let replies = handler
11579 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
11580 .await
11581 .unwrap();
11582 let queued_by_hello = module_rx.len();
11583
11584 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
11585 let open_handler = handler.clone();
11586 let open = tokio::spawn(async move {
11587 open_handler
11588 .handle_control_frame(
11589 &client_ctx,
11590 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
11591 )
11592 .await
11593 .unwrap()
11594 });
11595 let mut spins = 0;
11597 while module_rx.len() == queued_by_hello {
11598 spins += 1;
11599 assert!(spins < 10_000, "route.open never queued a route.bind");
11600 tokio::task::yield_now().await;
11601 }
11602
11603 for reply in replies {
11605 module_ctx.egress.send(reply).await.unwrap();
11606 }
11607
11608 let first = module_rx.recv().await.unwrap().frame;
11609 assert_eq!(
11610 first.header.ty,
11611 FrameType::HelloAck,
11612 "the first frame a registering module reads must be its HELLO_ACK"
11613 );
11614 assert_eq!(first.header.corr, 7);
11615 let second = module_rx.recv().await.unwrap().frame;
11616 assert_eq!(second.header.ty, FrameType::Request);
11617 assert!(
11618 matches!(
11619 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
11620 ModuleControlRequest::RouteBind { .. }
11621 ),
11622 "the route.bind follows the ack"
11623 );
11624 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
11625
11626 handler
11627 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
11628 .await
11629 .unwrap();
11630 assert!(open.await.unwrap().is_empty());
11631 let _ = client_rx.recv().await.unwrap();
11632 }
11633
11634 #[tokio::test(start_paused = true)]
11635 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
11636 let handler = ControlHandler::with_forwarding(
11637 Arc::new(Registry::default()),
11638 Arc::new(ForwardingTable::default()),
11639 );
11640 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
11641 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
11642 hello_via_sink(
11643 &handler,
11644 &first_ctx,
11645 &mut first_rx,
11646 hello_frame("first", PROTOCOL_VERSION, 1),
11647 )
11648 .await;
11649 hello_via_sink(
11650 &handler,
11651 &second_ctx,
11652 &mut second_rx,
11653 hello_frame("second", PROTOCOL_VERSION, 2),
11654 )
11655 .await;
11656 let root = unique_project_root("second-only");
11657 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
11658 let cloned = handler.clone();
11659 let open = tokio::spawn(async move {
11660 cloned
11661 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
11662 .await
11663 .unwrap()
11664 });
11665 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
11666 .await
11667 .unwrap()
11668 .unwrap();
11669 let first = query_live_roots(&handler, &first_ctx).await;
11670 let second = query_live_roots(&handler, &second_ctx).await;
11671 assert!(
11672 matches!(
11673 first,
11674 ModuleControlResponseToModule::LiveRoots {
11675 total_bindings: 0,
11676 ..
11677 }
11678 ),
11679 "cross-module scope must not expose another module's roots"
11680 );
11681 assert!(
11682 matches!(
11683 second,
11684 ModuleControlResponseToModule::LiveRoots {
11685 total_bindings: 1,
11686 ..
11687 }
11688 ),
11689 "second module must see its pending route"
11690 );
11691 handler
11692 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
11693 .await
11694 .unwrap();
11695 assert!(open.await.unwrap().is_empty());
11696 }
11697
11698 #[tokio::test(start_paused = true)]
11699 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
11700 let handler = ControlHandler::with_forwarding(
11701 Arc::new(Registry::default()),
11702 Arc::new(ForwardingTable::default()),
11703 );
11704 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
11705 hello_via_sink(
11706 &handler,
11707 &target_ctx,
11708 &mut target_rx,
11709 hello_frame("target", PROTOCOL_VERSION, 1),
11710 )
11711 .await;
11712 let actual = query_live_roots(&handler, &target_ctx).await;
11713 let ModuleControlResponseToModule::LiveRoots {
11714 roots,
11715 unknown_root_bindings,
11716 total_bindings,
11717 } = actual
11718 else {
11719 panic!("expected live roots")
11720 };
11721 assert!(roots.is_empty());
11722 assert_eq!(unknown_root_bindings, 0);
11723 assert_eq!(total_bindings, 0);
11724 assert_eq!(
11725 total_bindings,
11726 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11727 );
11728 }
11729
11730 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
11737 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11738 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
11739 let text = std::fs::read_to_string(&path)
11740 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
11741 let vectors: Vec<(String, Value)> = text
11742 .lines()
11743 .filter(|line| !line.trim().is_empty())
11744 .map(|line| {
11745 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
11746 let id = entry["corpus_id"]
11747 .as_str()
11748 .expect("every vector carries a corpus_id")
11749 .to_string();
11750 (id, entry["package"].clone())
11751 })
11752 .collect();
11753 assert_eq!(
11757 vectors.len(),
11758 3,
11759 "vendored fed corpus changed size; re-sync from subc-federation"
11760 );
11761
11762 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
11777 let richest = vectors
11778 .iter()
11779 .filter_map(|(_, package)| package.as_object())
11780 .map(|object| {
11781 object
11782 .keys()
11783 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
11784 .count()
11785 })
11786 .max()
11787 .unwrap_or(0);
11788 assert!(
11789 richest >= 2,
11790 "vendored corpus no longer carries a package with unmodelled fields, \
11791 so the relay test can no longer distinguish a verbatim relay from a lossy one"
11792 );
11793
11794 vectors
11795 }
11796
11797 #[tokio::test]
11805 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
11806 for (corpus_id, package) in fed_admission_facts_vectors() {
11807 let registry = Arc::new(Registry::default());
11808 let forwarding = Arc::new(ForwardingTable::default());
11809 let supervisor = SupervisorHandle::new();
11810 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11811 let handler =
11812 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11813 .with_supervisor(supervisor)
11814 .with_admission_facts_config(
11815 Some("fed".to_string()),
11816 Some(vec!["target".to_string()]),
11817 );
11818
11819 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11820 hello_via_sink(
11821 &handler,
11822 &target_ctx,
11823 &mut target_rx,
11824 hello_frame("target", PROTOCOL_VERSION, 1),
11825 )
11826 .await;
11827
11828 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
11829 let route_handler = handler.clone();
11830 let expected = package.clone();
11831 let route_task = tokio::spawn(async move {
11832 route_handler
11833 .handle_control_frame(
11834 &client_ctx,
11835 route_open_frame_with_admission_facts(
11836 20,
11837 "target",
11838 unique_project_root("admission-facts"),
11839 Some(subc_control::ConsumerIdentity {
11840 module_id: "fed".to_string(),
11841 launch_nonce: "fed-nonce".to_string(),
11842 }),
11843 Some(package),
11844 ),
11845 )
11846 .await
11847 .unwrap()
11848 });
11849
11850 let bind_frame = target_rx.recv().await.unwrap();
11851 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11852 let ModuleControlRequest::RouteBind {
11853 admission_facts, ..
11854 } = bind
11855 else {
11856 panic!("{corpus_id}: expected route.bind")
11857 };
11858 assert_eq!(
11859 admission_facts,
11860 Some(expected),
11861 "{corpus_id}: relay must not add, drop or reshape any field"
11862 );
11863
11864 handler
11865 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11866 .await
11867 .unwrap();
11868 route_task.await.unwrap();
11869 }
11870 }
11871
11872 #[tokio::test]
11873 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
11874 let registry = Arc::new(Registry::default());
11875 let forwarding = Arc::new(ForwardingTable::default());
11876 let supervisor = SupervisorHandle::new();
11877 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11878 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
11879 let handler =
11880 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11881 .with_supervisor(supervisor)
11882 .with_admission_facts_config(
11883 Some("fed".to_string()),
11884 Some(vec!["target".to_string()]),
11885 );
11886
11887 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
11888 hello_via_sink(
11889 &handler,
11890 &target_ctx,
11891 &mut target_rx,
11892 hello_frame("target", PROTOCOL_VERSION, 1),
11893 )
11894 .await;
11895 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
11896 hello_via_sink(
11897 &handler,
11898 &other_ctx,
11899 &mut other_rx,
11900 hello_frame("other", PROTOCOL_VERSION, 2),
11901 )
11902 .await;
11903
11904 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
11905 let expected_facts = facts.clone();
11906 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
11907 let route_handler = handler.clone();
11908 let route_task = tokio::spawn(async move {
11909 route_handler
11910 .handle_control_frame(
11911 &client_ctx,
11912 route_open_frame_with_admission_facts(
11913 10,
11914 "target",
11915 unique_project_root("admission-facts"),
11916 Some(subc_control::ConsumerIdentity {
11917 module_id: "fed".to_string(),
11918 launch_nonce: "fed-nonce".to_string(),
11919 }),
11920 Some(facts.clone()),
11921 ),
11922 )
11923 .await
11924 .unwrap()
11925 });
11926 let bind_frame = target_rx.recv().await.unwrap();
11927 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11928 let ModuleControlRequest::RouteBind {
11929 admission_facts, ..
11930 } = bind
11931 else {
11932 panic!("expected route.bind")
11933 };
11934 assert_eq!(admission_facts, Some(expected_facts));
11935 handler
11936 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11937 .await
11938 .unwrap();
11939 assert!(route_task.await.unwrap().is_empty());
11940 assert!(matches!(
11941 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
11942 .unwrap(),
11943 ClientControlResponse::RouteOpen { .. }
11944 ));
11945
11946 let direct = handler
11947 .handle_control_frame(
11948 &route_ctx(ConnectionId::new(73)).0,
11949 route_open_frame_with_admission_facts(
11950 11,
11951 "target",
11952 unique_project_root("admission-facts"),
11953 None,
11954 Some(json!({"x": 1})),
11955 ),
11956 )
11957 .await
11958 .unwrap();
11959 assert_eq!(
11960 parse_error(&direct[0])["code"],
11961 "admission_facts_not_permitted"
11962 );
11963
11964 let different_reserved = handler
11965 .handle_control_frame(
11966 &route_ctx(ConnectionId::new(77)).0,
11967 route_open_frame_with_admission_facts(
11968 15,
11969 "target",
11970 unique_project_root("admission-facts"),
11971 Some(subc_control::ConsumerIdentity {
11972 module_id: "other".to_string(),
11973 launch_nonce: "other-nonce".to_string(),
11974 }),
11975 Some(json!({"x": 1})),
11976 ),
11977 )
11978 .await
11979 .unwrap();
11980 assert_eq!(
11981 parse_error(&different_reserved[0])["code"],
11982 "admission_facts_not_permitted"
11983 );
11984
11985 let other_target = handler
11986 .handle_control_frame(
11987 &route_ctx(ConnectionId::new(74)).0,
11988 route_open_frame_with_admission_facts(
11989 12,
11990 "other",
11991 unique_project_root("admission-facts"),
11992 Some(subc_control::ConsumerIdentity {
11993 module_id: "fed".to_string(),
11994 launch_nonce: "fed-nonce".to_string(),
11995 }),
11996 Some(json!({"x": 1})),
11997 ),
11998 )
11999 .await
12000 .unwrap();
12001 assert_eq!(
12002 parse_error(&other_target[0])["code"],
12003 "admission_facts_target_not_allowed"
12004 );
12005
12006 let nonexistent = handler
12007 .handle_control_frame(
12008 &route_ctx(ConnectionId::new(75)).0,
12009 route_open_frame_with_admission_facts(
12010 13,
12011 "missing",
12012 unique_project_root("admission-facts"),
12013 None,
12014 Some(json!({"x": 1})),
12015 ),
12016 )
12017 .await
12018 .unwrap();
12019 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
12020
12021 let described = handler
12022 .handle_control_frame(
12023 &route_ctx(ConnectionId::new(76)).0,
12024 Frame::build(
12025 FrameType::Request,
12026 control_flags(),
12027 0,
12028 0,
12029 14,
12030 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12031 )
12032 .unwrap(),
12033 )
12034 .await
12035 .unwrap();
12036 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12037 serde_json::from_slice(&described[0].body).unwrap()
12038 else {
12039 panic!("expected server.describe response")
12040 };
12041 assert!(capabilities
12042 .iter()
12043 .any(|cap| cap == "admission_facts_relay_v1"));
12044 }
12045
12046 #[tokio::test]
12047 async fn admission_facts_without_configured_carrier_are_rejected() {
12048 let registry = Arc::new(Registry::default());
12049 let forwarding = Arc::new(ForwardingTable::default());
12050 let handler = ControlHandler::with_forwarding(registry, forwarding);
12051 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
12052 hello_via_sink(
12053 &handler,
12054 &target_ctx,
12055 &mut target_rx,
12056 hello_frame("target", PROTOCOL_VERSION, 1),
12057 )
12058 .await;
12059
12060 let responses = handler
12061 .handle_control_frame(
12062 &route_ctx(ConnectionId::new(79)).0,
12063 route_open_frame_with_admission_facts(
12064 16,
12065 "target",
12066 unique_project_root("admission-facts"),
12067 None,
12068 Some(json!({"x": 1})),
12069 ),
12070 )
12071 .await
12072 .unwrap();
12073 assert_eq!(
12074 parse_error(&responses[0])["code"],
12075 "admission_facts_not_permitted"
12076 );
12077 }
12078
12079 #[tokio::test]
12080 async fn route_open_relays_consumer_capabilities_verbatim() {
12081 let registry = Arc::new(Registry::default());
12082 let forwarding = Arc::new(ForwardingTable::default());
12083 let handler =
12084 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12085 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
12086 hello_via_sink(
12087 &handler,
12088 &module_ctx,
12089 &mut module_rx,
12090 hello_frame("aft", PROTOCOL_VERSION, 7),
12091 )
12092 .await;
12093
12094 let expected = vec!["elicitation".to_string(), "roots".to_string()];
12095 let expected_for_request = expected.clone();
12096 let project_root = unique_project_root("consumer-capabilities-present");
12097 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
12098 let route_handler = handler.clone();
12099 let route_task = tokio::spawn(async move {
12100 route_handler
12101 .handle_control_frame(
12102 &client_ctx,
12103 route_open_frame_with_consumer_capabilities(
12104 401,
12105 "aft",
12106 project_root,
12107 Some(expected_for_request),
12108 ),
12109 )
12110 .await
12111 .unwrap()
12112 });
12113 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12114 .await
12115 .unwrap()
12116 .unwrap();
12117 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12118 let ModuleControlRequest::RouteBind {
12119 consumer_capabilities,
12120 ..
12121 } = bind
12122 else {
12123 panic!("expected route.bind request, got {bind:?}");
12124 };
12125 assert_eq!(consumer_capabilities, Some(expected.clone()));
12126
12127 handler
12128 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12129 .await
12130 .unwrap();
12131 let route_response = route_task.await.unwrap();
12132 assert!(route_response.is_empty());
12133 let published = client_rx.recv().await.unwrap();
12134 assert!(matches!(
12135 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12136 ClientControlResponse::RouteOpen { .. }
12137 ));
12138 }
12139
12140 #[tokio::test]
12141 async fn route_open_without_consumer_capabilities_relays_none() {
12142 let registry = Arc::new(Registry::default());
12143 let forwarding = Arc::new(ForwardingTable::default());
12144 let handler =
12145 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12146 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
12147 hello_via_sink(
12148 &handler,
12149 &module_ctx,
12150 &mut module_rx,
12151 hello_frame("aft", PROTOCOL_VERSION, 7),
12152 )
12153 .await;
12154
12155 let project_root = unique_project_root("consumer-capabilities-absent");
12156 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
12157 let route_handler = handler.clone();
12158 let route_task = tokio::spawn(async move {
12159 route_handler
12160 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
12161 .await
12162 .unwrap()
12163 });
12164 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12165 .await
12166 .unwrap()
12167 .unwrap();
12168 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12169 let ModuleControlRequest::RouteBind {
12170 consumer_capabilities,
12171 ..
12172 } = bind
12173 else {
12174 panic!("expected route.bind request, got {bind:?}");
12175 };
12176 assert_eq!(consumer_capabilities, None);
12177
12178 handler
12179 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12180 .await
12181 .unwrap();
12182 let route_response = route_task.await.unwrap();
12183 assert!(route_response.is_empty());
12184 let published = client_rx.recv().await.unwrap();
12185 assert!(matches!(
12186 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12187 ClientControlResponse::RouteOpen { .. }
12188 ));
12189 }
12190
12191 async fn bind_role_versions_for(
12195 sent: Option<BTreeMap<String, String>>,
12196 connection: u64,
12197 ) -> Option<BTreeMap<String, String>> {
12198 let registry = Arc::new(Registry::default());
12199 let forwarding = Arc::new(ForwardingTable::default());
12200 let handler =
12201 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12202 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
12203 hello_via_sink(
12204 &handler,
12205 &module_ctx,
12206 &mut module_rx,
12207 hello_frame("aft", PROTOCOL_VERSION, 7),
12208 )
12209 .await;
12210 let project_root = unique_project_root("role-versions");
12211 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
12212 let route_handler = handler.clone();
12213 let route_task = tokio::spawn(async move {
12214 route_handler
12215 .handle_control_frame(
12216 &client_ctx,
12217 route_open_frame_with_role_versions(403, "aft", project_root, sent),
12218 )
12219 .await
12220 .unwrap()
12221 });
12222 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12223 .await
12224 .expect("a well-formed route.open reaches the module as a bind")
12225 .unwrap();
12226 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12227 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
12228 panic!("expected route.bind request, got {bind:?}");
12229 };
12230 handler
12231 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12232 .await
12233 .unwrap();
12234 assert!(route_task.await.unwrap().is_empty());
12235 let published = client_rx.recv().await.unwrap();
12236 assert!(matches!(
12237 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12238 ClientControlResponse::RouteOpen { .. }
12239 ));
12240 role_versions
12241 }
12242
12243 #[tokio::test]
12244 async fn route_open_relays_role_versions_verbatim() {
12245 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
12246 assert_eq!(
12247 bind_role_versions_for(Some(sent.clone()), 141).await,
12248 Some(sent)
12249 );
12250 }
12251
12252 #[tokio::test]
12255 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
12256 assert_eq!(bind_role_versions_for(None, 143).await, None);
12257 assert_eq!(
12258 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
12259 None
12260 );
12261 }
12262
12263 #[tokio::test]
12264 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
12265 let registry = Arc::new(Registry::default());
12266 let forwarding = Arc::new(ForwardingTable::default());
12267 let handler =
12268 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12269 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
12270 hello_via_sink(
12271 &handler,
12272 &module_ctx,
12273 &mut module_rx,
12274 hello_frame("aft", PROTOCOL_VERSION, 7),
12275 )
12276 .await;
12277
12278 let nine: BTreeMap<String, String> = (0..9)
12279 .map(|index| (format!("role-{index}"), "v1".to_string()))
12280 .collect();
12281 for (label, malformed) in [
12282 (
12283 "invalid role name",
12284 role_versions(&[("Tool_Provider", "v1")]),
12285 ),
12286 ("invalid version", role_versions(&[("tool-provider", "v0")])),
12287 ("nine entries", nine),
12288 ] {
12289 let responses = tokio::time::timeout(
12292 Duration::from_secs(1),
12293 handler.handle_control_frame(
12294 &route_ctx(ConnectionId::new(148)).0,
12295 route_open_frame_with_role_versions(
12296 404,
12297 "aft",
12298 unique_project_root("role-versions-malformed"),
12299 Some(malformed),
12300 ),
12301 ),
12302 )
12303 .await
12304 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
12305 .unwrap();
12306 assert_eq!(responses.len(), 1, "{label}");
12307 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
12308 let error = parse_error(&responses[0]);
12309 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
12310 assert_eq!(
12311 error["detail"]["field"], "role_versions",
12312 "{label}: {error}"
12313 );
12314 assert!(
12315 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
12316 "{label}: a malformed declaration is terminal"
12317 );
12318 assert!(
12319 module_rx.try_recv().is_err(),
12320 "{label}: the module must never see a bind"
12321 );
12322 }
12323 }
12324
12325 #[tokio::test]
12329 async fn route_role_versions_capability_is_advertised() {
12330 let handler = ControlHandler::new(Arc::new(Registry::default()));
12331 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
12332 let ack = hello_via_sink(
12333 &handler,
12334 &ctx,
12335 &mut rx,
12336 hello_frame("m", PROTOCOL_VERSION, 1),
12337 )
12338 .await;
12339 let ack = parse_ack(&ack);
12340 assert!(
12341 ack.subc_capabilities
12342 .iter()
12343 .any(|c| c == "route-role-versions/v1"),
12344 "{:?}",
12345 ack.subc_capabilities
12346 );
12347
12348 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12349 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12350 let reply = handler
12351 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
12352 .await
12353 .unwrap()
12354 .pop()
12355 .unwrap();
12356 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12357 serde_json::from_slice(&reply.body).unwrap()
12358 else {
12359 panic!("not a server.describe reply");
12360 };
12361 assert!(
12362 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
12363 "{capabilities:?}"
12364 );
12365 }
12366
12367 #[tokio::test]
12371 async fn catalog_events_capability_is_advertised() {
12372 let handler = ControlHandler::new(Arc::new(Registry::default()));
12373 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12374 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12375 let reply = handler
12376 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
12377 .await
12378 .unwrap()
12379 .pop()
12380 .unwrap();
12381 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12382 serde_json::from_slice(&reply.body).unwrap()
12383 else {
12384 panic!("not a server.describe reply");
12385 };
12386 assert!(
12387 capabilities
12388 .iter()
12389 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
12390 "{capabilities:?}"
12391 );
12392 }
12393
12394 #[tokio::test]
12395 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
12396 let handler = ControlHandler::new(Arc::new(Registry::default()));
12397 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
12398 let ack = hello_via_sink(
12399 &handler,
12400 &module,
12401 &mut module_rx,
12402 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
12403 )
12404 .await;
12405 let ack = parse_ack(&ack);
12406 assert!(
12407 ack.subc_capabilities
12408 .iter()
12409 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12410 "{:?}",
12411 ack.subc_capabilities
12412 );
12413 assert!(ack
12414 .subc_capabilities
12415 .iter()
12416 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12417
12418 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
12419 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12420 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12421 let reply = handler
12422 .handle_control_frame(&client, frame)
12423 .await
12424 .unwrap()
12425 .pop()
12426 .unwrap();
12427 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12428 serde_json::from_slice(&reply.body).unwrap()
12429 else {
12430 panic!("not a server.describe reply");
12431 };
12432 assert!(
12433 capabilities
12434 .iter()
12435 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12436 "{capabilities:?}"
12437 );
12438 assert!(capabilities
12439 .iter()
12440 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12441 }
12442
12443 #[tokio::test]
12444 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
12445 let handler = ControlHandler::new(Arc::new(Registry::default()));
12446 let (module, mut rx) = route_ctx(ConnectionId::new(156));
12447 let ack = hello_via_sink(
12448 &handler,
12449 &module,
12450 &mut rx,
12451 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
12452 )
12453 .await;
12454 assert!(parse_ack(&ack)
12455 .subc_capabilities
12456 .iter()
12457 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12458 assert!(parse_ack(&ack)
12459 .subc_capabilities
12460 .iter()
12461 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12462 let (client, _rx) = route_ctx(ConnectionId::new(157));
12463 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12464 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12465 let reply = handler
12466 .handle_control_frame(&client, frame)
12467 .await
12468 .unwrap()
12469 .pop()
12470 .unwrap();
12471 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12472 serde_json::from_slice(&reply.body).unwrap()
12473 else {
12474 panic!("server.describe reply")
12475 };
12476 assert!(capabilities
12477 .iter()
12478 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12479 assert!(capabilities
12480 .iter()
12481 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12482 }
12483
12484 #[tokio::test]
12485 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
12486 let handler = ControlHandler::new(Arc::new(Registry::default()));
12487 let (module, mut rx) = route_ctx(ConnectionId::new(158));
12488 let ack = hello_via_sink(
12489 &handler,
12490 &module,
12491 &mut rx,
12492 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
12493 )
12494 .await;
12495 assert!(parse_ack(&ack)
12496 .subc_capabilities
12497 .iter()
12498 .any(|cap| cap == "scope-workdir/v1"));
12499 let (client, _rx) = route_ctx(ConnectionId::new(159));
12500 let frame = Frame::build(
12501 FrameType::Request,
12502 control_flags(),
12503 0,
12504 0,
12505 5,
12506 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12507 )
12508 .unwrap();
12509 let reply = handler
12510 .handle_control_frame(&client, frame)
12511 .await
12512 .unwrap()
12513 .remove(0);
12514 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12515 serde_json::from_slice(&reply.body).unwrap()
12516 else {
12517 panic!("server.describe reply")
12518 };
12519 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
12520 }
12521
12522 #[tokio::test]
12523 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
12524 let handler = ControlHandler::new(Arc::new(Registry::default()));
12525 let (module, mut rx) = route_ctx(ConnectionId::new(154));
12526 let ack = hello_via_sink(
12527 &handler,
12528 &module,
12529 &mut rx,
12530 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
12531 )
12532 .await;
12533 assert!(parse_ack(&ack)
12534 .subc_capabilities
12535 .iter()
12536 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12537 let (client, _rx) = route_ctx(ConnectionId::new(155));
12538 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12539 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12540 let reply = handler
12541 .handle_control_frame(&client, frame)
12542 .await
12543 .unwrap()
12544 .pop()
12545 .unwrap();
12546 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12547 serde_json::from_slice(&reply.body).unwrap()
12548 else {
12549 panic!("not a server.describe reply")
12550 };
12551 assert!(capabilities
12552 .iter()
12553 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12554 }
12555
12556 #[tokio::test]
12557 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
12558 let registry = Arc::new(Registry::default());
12559 let forwarding = Arc::new(ForwardingTable::default());
12560 let handler =
12561 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12562 .with_health_probe_timeout(Duration::from_secs(5));
12563 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
12564 hello_via_sink(
12565 &handler,
12566 &module_ctx,
12567 &mut module_rx,
12568 non_routable_hello_frame_with_control_ops(
12569 "mcp",
12570 300,
12571 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12572 ),
12573 )
12574 .await;
12575 assert!(registry
12576 .get_module("mcp")
12577 .unwrap()
12578 .unwrap()
12579 .manifest
12580 .provides
12581 .is_empty());
12582
12583 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
12584 let route_response = handler
12585 .handle_control_frame(
12586 &route_client_ctx,
12587 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
12588 )
12589 .await
12590 .unwrap();
12591 assert_eq!(route_response[0].header.ty, FrameType::Error);
12592 assert_eq!(
12593 parse_error(&route_response[0])["code"],
12594 "target_unavailable"
12595 );
12596 assert!(parse_error(&route_response[0])["message"]
12597 .as_str()
12598 .unwrap()
12599 .contains("does not provide the requested target"));
12600 assert!(module_rx.try_recv().is_err());
12601
12602 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
12603 let health_handler = handler.clone();
12604 let health_task = tokio::spawn(async move {
12605 health_handler
12606 .handle_control_frame(
12607 &health_client_ctx,
12608 supervisor_health_probe_frame(302, "mcp"),
12609 )
12610 .await
12611 .unwrap()
12612 });
12613 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12614 .await
12615 .unwrap()
12616 .unwrap();
12617 assert_eq!(
12618 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
12619 ModuleControlRequest::HealthCheck {}
12620 );
12621 handler
12622 .handle_control_frame(
12623 &module_ctx,
12624 health_response(health_frame.header.corr, HealthStatus::Ok),
12625 )
12626 .await
12627 .unwrap();
12628 let health_response = health_task.await.unwrap();
12629 assert_eq!(health_response[0].header.ty, FrameType::Response);
12630 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
12631 ClientControlResponse::SupervisorHealthProbe {
12632 module_id, status, ..
12633 } => {
12634 assert_eq!(module_id, "mcp");
12635 assert_eq!(status, HealthStatus::Ok);
12636 }
12637 other => panic!("unexpected health response: {other:?}"),
12638 }
12639
12640 forwarding
12645 .cleanup_connection(module_ctx.connection_id)
12646 .unwrap();
12647 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
12648 let cleanup_response = tokio::time::timeout(
12649 Duration::from_millis(200),
12650 handler.handle_control_frame(
12651 &cleanup_probe_ctx,
12652 supervisor_health_probe_frame(303, "mcp"),
12653 ),
12654 )
12655 .await
12656 .expect("probe should fail immediately when the control lane is gone")
12657 .unwrap();
12658 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
12659 assert_eq!(
12660 parse_error(&cleanup_response[0])["code"],
12661 "target_unavailable"
12662 );
12663 assert!(parse_error(&cleanup_response[0])["message"]
12664 .as_str()
12665 .unwrap()
12666 .contains("no module connection"));
12667
12668 handler
12669 .cleanup_connection(module_ctx.connection_id)
12670 .unwrap();
12671 }
12672
12673 #[tokio::test]
12674 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
12675 let registry = Arc::new(Registry::default());
12676 let supervisor_handle = SupervisorHandle::new();
12677 let supervisor =
12678 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12679 .with_handle(supervisor_handle.clone())
12680 .with_connection_file_path(
12681 std::env::temp_dir()
12682 .join(format!("subc-route-open-warming-{}", std::process::id())),
12683 );
12684 let module = supervisor
12685 .supervise_configured(
12686 ModuleSpec {
12687 module_id: "warming".to_string(),
12688 program: fake_aft_stub_path(),
12689 args: Vec::new(),
12690 env: Vec::new(),
12691 reserved: false,
12692 reserved_prefixes: Vec::new(),
12693 protocol: ModuleProtocol::Subc,
12694 overlap: Default::default(),
12695 },
12696 true,
12697 )
12698 .unwrap();
12699 assert_eq!(module.state().unwrap(), ModuleState::Running);
12700
12701 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12702 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
12703 let response = handler
12704 .handle_control_frame(
12705 &ctx,
12706 route_open_frame(304, "warming", unique_project_root("warming")),
12707 )
12708 .await
12709 .unwrap();
12710 module.stop().await.unwrap();
12711
12712 assert_eq!(response[0].header.ty, FrameType::Error);
12713 let error = parse_error(&response[0]);
12714 assert_eq!(error["code"], "module_warming");
12715 assert!(error["message"]
12716 .as_str()
12717 .unwrap()
12718 .contains("state=running, enabled=true, live=false"));
12719 }
12720
12721 #[test]
12722 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
12723 let handler = ControlHandler::new(Arc::new(Registry::default()));
12724 let capture = EventCapture::default();
12725 let _subscriber =
12726 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12727 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
12728 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
12729 let pending = (0..limit).collect::<Vec<_>>();
12730 let response = handler
12731 .route_open_capacity_refusal(
12732 &ctx,
12733 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
12734 "busy",
12735 pending.len(),
12736 limit,
12737 )
12738 .unwrap();
12739 assert_eq!(parse_error(&response)["code"], "target_unavailable");
12740 let event = capture
12741 .events()
12742 .into_iter()
12743 .find(|event| {
12744 event.target == "control"
12745 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
12746 })
12747 .expect("connection admission refusal event");
12748 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12749 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12750 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
12753 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
12754 }
12755
12756 #[test]
12757 fn route_open_target_cap_logs_admission_reason_and_capacity() {
12758 let handler = ControlHandler::new(Arc::new(Registry::default()));
12759 let capture = EventCapture::default();
12760 let _subscriber =
12761 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12762 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
12763 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
12764 let guards = (0..limit)
12765 .map(|_| {
12766 handler
12767 .route_bind_concurrency
12768 .try_admit("busy", limit)
12769 .unwrap()
12770 })
12771 .collect::<Vec<_>>();
12772 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
12773 Err(in_flight) => in_flight,
12774 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
12775 };
12776 let response = handler
12777 .route_open_target_capacity_refusal(
12778 &ctx,
12779 &RouteOpenCaller::unknown(),
12780 &route_open_frame(397, "busy", unique_project_root("target-cap")),
12781 "busy",
12782 in_flight,
12783 )
12784 .unwrap();
12785 assert_eq!(parse_error(&response)["code"], "target_unavailable");
12786 let event = capture
12787 .events()
12788 .into_iter()
12789 .find(|event| {
12790 event.target == "control"
12791 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
12792 })
12793 .expect("target admission refusal event");
12794 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12795 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12796 drop(guards);
12797 }
12798
12799 #[cfg(unix)]
12800 #[tokio::test(start_paused = true)]
12801 async fn supervisor_list_shows_a_failed_modules_next_retry() {
12802 let registry = Arc::new(Registry::default());
12803 let handle = SupervisorHandle::new();
12804 let supervisor =
12805 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
12806 .with_handle(handle.clone());
12807 let module = supervisor
12808 .spawn(crate::ModuleSpec {
12809 module_id: "retry-list".into(),
12810 program: PathBuf::from("/bin/sh"),
12811 args: vec!["-c".into(), "exit 17".into()],
12812 env: vec![],
12813 reserved: false,
12814 reserved_prefixes: vec![],
12815 protocol: ModuleProtocol::None,
12816 overlap: Default::default(),
12817 })
12818 .unwrap();
12819 let deadline = std::time::Instant::now() + Duration::from_secs(10);
12820 while module.state().unwrap() != crate::ModuleState::Failed {
12821 assert!(
12822 std::time::Instant::now() < deadline,
12823 "waiting for budget failure"
12824 );
12825 tokio::task::yield_now().await;
12826 }
12827 let handler = ControlHandler::new(registry).with_supervisor(handle);
12828 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12829 let replies = handler.handle_supervisor_list(frame).await.unwrap();
12830 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12831 assert_eq!(body["modules"][0]["state"], "failed");
12832 let retry = body["modules"][0]["next_retry_at_ms"]
12833 .as_u64()
12834 .expect("scheduled retry on the wire");
12835 let now = std::time::SystemTime::now()
12836 .duration_since(std::time::UNIX_EPOCH)
12837 .unwrap()
12838 .as_millis() as u64;
12839 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
12840 module.set_enabled(false).await.unwrap();
12841 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
12842 let replies = handler.handle_supervisor_list(frame).await.unwrap();
12843 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12844 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
12845 }
12846
12847 #[tokio::test]
12848 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
12849 let registry = Arc::new(Registry::default());
12850 let handle = SupervisorHandle::new();
12851 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
12852 .with_handle(handle.clone());
12853 supervisor
12854 .supervise_configured(
12855 crate::ModuleSpec {
12856 module_id: "drift".into(),
12857 program: PathBuf::from("disabled-module"),
12858 args: Vec::new(),
12859 env: Vec::new(),
12860 reserved: false,
12861 reserved_prefixes: Vec::new(),
12862 protocol: ModuleProtocol::Subc,
12863 overlap: Default::default(),
12864 },
12865 false,
12866 )
12867 .unwrap();
12868 let handler = ControlHandler::new(registry).with_supervisor(handle);
12869 for configured in [None, Some(false), Some(true)] {
12870 handler.capability_evaluator.configure(
12871 configured.map(|enabled| ("drift".to_string(), enabled)),
12872 BTreeMap::new(),
12873 );
12874 let frame =
12875 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12876 let replies = handler
12877 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
12878 .await
12879 .unwrap();
12880 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12881 assert!(body.get("configured_enabled").is_none());
12882 assert_eq!(body["op"], "supervisor.ack");
12883 let frame =
12884 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12885 let replies = handler.handle_supervisor_list(frame).await.unwrap();
12886 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12887 assert_eq!(
12888 body["modules"][0].get("configured_enabled"),
12889 configured.map(Value::Bool).as_ref()
12890 );
12891 }
12892 }
12893
12894 #[tokio::test(flavor = "current_thread")]
12895 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
12896 let handler = ControlHandler::new(Arc::new(Registry::default()));
12897 let capture = EventCapture::default();
12898 let _subscriber =
12899 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12900 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
12901 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12902
12903 let _ = handler
12904 .handle_client_control_request(
12905 &ctx,
12906 frame,
12907 ClientControlRequest::SupervisorSetEnabled {
12908 module_id: "broca".to_string(),
12909 enabled: false,
12910 },
12911 &mut None,
12912 )
12913 .await
12914 .unwrap();
12915
12916 let events = capture
12917 .events()
12918 .into_iter()
12919 .filter(|event| {
12920 event.target == "control"
12921 && event.fields.get("message").map(String::as_str)
12922 == Some("supervisor request received")
12923 })
12924 .collect::<Vec<_>>();
12925 assert_eq!(events.len(), 1, "one supervisor request log line");
12926 let event = &events[0];
12927 assert_eq!(event.level, tracing::Level::INFO);
12928 assert_eq!(
12929 event.fields.get("op"),
12930 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
12931 );
12932 assert_eq!(
12933 event.fields.get("module_id"),
12934 Some(&"\"broca\"".to_string())
12935 );
12936 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
12937 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
12938 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
12939 }
12940
12941 #[tokio::test(flavor = "current_thread")]
12942 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
12943 let registry = Arc::new(Registry::default());
12944 let module_connection = ConnectionId::new(303);
12945 registry
12946 .register_with_control_ops(
12947 manifest("aft", PROTOCOL_VERSION),
12948 PROTOCOL_VERSION,
12949 module_connection,
12950 module_baseline_control_ops(),
12951 )
12952 .unwrap();
12953 let handler = ControlHandler::new(registry);
12954 let capture = EventCapture::default();
12955 let _subscriber =
12956 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12957 let (ctx, _rx) = route_ctx(module_connection);
12958 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12959
12960 let _ = handler
12961 .handle_client_control_request(
12962 &ctx,
12963 frame,
12964 ClientControlRequest::SupervisorSetEnabled {
12965 module_id: "broca".to_string(),
12966 enabled: false,
12967 },
12968 &mut None,
12969 )
12970 .await
12971 .unwrap();
12972
12973 let callers = capture
12974 .events()
12975 .into_iter()
12976 .filter(|event| {
12977 event.fields.get("message").map(String::as_str)
12978 == Some("supervisor request received")
12979 })
12980 .map(|event| event.fields.get("caller").cloned())
12981 .collect::<Vec<_>>();
12982 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
12983 }
12984
12985 #[tokio::test(flavor = "current_thread")]
12986 async fn supervisor_list_does_not_log_request_received() {
12987 let handler = ControlHandler::new(Arc::new(Registry::default()));
12988 let capture = EventCapture::default();
12989 let _subscriber =
12990 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12991 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
12992 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12993
12994 handler
12995 .handle_client_control_request(
12996 &ctx,
12997 frame,
12998 ClientControlRequest::SupervisorList {},
12999 &mut None,
13000 )
13001 .await
13002 .unwrap();
13003
13004 assert!(capture.events().into_iter().all(|event| {
13005 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13006 }));
13007 }
13008
13009 #[tokio::test(flavor = "current_thread")]
13010 async fn supervisor_rescan_preview_does_not_log_request_received() {
13011 let handler = ControlHandler::new(Arc::new(Registry::default()));
13012 let capture = EventCapture::default();
13013 let _subscriber =
13014 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13015 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
13016 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13017
13018 handler
13019 .handle_client_control_request(
13020 &ctx,
13021 frame,
13022 ClientControlRequest::SupervisorRescan { preview: true },
13023 &mut None,
13024 )
13025 .await
13026 .unwrap();
13027
13028 assert!(capture.events().into_iter().all(|event| {
13029 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13030 }));
13031 }
13032
13033 struct ApproveOperator;
13034 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
13035 fn prompt(
13036 &self,
13037 _: &str,
13038 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
13039 ) -> crate::operator_confirm::ProviderResult {
13040 crate::operator_confirm::ProviderResult::Approved
13041 }
13042 }
13043
13044 #[tokio::test]
13045 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
13046 assert_eq!(
13047 ControlHandler::default().privileged_operator_prompt,
13048 PrivilegedOperatorPrompt::Log
13049 );
13050 for (initial, changed, running) in [
13051 ("log", "enforce", PrivilegedOperatorPrompt::Log),
13052 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
13053 ] {
13054 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
13055 let path = dir.join("subc.jsonc");
13056 std::fs::write(
13057 &path,
13058 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
13059 )
13060 .unwrap();
13061 let config = crate::daemon_config::load(&path).unwrap().unwrap();
13062 let registry = Arc::new(Registry::default());
13063 let handle = SupervisorHandle::new();
13064 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
13065 .with_handle(handle.clone());
13066 let handler = ControlHandler::new(registry)
13067 .with_supervisor(handle)
13068 .with_privileged_operator_prompt(config.privileged_operator_prompt)
13069 .with_supervisor_rescan(supervisor, &path, config.port);
13070 handler.forwarding.operator_confirms().configure(
13071 Arc::new(ApproveOperator),
13072 crate::operator_confirm::OperatorLimits::default(),
13073 );
13074 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
13075
13076 for (disk_mode, needs_restart) in [
13077 (Some(initial), false),
13078 (Some(changed), true),
13079 (Some(changed), true),
13080 (None, running == PrivilegedOperatorPrompt::Enforce),
13081 (Some(initial), false),
13082 ] {
13083 let mut doc = json!({"version":1});
13084 if let Some(mode) = disk_mode {
13085 doc["privileged_operator_prompt"] = json!(mode);
13086 }
13087 std::fs::write(&path, doc.to_string()).unwrap();
13088 for preview in [true, false] {
13089 let request = ClientControlRequest::SupervisorRescan { preview };
13090 let frame = Frame::build(
13091 FrameType::Request,
13092 control_flags(),
13093 0,
13094 0,
13095 1,
13096 serde_json::to_vec(&request).unwrap(),
13097 )
13098 .unwrap();
13099 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13100 assert_eq!(frames.len(), 1);
13101 let ClientControlResponse::SupervisorRescan { result } =
13102 serde_json::from_slice(&frames[0].body).unwrap()
13103 else {
13104 panic!("expected a rescan response");
13105 };
13106 let expected = if needs_restart {
13107 vec!["privileged_operator_prompt"]
13108 } else {
13109 Vec::new()
13110 };
13111 assert_eq!(result.restart_required, expected);
13112 assert_eq!(handler.privileged_operator_prompt, running);
13113 assert_eq!(
13114 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
13115 running
13116 );
13117 }
13118 }
13119 }
13120 }
13121
13122 #[tokio::test]
13123 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
13124 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
13125 let path = dir.join("subc.jsonc");
13126 let registry = Arc::new(Registry::default());
13127 let handle = SupervisorHandle::new();
13128 let supervisor =
13129 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
13130 let handler = ControlHandler::new(registry)
13131 .with_supervisor(handle)
13132 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
13133 .with_supervisor_rescan(supervisor, &path, None);
13134 handler.forwarding.operator_confirms().configure(
13135 Arc::new(ApproveOperator),
13136 crate::operator_confirm::OperatorLimits::default(),
13137 );
13138 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
13139 for preview in [true, false] {
13140 std::fs::write(
13141 &path,
13142 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
13143 )
13144 .unwrap();
13145 let request = ClientControlRequest::SupervisorRescan { preview };
13146 let frame = Frame::build(
13147 FrameType::Request,
13148 control_flags(),
13149 0,
13150 0,
13151 1,
13152 serde_json::to_vec(&request).unwrap(),
13153 )
13154 .unwrap();
13155 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13156 assert_eq!(frames.len(), 1);
13157 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
13158 assert_eq!(
13159 handler.privileged_operator_prompt,
13160 PrivilegedOperatorPrompt::Enforce
13161 );
13162 }
13163 }
13164
13165 #[tokio::test]
13170 async fn route_open_refusal_names_the_check_that_refused() {
13171 let handler = ControlHandler::new(Arc::new(Registry::default()));
13172 let capture = EventCapture::default();
13173 let _subscriber =
13174 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13175 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13176 let response = handler
13177 .handle_control_frame(
13178 &ctx,
13179 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
13180 )
13181 .await
13182 .unwrap();
13183
13184 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13185 let event = capture
13186 .events()
13187 .into_iter()
13188 .find(|event| {
13189 event.target == "control"
13190 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13191 })
13192 .expect("route.open refusal event");
13193 assert_eq!(
13194 event.fields.get("reason"),
13195 Some(&"\"not_registered\"".to_string())
13196 );
13197 }
13198
13199 #[tokio::test]
13200 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
13201 let registry = Arc::new(Registry::default());
13202 let supervisor_handle = SupervisorHandle::new();
13203 let supervisor =
13204 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13205 .with_handle(supervisor_handle.clone())
13206 .with_connection_file_path(std::env::temp_dir().join(format!(
13207 "subc-route-open-refusal-info-{}",
13208 std::process::id()
13209 )));
13210 let module = supervisor
13211 .supervise_configured(
13212 ModuleSpec {
13213 module_id: "warming".to_string(),
13214 program: fake_aft_stub_path(),
13215 args: Vec::new(),
13216 env: Vec::new(),
13217 reserved: false,
13218 reserved_prefixes: Vec::new(),
13219 protocol: ModuleProtocol::Subc,
13220 overlap: Default::default(),
13221 },
13222 true,
13223 )
13224 .unwrap();
13225 assert_eq!(module.state().unwrap(), ModuleState::Running);
13226
13227 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13228 assert!(handler
13229 .counters()
13230 .snapshot()
13231 .get("route_open_refused_by_code")
13232 .is_none());
13233 let capture = EventCapture::default();
13234 let _subscriber =
13235 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13236 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
13237 let response = handler
13238 .handle_control_frame(
13239 &ctx,
13240 route_open_frame(394, "warming", unique_project_root("refusal-info")),
13241 )
13242 .await
13243 .unwrap();
13244 module.stop().await.unwrap();
13245
13246 assert_eq!(parse_error(&response[0])["code"], "module_warming");
13247 let event = capture
13248 .events()
13249 .into_iter()
13250 .find(|event| {
13251 event.target == "control"
13252 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
13253 })
13254 .expect("route.open refusal event");
13255 assert_eq!(
13256 event.fields.get("module_id"),
13257 Some(&"\"warming\"".to_string())
13258 );
13259 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
13260 assert_eq!(
13261 event.fields.get("reason"),
13262 Some(&"\"supervised_not_registered\"".to_string())
13263 );
13264 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
13265 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
13266 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
13267 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13268 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13269 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
13270 assert_eq!(
13271 handler.counters().snapshot()["route_open_refused_by_code"],
13272 json!({ "module_warming": 1 })
13273 );
13274 }
13275
13276 const OUTAGE_START: &str = "route.open refusing module: not serving";
13277 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
13278
13279 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
13280 capture
13281 .events()
13282 .into_iter()
13283 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
13284 .collect()
13285 }
13286
13287 fn supervise_stub(
13288 registry: &Arc<Registry>,
13289 module_id: &str,
13290 enabled: bool,
13291 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
13292 let supervisor_handle = SupervisorHandle::new();
13293 let supervisor =
13294 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
13295 .with_handle(supervisor_handle.clone())
13296 .with_connection_file_path(std::env::temp_dir().join(format!(
13297 "subc-route-outage-{module_id}-{}",
13298 std::process::id()
13299 )));
13300 let module = supervisor
13301 .supervise_configured(
13302 ModuleSpec {
13303 module_id: module_id.to_string(),
13304 program: fake_aft_stub_path(),
13305 args: Vec::new(),
13306 env: Vec::new(),
13307 reserved: false,
13308 reserved_prefixes: Vec::new(),
13309 protocol: ModuleProtocol::Subc,
13310 overlap: Default::default(),
13311 },
13312 enabled,
13313 )
13314 .unwrap();
13315 (supervisor_handle, module)
13316 }
13317
13318 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
13319 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
13320 module_id: module_id.to_string(),
13321 drain_timeout_ms: Some(50),
13322 })
13323 .unwrap();
13324 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
13325 }
13326
13327 #[test]
13331 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
13332 let registry = Arc::new(Registry::default());
13333 let forwarding = Arc::new(ForwardingTable::default());
13334 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13335 let second = ControlHandler::with_forwarding(registry, forwarding);
13336 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
13337 }
13338
13339 #[tokio::test(flavor = "current_thread")]
13343 async fn route_open_unknown_module_refusals_add_no_outage_state() {
13344 let handler = ControlHandler::new(Arc::new(Registry::default()));
13345 let capture = EventCapture::default();
13346 let _subscriber =
13347 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13348 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
13349 for corr in 0..8 {
13350 let response = handler
13351 .handle_control_frame(
13352 &ctx,
13353 route_open_frame(
13354 380 + corr,
13355 &format!("nobody-{corr}"),
13356 unique_project_root("outage-unknown"),
13357 ),
13358 )
13359 .await
13360 .unwrap();
13361 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13362 }
13363
13364 assert_eq!(handler.route_outages.tracked_module_count(), 0);
13365 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
13366 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
13367 }
13368
13369 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13374 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
13375 let registry = Arc::new(Registry::default());
13376 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
13377 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13378 let capture = EventCapture::default();
13379 let _subscriber =
13380 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13381 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
13382 handler.route_outages.record_accepted("outage-restart");
13385
13386 let response = handler
13387 .handle_control_frame(
13388 &ctx,
13389 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
13390 )
13391 .await
13392 .unwrap();
13393 assert_eq!(response[0].header.ty, FrameType::Error);
13394 let starts = outage_lines(&capture, OUTAGE_START);
13395 assert_eq!(starts.len(), 1, "{starts:?}");
13396 assert_eq!(starts[0].level, tracing::Level::WARN);
13397 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
13398 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
13399 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
13400 handler.route_outages.record_accepted("outage-restart");
13401 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
13402
13403 let restart = handler
13404 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
13405 .await
13406 .unwrap();
13407 assert_eq!(
13408 restart[0].header.ty,
13409 FrameType::Response,
13410 "{:?}",
13411 parse_error(&restart[0])
13412 );
13413 handler
13414 .handle_control_frame(
13415 &ctx,
13416 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
13417 )
13418 .await
13419 .unwrap();
13420 module.stop().await.unwrap();
13421
13422 let starts = outage_lines(&capture, OUTAGE_START);
13423 assert_eq!(starts.len(), 2, "{starts:?}");
13424 assert_eq!(starts[1].level, tracing::Level::INFO);
13425 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
13426 }
13427
13428 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13432 async fn failed_operator_restart_leaves_no_operator_mark() {
13433 let registry = Arc::new(Registry::default());
13434 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
13435 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13436 let capture = EventCapture::default();
13437 let _subscriber =
13438 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13439 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
13440 handler.route_outages.record_accepted("outage-disabled");
13441
13442 let restart = handler
13443 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
13444 .await
13445 .unwrap();
13446 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
13447 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
13448
13449 handler
13450 .handle_control_frame(
13451 &ctx,
13452 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
13453 )
13454 .await
13455 .unwrap();
13456 let starts = outage_lines(&capture, OUTAGE_START);
13457 assert_eq!(starts.len(), 1, "{starts:?}");
13458 assert_eq!(starts[0].level, tracing::Level::WARN);
13459 }
13460
13461 #[tokio::test(flavor = "current_thread")]
13462 async fn route_open_unknown_module_escapes_target_module_id() {
13463 let handler = ControlHandler::new(Arc::new(Registry::default()));
13464 let capture = EventCapture::default();
13465 let _subscriber =
13466 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13467 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
13468 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13469 let response = handler
13470 .handle_control_frame(
13471 &ctx,
13472 route_open_frame(
13473 395,
13474 hostile_module_id,
13475 unique_project_root("hostile-target-module-id"),
13476 ),
13477 )
13478 .await
13479 .unwrap();
13480
13481 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13482 let event = capture
13483 .events()
13484 .into_iter()
13485 .find(|event| {
13486 event.target == "control"
13487 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13488 })
13489 .expect("route.open unknown-module refusal event");
13490 let logged = event.fields.get("module_id").expect("module_id field");
13491 assert!(!logged.bytes().any(|byte| byte < 0x20));
13492 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13493 }
13494
13495 #[tokio::test(flavor = "current_thread")]
13496 async fn route_open_module_rejection_uses_daemon_counter_key() {
13497 let registry = Arc::new(Registry::default());
13498 let forwarding = Arc::new(ForwardingTable::default());
13499 let handler =
13500 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13501 let module_connection = ConnectionId::new(95);
13502 let (module_ctx, mut module_rx) = route_ctx(module_connection);
13503 hello_via_sink(
13504 &handler,
13505 &module_ctx,
13506 &mut module_rx,
13507 hello_frame("aft", PROTOCOL_VERSION, 395),
13508 )
13509 .await;
13510
13511 let client_connection = ConnectionId::new(96);
13512 let (client_ctx, _client_rx) = route_ctx(client_connection);
13513 let capture = EventCapture::default();
13514 let _subscriber =
13515 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13516 let (route_task, bind) = relay_route_open(
13517 &handler,
13518 client_connection,
13519 &client_ctx.egress,
13520 &mut module_rx,
13521 396,
13522 "aft",
13523 "hostile-module-code",
13524 )
13525 .await;
13526 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
13527 let rejection = Frame::build(
13528 FrameType::Error,
13529 control_flags(),
13530 0,
13531 0,
13532 bind.header.corr,
13533 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
13534 )
13535 .unwrap();
13536 handler
13537 .handle_control_frame(&module_ctx, rejection)
13538 .await
13539 .unwrap();
13540
13541 let response = route_task.await.unwrap();
13542 assert_eq!(parse_error(&response[0])["code"], hostile_code);
13543 let counters = handler.counters().snapshot();
13544 assert_eq!(
13545 counters["route_open_refused_by_code"],
13546 json!({ "module_rejected": 1 })
13547 );
13548 assert!(counters["route_open_refused_by_code"]
13549 .get(hostile_code)
13550 .is_none());
13551
13552 let event = capture
13553 .events()
13554 .into_iter()
13555 .find(|event| {
13556 event.target == "control"
13557 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
13558 })
13559 .expect("route.open module-rejection refusal event");
13560 let logged = event.fields.get("module_code").expect("module_code field");
13561 assert!(!logged.bytes().any(|byte| byte < 0x20));
13562 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13563 }
13564
13565 #[tokio::test]
13566 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
13567 let registry = Arc::new(Registry::default());
13568 let supervisor_handle = SupervisorHandle::new();
13569 let missing_program = std::env::temp_dir().join(format!(
13570 "subc-route-open-missing-program-{}",
13571 std::process::id()
13572 ));
13573 let supervisor =
13574 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13575 .with_handle(supervisor_handle.clone());
13576 let module = supervisor
13577 .supervise_configured(
13578 ModuleSpec {
13579 module_id: "failed".to_string(),
13580 program: missing_program,
13581 args: Vec::new(),
13582 env: Vec::new(),
13583 reserved: false,
13584 reserved_prefixes: Vec::new(),
13585 protocol: ModuleProtocol::Subc,
13586 overlap: Default::default(),
13587 },
13588 true,
13589 )
13590 .unwrap();
13591 assert_eq!(module.state().unwrap(), ModuleState::Failed);
13592
13593 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13594 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
13595 let response = handler
13596 .handle_control_frame(
13597 &ctx,
13598 route_open_frame(305, "failed", unique_project_root("failed")),
13599 )
13600 .await
13601 .unwrap();
13602
13603 assert_eq!(response[0].header.ty, FrameType::Error);
13604 let error = parse_error(&response[0]);
13605 assert_eq!(error["code"], "target_unavailable");
13606 assert!(error["message"]
13607 .as_str()
13608 .unwrap()
13609 .contains("state=failed, enabled=true, live=false"));
13610 }
13611
13612 #[tokio::test]
13613 async fn route_open_role_mismatch_remains_target_unavailable() {
13614 let registry = Arc::new(Registry::default());
13615 let handler = ControlHandler::new(Arc::clone(®istry));
13616 handler
13617 .handle_control(
13618 ConnectionId::new(41),
13619 non_routable_hello_frame_with_control_ops("health-only", 306, None),
13620 )
13621 .unwrap();
13622
13623 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
13624 let response = handler
13625 .handle_control_frame(
13626 &ctx,
13627 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
13628 )
13629 .await
13630 .unwrap();
13631
13632 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13633 assert!(parse_error(&response[0])["message"]
13634 .as_str()
13635 .unwrap()
13636 .contains("does not provide the requested target"));
13637 }
13638
13639 #[tokio::test]
13640 async fn route_open_inactive_registration_remains_target_unavailable() {
13641 let registry = Arc::new(Registry::default());
13642 let handler = ControlHandler::new(Arc::clone(®istry));
13643 handler
13644 .handle_control(
13645 ConnectionId::new(43),
13646 hello_frame("inactive", PROTOCOL_VERSION, 308),
13647 )
13648 .unwrap();
13649 assert!(registry
13650 .set_module_state_for_test("inactive", ChannelState::Closed)
13651 .unwrap());
13652
13653 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
13654 let response = handler
13655 .handle_control_frame(
13656 &ctx,
13657 route_open_frame(309, "inactive", unique_project_root("inactive")),
13658 )
13659 .await
13660 .unwrap();
13661
13662 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13663 assert!(parse_error(&response[0])["message"]
13664 .as_str()
13665 .unwrap()
13666 .contains("is not active"));
13667 }
13668
13669 #[tokio::test]
13670 async fn late_health_reply_is_recorded_through_the_module_response_path() {
13671 let registry = Arc::new(Registry::default());
13672 let forwarding = Arc::new(ForwardingTable::default());
13673 let supervisor_handle = SupervisorHandle::new();
13674 let supervisor =
13675 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
13676 .with_forwarding(Arc::clone(&forwarding))
13677 .with_handle(supervisor_handle.clone());
13678 let module = supervisor
13679 .supervise_configured(
13680 crate::ModuleSpec {
13681 module_id: "late-health-response".to_string(),
13682 program: PathBuf::from("disabled-module"),
13683 args: Vec::new(),
13684 env: Vec::new(),
13685 reserved: false,
13686 reserved_prefixes: Vec::new(),
13687 protocol: ModuleProtocol::Subc,
13688 overlap: Default::default(),
13689 },
13690 false,
13691 )
13692 .unwrap();
13693 let handler =
13694 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13695 .with_supervisor(supervisor_handle);
13696 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
13697 handler
13698 .handle_control_frame(
13699 &module_ctx,
13700 hello_frame_with_control_ops(
13701 "late-health-response",
13702 PROTOCOL_VERSION,
13703 7,
13704 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13705 ),
13706 )
13707 .await
13708 .unwrap();
13709 let probe_started_at = Instant::now() - Duration::from_millis(80);
13710 let pending = forwarding
13711 .begin_health_probe_rpc_for(
13712 "late-health-response",
13713 MODULE_CONTROL_OP_HEALTH_CHECK,
13714 probe_started_at,
13715 Instant::now() - Duration::from_millis(1),
13716 )
13717 .unwrap();
13718 assert!(forwarding
13719 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
13720 .unwrap());
13721
13722 let responses = handler
13723 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
13724 .await
13725 .unwrap();
13726
13727 assert!(responses.is_empty());
13728 let health = module.status().unwrap().health;
13729 assert_eq!(health.late_answer_count, 1);
13730 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
13731 }
13732
13733 #[tokio::test]
13734 async fn health_probe_timeout_and_module_death_are_typed() {
13735 let registry = Arc::new(Registry::default());
13736 let forwarding = Arc::new(ForwardingTable::default());
13737 let handler =
13738 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13739 .with_health_probe_timeout(Duration::from_millis(50));
13740 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
13741 hello_via_sink(
13742 &handler,
13743 &module_ctx,
13744 &mut module_rx,
13745 hello_frame_with_control_ops(
13746 "aft",
13747 PROTOCOL_VERSION,
13748 7,
13749 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13750 ),
13751 )
13752 .await;
13753
13754 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
13755 let responses = handler
13756 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
13757 .await
13758 .unwrap();
13759 assert_eq!(responses[0].header.ty, FrameType::Error);
13760 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
13761 let _ = module_rx.try_recv();
13762
13763 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
13764 let health_handler = handler.clone();
13765 let death_task = tokio::spawn(async move {
13766 health_handler
13767 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
13768 .await
13769 .unwrap()
13770 });
13771 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13772 .await
13773 .unwrap()
13774 .unwrap();
13775 handler
13776 .cleanup_connection(module_ctx.connection_id)
13777 .unwrap();
13778 let responses = death_task.await.unwrap();
13779 assert_eq!(responses[0].header.ty, FrameType::Error);
13780 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
13781 }
13782
13783 #[test]
13784 fn hello_requires_exact_protocol_version() {
13785 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
13786 let registry = Arc::new(Registry::default());
13787 let handler = ControlHandler::new(Arc::clone(®istry));
13788 let responses = handler
13789 .handle_control(
13790 ConnectionId::new(connection),
13791 hello_frame("aft", offered, 9),
13792 )
13793 .unwrap();
13794
13795 assert_eq!(responses.len(), 1);
13796 assert_eq!(responses[0].header.ty, FrameType::Error);
13797 let error = parse_error(&responses[0]);
13798 assert_eq!(error["code"], "version_unsupported");
13799 assert!(registry.get_module("aft").unwrap().is_none());
13800 assert_eq!(registry.active_registration_count().unwrap(), 0);
13801 }
13802 }
13803
13804 #[test]
13805 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
13806 let registry = Arc::new(Registry::default());
13807 let forwarding = Arc::new(ForwardingTable::default());
13808 let handler =
13809 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13810 let module_connection = ConnectionId::new(301);
13811 let registration = registry
13812 .register_with_control_ops(
13813 manifest("aft-push", PROTOCOL_VERSION),
13814 PROTOCOL_VERSION,
13815 module_connection,
13816 module_baseline_control_ops(),
13817 )
13818 .unwrap();
13819 let (module_tx, _module_rx) = mpsc::channel(8);
13820 let endpoint = forwarding
13821 .register_module_connection(
13822 module_connection,
13823 "aft-push".to_string(),
13824 PROTOCOL_VERSION,
13825 manifest_concurrency(®istration.manifest),
13826 FrameSink::new(module_tx),
13827 )
13828 .unwrap();
13829
13830 let unknown = Frame::build(
13832 FrameType::Push,
13833 control_flags(),
13834 0,
13835 0,
13836 5,
13837 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
13838 )
13839 .unwrap();
13840 let out = handler.handle_status_update(endpoint, unknown).unwrap();
13841 assert!(
13842 out.is_empty(),
13843 "unknown push op must be ignored, got {out:?}"
13844 );
13845
13846 let malformed = Frame::build(
13848 FrameType::Push,
13849 control_flags(),
13850 0,
13851 0,
13852 6,
13853 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
13854 )
13855 .unwrap();
13856 let out = handler.handle_status_update(endpoint, malformed).unwrap();
13857 assert_eq!(out.len(), 1);
13858 assert_eq!(out[0].header.ty, FrameType::Error);
13859 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
13860 }
13861
13862 #[test]
13863 fn hello_rejected_when_connection_already_owns_client_routes() {
13864 let registry = Arc::new(Registry::default());
13865 let forwarding = Arc::new(ForwardingTable::default());
13866 let handler =
13867 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13868 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
13870 let client_connection = ConnectionId::new(202);
13871
13872 let responses = handler
13875 .handle_control(
13876 client_connection,
13877 hello_frame("aft-second", PROTOCOL_VERSION, 9),
13878 )
13879 .unwrap();
13880 assert_eq!(responses[0].header.ty, FrameType::Error);
13881 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
13882 assert!(registry.get_module("aft-second").unwrap().is_none());
13883 }
13884
13885 #[tokio::test]
13886 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
13887 let registry = Arc::new(Registry::default());
13888 let forwarding = Arc::new(ForwardingTable::default());
13889 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
13890 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
13891 hello_via_sink(
13892 &handler,
13893 &module_ctx,
13894 &mut module_rx,
13895 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
13896 )
13897 .await;
13898 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
13899 let pending = forwarding
13900 .begin_route_bind_relay_for_test(
13901 client_ctx.connection_id,
13902 client_ctx.egress.clone(),
13903 2,
13904 "alpha",
13905 )
13906 .unwrap();
13907 forwarding
13908 .complete_pending_relay(
13909 module_ctx.connection_id,
13910 pending.corr,
13911 RouteBindRelayOutcome::Accepted,
13912 )
13913 .unwrap();
13914 client_rx.try_recv().unwrap();
13915 for module_id in ["beta", "alpha"] {
13916 let replies = handler
13917 .handle_control_frame(
13918 &module_ctx,
13919 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
13920 )
13921 .await
13922 .unwrap();
13923 assert_eq!(replies.len(), 1, "second HELLO must be refused");
13924 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
13925 }
13926 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
13927 assert!(registry.get_module("beta").unwrap().is_none());
13928 assert!(matches!(
13929 forwarding
13930 .lookup_data_route(
13931 client_ctx.connection_id,
13932 pending.client_channel,
13933 pending.client_epoch,
13934 )
13935 .unwrap(),
13936 DataRoute::Client(DataRouteState::Bound(_))
13937 ));
13938 assert!(handler
13939 .hello_launch_nonces
13940 .lock()
13941 .unwrap()
13942 .presented(module_ctx.connection_id, Some("alpha-nonce")));
13943 assert!(module_rx.try_recv().is_err());
13944 }
13945
13946 #[test]
13947 fn reserved_module_hello_requires_matching_launch_nonce() {
13948 let registry = Arc::new(Registry::default());
13949 let supervisor = SupervisorHandle::new();
13950 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
13953 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
13954
13955 let no_nonce = handler
13957 .handle_control(
13958 ConnectionId::new(1),
13959 hello_frame("vault", PROTOCOL_VERSION, 1),
13960 )
13961 .unwrap();
13962 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
13963 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
13964 assert!(registry.get_module("vault").unwrap().is_none());
13965
13966 let wrong = handler
13968 .handle_control(
13969 ConnectionId::new(2),
13970 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
13971 )
13972 .unwrap();
13973 assert_eq!(wrong[0].header.ty, FrameType::Error);
13974 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
13975 assert!(registry.get_module("vault").unwrap().is_none());
13976
13977 let ok = handler
13979 .handle_control(
13980 ConnectionId::new(3),
13981 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
13982 )
13983 .unwrap();
13984 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
13985 assert!(registry.get_module("vault").unwrap().is_some());
13986 }
13987
13988 #[test]
13989 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
13990 let registry = Arc::new(Registry::default());
13991 let supervisor = SupervisorHandle::new();
13992 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13993 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13994 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
13995
13996 let squat = handler
13997 .handle_control(
13998 ConnectionId::new(1),
13999 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
14000 )
14001 .unwrap();
14002 assert_eq!(squat[0].header.ty, FrameType::Error);
14003 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
14004 assert!(parse_error(&squat[0])["message"]
14005 .as_str()
14006 .unwrap()
14007 .contains("fed:"));
14008
14009 let accepted_peer = handler
14010 .handle_control(
14011 ConnectionId::new(2),
14012 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
14013 )
14014 .unwrap();
14015 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
14016
14017 let accepted_short = handler
14018 .handle_control(
14019 ConnectionId::new(3),
14020 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
14021 )
14022 .unwrap();
14023 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
14024
14025 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
14026 let response = handler
14027 .handle_control(
14028 ConnectionId::new(conn),
14029 hello_frame(module_id, PROTOCOL_VERSION, conn),
14030 )
14031 .unwrap();
14032 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
14033 }
14034 }
14035
14036 #[test]
14037 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
14038 let registry = Arc::new(Registry::default());
14039 let supervisor = SupervisorHandle::new();
14040 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14041 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14042 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
14043 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14044
14045 let owner_nonce = handler
14046 .handle_control(
14047 ConnectionId::new(1),
14048 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
14049 )
14050 .unwrap();
14051 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
14052 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
14053 assert!(registry.get_module("fed:special").unwrap().is_none());
14054
14055 let exact_nonce = handler
14056 .handle_control(
14057 ConnectionId::new(2),
14058 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
14059 )
14060 .unwrap();
14061 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
14062 assert!(registry.get_module("fed:special").unwrap().is_some());
14063 }
14064
14065 #[test]
14066 fn non_reserved_module_ignores_launch_nonce() {
14067 let registry = Arc::new(Registry::default());
14068 let handler = ControlHandler::new(Arc::clone(®istry));
14071 let no_nonce = handler
14072 .handle_control(
14073 ConnectionId::new(1),
14074 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
14075 )
14076 .unwrap();
14077 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
14078 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
14079
14080 let echoed_nonce = handler
14081 .handle_control(
14082 ConnectionId::new(2),
14083 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
14084 )
14085 .unwrap();
14086 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
14087 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
14088 }
14089
14090 #[test]
14091 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
14092 let handler = ControlHandler::default();
14093 let conn = ConnectionId::new(1);
14094 let malformed = Frame::build(
14095 FrameType::Hello,
14096 control_flags(),
14097 0,
14098 0,
14099 3,
14100 b"{not json".to_vec(),
14101 )
14102 .unwrap();
14103
14104 let error = handler.handle_control(conn, malformed).unwrap();
14105 assert_eq!(error[0].header.ty, FrameType::Error);
14106 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
14107
14108 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14109 let pong = handler.handle_control(conn, ping).unwrap();
14110 assert_eq!(pong[0].header.ty, FrameType::Pong);
14111 assert_eq!(pong[0].header.corr, 4);
14112 }
14113
14114 #[test]
14115 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
14116 let registry = Arc::new(Registry::default());
14117 let handler = ControlHandler::new(Arc::clone(®istry));
14118
14119 handler
14120 .handle_control(
14121 ConnectionId::new(1),
14122 hello_frame("aft", PROTOCOL_VERSION, 1),
14123 )
14124 .unwrap();
14125 let duplicate = handler
14126 .handle_control(
14127 ConnectionId::new(2),
14128 hello_frame("aft", PROTOCOL_VERSION, 2),
14129 )
14130 .unwrap();
14131
14132 assert_eq!(duplicate[0].header.ty, FrameType::Error);
14133 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14134 let registration = registry.get_module("aft").unwrap().unwrap();
14135 assert_eq!(registration.connection_id, ConnectionId::new(1));
14136 }
14137
14138 #[test]
14139 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
14140 let registry = Arc::new(Registry::default());
14141 let forwarding = Arc::new(ForwardingTable::default());
14142 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
14143 let handler =
14144 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14145 .with_process_liveness(process_liveness);
14146 let (ctx, route_channel, route_epoch) =
14147 bind_liveness_route(®istry, &forwarding, "aft-dead");
14148 let responses = handler
14149 .handle_route_poll(
14150 &ctx,
14151 route_poll_frame(41, PollKind::Liveness, route_channel),
14152 route_channel,
14153 route_epoch,
14154 PollKind::Liveness,
14155 )
14156 .unwrap();
14157
14158 assert_eq!(responses.len(), 1);
14159 assert_eq!(responses[0].header.ty, FrameType::Response);
14160 assert_route_poll_liveness(&responses[0], false);
14161 }
14162
14163 #[test]
14164 fn liveness_poll_without_process_source_uses_bound_route() {
14165 let registry = Arc::new(Registry::default());
14166 let forwarding = Arc::new(ForwardingTable::default());
14167 let handler =
14168 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14169 let (ctx, route_channel, route_epoch) =
14170 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
14171 let responses = handler
14172 .handle_route_poll(
14173 &ctx,
14174 route_poll_frame(42, PollKind::Liveness, route_channel),
14175 route_channel,
14176 route_epoch,
14177 PollKind::Liveness,
14178 )
14179 .unwrap();
14180
14181 assert_route_poll_liveness(&responses[0], true);
14182 }
14183
14184 #[test]
14185 fn liveness_poll_untracked_process_source_uses_bound_route() {
14186 let registry = Arc::new(Registry::default());
14187 let forwarding = Arc::new(ForwardingTable::default());
14188 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
14189 let handler =
14190 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14191 .with_process_liveness(process_liveness);
14192 let (ctx, route_channel, route_epoch) =
14193 bind_liveness_route(®istry, &forwarding, "aft-untracked");
14194 let responses = handler
14195 .handle_route_poll(
14196 &ctx,
14197 route_poll_frame(43, PollKind::Liveness, route_channel),
14198 route_channel,
14199 route_epoch,
14200 PollKind::Liveness,
14201 )
14202 .unwrap();
14203
14204 assert_route_poll_liveness(&responses[0], true);
14205 }
14206
14207 #[tokio::test]
14208 async fn unknown_op_returns_unknown_control_op() {
14209 let handler = ControlHandler::default();
14210 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
14211 let request = Frame::build(
14212 FrameType::Request,
14213 control_flags(),
14214 0,
14215 0,
14216 55,
14217 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
14218 )
14219 .unwrap();
14220
14221 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14222
14223 assert_eq!(response.len(), 1);
14224 assert_eq!(response[0].header.ty, FrameType::Error);
14225 assert_eq!(response[0].header.corr, 55);
14226 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
14227 }
14228
14229 #[tokio::test]
14230 async fn supervisor_provenance_rejects_unknown_exact_module() {
14231 let handler = ControlHandler::default();
14232 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
14233 let request = Frame::build(
14234 FrameType::Request,
14235 control_flags(),
14236 0,
14237 0,
14238 57,
14239 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
14240 )
14241 .unwrap();
14242
14243 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14244
14245 assert_eq!(response.len(), 1);
14246 assert_eq!(response[0].header.ty, FrameType::Error);
14247 assert_eq!(response[0].header.corr, 57);
14248 let error = parse_error(&response[0]);
14249 assert_eq!(error["code"], "unknown_module");
14250 assert_eq!(error["message"], "module_id 'missing' is not supervised");
14251 }
14252
14253 #[test]
14254 fn provenance_probe_override_keeps_handler_tests_deterministic() {
14255 let expected = subc_control::RunningImageAgreement::Unavailable {
14256 reason: subc_control::RunningImageUnavailableReason::HashFailed,
14257 };
14258 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
14259 assert_eq!(handler.provenance_probe_override, Some(expected));
14260 }
14261
14262 #[test]
14263 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
14264 let verdict = reload_verdict(
14265 std::path::Path::new("/bin/new"),
14266 Some(std::path::Path::new("/bin/old")),
14267 subc_control::RunningImageAgreement::Unavailable {
14268 reason: subc_control::RunningImageUnavailableReason::HashFailed,
14269 },
14270 );
14271 assert!(matches!(
14272 verdict.path,
14273 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
14274 if configured == std::path::Path::new("/bin/new")
14275 && spawned_from == std::path::Path::new("/bin/old")
14276 ));
14277 }
14278
14279 #[test]
14280 fn launch_spec_without_running_digest_is_unavailable_not_match() {
14281 use subc_control::{
14282 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
14283 };
14284 let configured = crate::supervise::ModuleSpec {
14285 module_id: "legacy".into(),
14286 program: "/module".into(),
14287 args: Vec::new(),
14288 env: Vec::new(),
14289 reserved: false,
14290 reserved_prefixes: Vec::new(),
14291 protocol: subc_control::ModuleProtocol::Subc,
14292 overlap: Default::default(),
14293 };
14294 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
14295 assert_eq!(
14296 launch_spec_agreement(&configured, false, Some(&digest)),
14297 Agreement::Unavailable {
14298 reason: Reason::NotRunning
14299 }
14300 );
14301 assert_eq!(
14302 launch_spec_agreement(&configured, true, None),
14303 Agreement::Unavailable {
14304 reason: Reason::SpawnedSpecUnavailable
14305 }
14306 );
14307 assert_eq!(
14308 launch_spec_agreement(&configured, true, Some(&digest)),
14309 Agreement::Match
14310 );
14311 }
14312
14313 #[cfg(windows)]
14314 #[test]
14315 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
14316 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
14317 let configured = dir.join("module.exe");
14318 std::fs::write(&configured, b"executable path fixture").unwrap();
14319 let spawned = std::fs::canonicalize(&configured).unwrap();
14320 assert_ne!(
14321 configured, spawned,
14322 "fixture must exercise different path spellings"
14323 );
14324 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
14325 let lower_case = configured.to_string_lossy().to_lowercase();
14326 for spelling in [
14327 &configured,
14328 Path::new(&forward_slashes),
14329 Path::new(&lower_case),
14330 ] {
14331 let verdict = reload_verdict(
14332 spelling,
14333 Some(&spawned),
14334 subc_control::RunningImageAgreement::Unavailable {
14335 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14336 },
14337 );
14338 assert_eq!(
14339 verdict.path,
14340 ReloadPathAgreement::Match,
14341 "configured {spelling:?}, spawned {spawned:?}"
14342 );
14343 }
14344 }
14345
14346 #[cfg(windows)]
14347 #[test]
14348 fn windows_reload_path_keeps_different_existing_program_pending() {
14349 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
14350 let old = dir.join("old.exe");
14351 let new = dir.join("new.exe");
14352 std::fs::write(&old, b"same executable content").unwrap();
14353 std::fs::copy(&old, &new).unwrap();
14354 let spawned = std::fs::canonicalize(&old).unwrap();
14355 let image = subc_control::RunningImageAgreement::Unavailable {
14356 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14357 };
14358 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
14359 assert!(
14360 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
14361 if configured == new && spawned_from == spawned)
14362 );
14363 let missing = dir.join("missing.exe");
14364 assert!(matches!(
14365 reload_verdict(&missing, Some(&spawned), image).path,
14366 ReloadPathAgreement::Mismatch { .. }
14367 ));
14368 }
14369
14370 #[test]
14371 fn reload_verdict_detects_replaced_image_at_same_path() {
14372 let image = subc_control::RunningImageAgreement::Mismatch {
14373 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
14374 digest: "old".into(),
14375 },
14376 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
14377 digest: "new".into(),
14378 },
14379 };
14380 let verdict = reload_verdict(
14381 std::path::Path::new("/bin/same"),
14382 Some(std::path::Path::new("/bin/same")),
14383 image.clone(),
14384 );
14385 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14386 assert_eq!(verdict.image, image);
14387 }
14388
14389 #[test]
14390 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
14391 let image = subc_control::RunningImageAgreement::Unavailable {
14392 reason: subc_control::RunningImageUnavailableReason::NotRunning,
14393 };
14394 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
14395 assert_eq!(
14396 verdict.path,
14397 subc_control::ReloadPathAgreement::Unavailable {
14398 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
14399 }
14400 );
14401 assert_eq!(verdict.image, image);
14402
14403 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
14404 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14405 };
14406 let verdict = reload_verdict(
14407 std::path::Path::new("/bin/same"),
14408 Some(std::path::Path::new("/bin/same")),
14409 unconfirmed.clone(),
14410 );
14411 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14412 assert_eq!(verdict.image, unconfirmed);
14413 }
14414
14415 #[test]
14416 fn reload_verdict_preserves_each_image_unavailability_reason() {
14417 use subc_control::RunningImageUnavailableReason as Reason;
14418
14419 for reason in [
14420 Reason::NotRunning,
14421 Reason::UnsupportedPlatform,
14422 Reason::RunningExecutableUnreadable,
14423 Reason::SpawnedPathUnreadable,
14424 Reason::HashFailed,
14425 Reason::ProcessIdentityUnconfirmed,
14426 Reason::Unknown("future_probe_reason".to_string()),
14427 ] {
14428 let image = subc_control::RunningImageAgreement::Unavailable {
14429 reason: reason.clone(),
14430 };
14431 let verdict = reload_verdict(
14432 std::path::Path::new("/bin/same"),
14433 Some(std::path::Path::new("/bin/same")),
14434 image.clone(),
14435 );
14436 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14437 assert_eq!(verdict.image, image, "{reason:?}");
14438 }
14439 }
14440
14441 #[tokio::test]
14442 async fn malformed_control_bodies_return_invalid_control_body() {
14443 let handler = ControlHandler::default();
14444 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
14445
14446 for (corr, body) in [
14447 (56, br#"{"route_channel":1}"#.as_slice()),
14448 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
14449 (
14450 58,
14451 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
14452 ),
14453 ] {
14454 let request = Frame::build(
14455 FrameType::Request,
14456 control_flags(),
14457 0,
14458 0,
14459 corr,
14460 body.to_vec(),
14461 )
14462 .unwrap();
14463 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14464
14465 assert_eq!(response.len(), 1);
14466 assert_eq!(response[0].header.ty, FrameType::Error);
14467 assert_eq!(response[0].header.corr, corr);
14468 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
14469 }
14470 }
14471
14472 #[tokio::test]
14473 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
14474 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14475 let registry = Arc::new(Registry::default());
14476 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
14477 let router = Router::with_control_handler(Arc::clone(&control));
14478 let connection = router.begin_connection();
14479 let (ctx, mut rx) = route_ctx(connection.id());
14480
14481 router
14482 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
14483 .await
14484 .unwrap();
14485 let response = rx.recv().await.unwrap();
14486 let ack = parse_ack(&response);
14487 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14488 let channel = 1;
14489
14490 let goodbye =
14491 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
14492 router.route_for_connection(&ctx, goodbye).await.unwrap();
14493 assert!(rx.try_recv().is_err());
14494 assert!(registry.get_module("aft").unwrap().is_none());
14495
14496 router
14497 .route_for_connection(&ctx, channel_request(channel, 13))
14498 .await
14499 .unwrap();
14500 let error_frame = rx.recv().await.unwrap();
14501 assert_eq!(error_frame.header.ty, FrameType::Error);
14502 assert_eq!(error_frame.header.channel, channel);
14503 let captured = crate::router::test_log::captured_logs(&logs);
14504 assert_eq!(
14505 captured
14506 .lines()
14507 .filter(|line| {
14508 line.contains("module_id=aft")
14509 && line.contains("reason=explicit_goodbye")
14510 && line.contains("module registration ended")
14511 })
14512 .count(),
14513 1,
14514 "unexpected GOODBYE registry log: {captured}"
14515 );
14516 }
14517
14518 #[tokio::test]
14519 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
14520 let registry = Arc::new(Registry::default());
14521 let handler = ControlHandler::new(registry).with_capability_config(
14522 [("prov".to_string(), true), ("cons".to_string(), true)],
14523 BTreeMap::new(),
14524 );
14525 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
14526 register_capability_manifest(
14527 &handler,
14528 &provider_ctx,
14529 &mut provider_rx,
14530 capability_manifest("prov", &["thing/v1"], &[]),
14531 1,
14532 )
14533 .await;
14534 let mut consumer = capability_manifest("cons", &[], &[]);
14535 consumer.capabilities.as_mut().unwrap().requires.push(
14536 subc_protocol::manifest::CapabilityRequirement {
14537 capability: "thing/v1".to_string(),
14538 need: subc_protocol::manifest::CapabilityNeed::Required,
14539 },
14540 );
14541 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
14542 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
14543 assert_eq!(
14544 handler.capability_evaluator.verdict("cons", "thing/v1"),
14545 Some(CapabilityVerdict::Provided)
14546 );
14547 let (mut client_rx, _) = open_route_for_capability_test(
14548 &handler,
14549 &provider_ctx,
14550 &mut provider_rx,
14551 703,
14552 3,
14553 "prov",
14554 None,
14555 )
14556 .await;
14557 let goodbye =
14558 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14559 handler
14560 .handle_control_frame(&provider_ctx, goodbye)
14561 .await
14562 .unwrap();
14563 assert_eq!(
14564 handler.capability_evaluator.verdict("cons", "thing/v1"),
14565 Some(CapabilityVerdict::NeverProvided)
14566 );
14567 let closed = client_rx
14568 .try_recv()
14569 .expect("GOODBYE pushes route.closed before route GOODBYE");
14570 assert!(
14571 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
14572 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
14573 );
14574 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
14575 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14576 }
14577
14578 struct RouteClosedFixture {
14579 handler: ControlHandler,
14580 #[cfg(unix)]
14581 supervisor: Supervisor,
14582 module_ctx: RouteCtx,
14583 #[cfg(unix)]
14584 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14585 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14586 route: Arc<crate::forwarding::RouteBinding>,
14587 }
14588
14589 async fn route_closed_fixture() -> RouteClosedFixture {
14590 let registry = Arc::new(Registry::default());
14591 let handler = ControlHandler::new(Arc::clone(®istry));
14592 #[cfg(unix)]
14593 let supervisor = Supervisor::new(registry, RestartPolicy::default())
14594 .with_forwarding(Arc::clone(&handler.forwarding));
14595 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
14596 register_capability_manifest(
14597 &handler,
14598 &module_ctx,
14599 &mut module_rx,
14600 capability_manifest("prov", &[], &[]),
14601 1,
14602 )
14603 .await;
14604 let (client_rx, bind) = open_route_for_capability_test(
14605 &handler,
14606 &module_ctx,
14607 &mut module_rx,
14608 702,
14609 2,
14610 "prov",
14611 None,
14612 )
14613 .await;
14614 let ModuleControlRequest::RouteBind {
14615 route_channel: channel,
14616 epoch: route_epoch,
14617 ..
14618 } = bind
14619 else {
14620 panic!("expected route.bind");
14621 };
14622 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
14623 handler
14624 .forwarding
14625 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
14626 .unwrap()
14627 else {
14628 panic!("the acknowledged route must be bound");
14629 };
14630 RouteClosedFixture {
14631 handler,
14632 #[cfg(unix)]
14633 supervisor,
14634 module_ctx,
14635 #[cfg(unix)]
14636 module_rx,
14637 client_rx,
14638 route,
14639 }
14640 }
14641
14642 impl RouteClosedFixture {
14643 #[cfg(unix)]
14644 async fn cut(&mut self) {
14645 let notices = async {
14646 let mut draining = self.module_rx.recv().await.unwrap();
14647 assert!(matches!(
14648 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
14649 &draining.body
14650 )
14651 .unwrap(),
14652 subc_protocol::session::ModuleControlCommand::Draining {
14653 reason: RouteCloseReason::Restart,
14654 ..
14655 }
14656 ));
14657 draining.flushed.take().unwrap().send(()).unwrap();
14658 let mut closing = self.client_rx.recv().await.unwrap();
14659 assert!(matches!(
14660 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
14661 ClientControlPush::RouteClosing {
14662 reason: RouteCloseReason::Restart,
14663 ..
14664 }
14665 ));
14666 closing.flushed.take().unwrap().send(()).unwrap();
14667 };
14668 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
14669 result.unwrap();
14670 }
14671
14672 fn disconnect(&mut self) -> ClientControlPush {
14673 self.handler
14674 .cleanup_connection(self.module_ctx.connection_id)
14675 .unwrap();
14676 let closed = self.client_rx.try_recv().unwrap();
14677 assert_eq!(closed.header.ty, FrameType::Push);
14678 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
14679 assert!(matches!(
14680 &push,
14681 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
14682 if channels == &[self.route.client_channel]
14683 ));
14684 assert_eq!(
14685 self.client_rx.try_recv().unwrap().header.ty,
14686 FrameType::Goodbye
14687 );
14688 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
14689 push
14690 }
14691 }
14692
14693 #[cfg(unix)]
14694 #[tokio::test(start_paused = true)]
14695 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
14696 let mut fixture = route_closed_fixture().await;
14697 fixture.cut().await;
14698 assert!(matches!(
14699 fixture.disconnect(),
14700 ClientControlPush::RouteClosed {
14701 reason: RouteCloseReason::Restart,
14702 drained: true,
14703 ..
14704 }
14705 ));
14706 }
14707
14708 #[cfg(unix)]
14709 #[tokio::test(start_paused = true)]
14710 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
14711 let mut fixture = route_closed_fixture().await;
14712 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
14713 let started = tokio::time::Instant::now();
14714 fixture.cut().await;
14715 assert!(started.elapsed() >= Duration::from_secs(2));
14716 assert_eq!(fixture.route.flow.in_flight(), 1);
14717 assert!(matches!(
14718 fixture.disconnect(),
14719 ClientControlPush::RouteClosed {
14720 reason: RouteCloseReason::Restart,
14721 drained: false,
14722 ..
14723 }
14724 ));
14725 }
14726
14727 #[tokio::test]
14728 async fn module_connection_crash_reports_crash_and_not_drained() {
14729 let mut fixture = route_closed_fixture().await;
14730 assert!(matches!(
14734 fixture.disconnect(),
14735 ClientControlPush::RouteClosed {
14736 reason: RouteCloseReason::Crash,
14737 drained: false,
14738 ..
14739 }
14740 ));
14741 }
14742
14743 #[tokio::test]
14744 async fn dropping_router_connection_releases_registration() {
14745 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14746 let registry = Arc::new(Registry::default());
14747 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
14748 let router = Router::with_control_handler(Arc::clone(&control));
14749 let connection = router.begin_connection();
14750 let connection_id = connection.id();
14751 let (ctx, mut rx) = route_ctx(connection_id);
14752
14753 router
14754 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
14755 .await
14756 .unwrap();
14757 let response = rx.recv().await.unwrap();
14758 let ack = parse_ack(&response);
14759 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14760 assert!(registry.get_module("aft").unwrap().is_some());
14761
14762 drop(connection);
14763
14764 assert!(registry.get_module("aft").unwrap().is_none());
14765 assert_eq!(registry.active_registration_count().unwrap(), 0);
14766
14767 control
14768 .cleanup_connection(ConnectionId::new(u64::MAX))
14769 .unwrap();
14770 let captured = crate::router::test_log::captured_logs(&logs);
14771 println!("captured registry lifecycle logs:\n{captured}");
14772 let events: Vec<_> = captured
14773 .lines()
14774 .filter(|line| {
14775 line.contains("module registered module_id=aft ")
14776 || line.contains("module registration ended")
14777 })
14778 .collect();
14779 assert_eq!(
14780 events.len(),
14781 2,
14782 "unexpected registry lifecycle logs: {captured}"
14783 );
14784 assert!(events[0].contains("module registered module_id=aft "));
14785 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
14786 assert!(events[1].contains(&format!(
14787 "module_id=aft connection_id={}",
14788 connection_id.get()
14789 )));
14790 assert!(events[1].contains("reason=connection_closed"));
14791 assert!(events[1].contains("module registration ended"));
14792 assert_eq!(
14793 captured
14794 .lines()
14795 .filter(|line| line.contains("module registered module_id=aft "))
14796 .count(),
14797 1,
14798 "legacy registration admission line must appear once: {captured}"
14799 );
14800 }
14801
14802 #[tokio::test]
14803 async fn hello_registration_keeps_legacy_module_registered_line_once() {
14804 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14805 let registry = Arc::new(Registry::default());
14806 let control = ControlHandler::new(Arc::clone(®istry));
14807 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
14808 hello_via_sink(
14809 &control,
14810 &ctx,
14811 &mut rx,
14812 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
14813 )
14814 .await;
14815
14816 let captured = crate::router::test_log::captured_logs(&logs);
14817 let admissions: Vec<_> = captured
14818 .lines()
14819 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
14820 .collect();
14821 assert_eq!(
14822 admissions.len(),
14823 1,
14824 "legacy admission line must remain exactly once: {captured}"
14825 );
14826 assert!(admissions[0].contains("routable_provider=true"));
14827 assert!(admissions[0].contains("connection_id=777"));
14828 }
14829
14830 fn capability_manifest(
14831 module_id: &str,
14832 provides: &[&str],
14833 must_never_reach: &[&str],
14834 ) -> ModuleManifest {
14835 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
14836 manifest.capabilities = Some(CapabilityDeclarations {
14837 provides: provides
14838 .iter()
14839 .map(|capability| (*capability).to_string())
14840 .collect(),
14841 requires: Vec::new(),
14842 must_never_reach: must_never_reach
14843 .iter()
14844 .map(|capability| (*capability).to_string())
14845 .collect(),
14846 });
14847 manifest
14848 }
14849
14850 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
14851 Frame::build(
14852 FrameType::Hello,
14853 control_flags(),
14854 0,
14855 0,
14856 corr,
14857 serde_json::to_vec(&ModuleHelloBody {
14858 protocol_ver: manifest.protocol_ver,
14859 manifest,
14860 control_ops: None,
14861 launch_nonce: None,
14862 })
14863 .expect("capability test HELLO serializes"),
14864 )
14865 .expect("capability test HELLO frame builds")
14866 }
14867
14868 fn catalog_update_with_capabilities_frame(
14869 corr: u64,
14870 capabilities: CapabilityDeclarations,
14871 ) -> Frame {
14872 Frame::build(
14873 FrameType::Request,
14874 control_flags(),
14875 0,
14876 0,
14877 corr,
14878 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14879 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
14880 capabilities: Some(capabilities),
14881 ready: None,
14882 })
14883 .expect("capability catalog.update serializes"),
14884 )
14885 .expect("capability catalog.update frame builds")
14886 }
14887
14888 async fn register_capability_manifest(
14889 handler: &ControlHandler,
14890 ctx: &RouteCtx,
14891 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14892 manifest: ModuleManifest,
14893 corr: u64,
14894 ) {
14895 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
14896 }
14897
14898 async fn open_route_for_capability_test(
14899 handler: &ControlHandler,
14900 target_ctx: &RouteCtx,
14901 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14902 client_connection_id: u64,
14903 corr: u64,
14904 target_module_id: &str,
14905 consumer_identity: Option<ConsumerIdentity>,
14906 ) -> (
14907 mpsc::Receiver<crate::router::OutboundFrame>,
14908 ModuleControlRequest,
14909 ) {
14910 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
14911 let route_handler = handler.clone();
14912 let target_module_id = target_module_id.to_string();
14913 let route_task = tokio::spawn(async move {
14914 route_handler
14915 .handle_control_frame(
14916 &client_ctx,
14917 route_open_frame_with_admission_facts(
14918 corr,
14919 &target_module_id,
14920 unique_project_root("admission-facts"),
14921 consumer_identity,
14922 None,
14923 ),
14924 )
14925 .await
14926 .expect("capability test route.open succeeds")
14927 });
14928 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
14929 .await
14930 .expect("capability test route.open must reach route.bind")
14931 .expect("target control receiver stays open");
14932 let bind_request: ModuleControlRequest =
14933 serde_json::from_slice(&bind.body).expect("route.bind decodes");
14934 handler
14935 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
14936 .await
14937 .expect("capability test route.bind ACK succeeds");
14938 assert!(route_task.await.expect("route.open task joins").is_empty());
14939 let opened = client_rx
14940 .recv()
14941 .await
14942 .expect("successful route.open publishes a response");
14943 assert!(matches!(
14944 serde_json::from_slice::<ClientControlResponse>(&opened.body),
14945 Ok(ClientControlResponse::RouteOpen { .. })
14946 ));
14947 (client_rx, bind_request)
14948 }
14949
14950 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
14951 assert_eq!(frame.header.ty, FrameType::Push);
14952 assert_eq!(frame.header.channel, 0);
14953 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
14954 .expect("route.closed control push decodes");
14955 let ClientControlPush::RouteClosed { channels, .. } = &push else {
14956 panic!("expected route.closed");
14957 };
14958 assert_eq!(channels.len(), 1, "exactly one violating route closed");
14959 let closed_channel = channels[0];
14960 let channels = channels.clone();
14961 assert_eq!(
14962 push,
14963 ClientControlPush::RouteClosed {
14964 module_id: target_module_id.to_string(),
14965 channels,
14966 reason: RouteCloseReason::CapabilityDenied,
14967 drained: false,
14968 abandoned: 0,
14969 excluded_subscriptions: 0,
14970 terminal: Some(false),
14971 }
14972 );
14973 closed_channel
14974 }
14975
14976 #[tokio::test]
14977 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
14978 let registry = Arc::new(Registry::default());
14979 let forwarding = Arc::new(ForwardingTable::default());
14980 let supervisor = SupervisorHandle::new();
14981 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14982 let handler =
14983 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14984 .with_supervisor(supervisor);
14985 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
14986 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
14987 register_capability_manifest(
14988 &handler,
14989 &target_ctx,
14990 &mut target_rx,
14991 capability_manifest("target", &["credentials-provider/v1"], &[]),
14992 1,
14993 )
14994 .await;
14995 register_capability_manifest(
14996 &handler,
14997 &opener_ctx,
14998 &mut opener_rx,
14999 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15000 2,
15001 )
15002 .await;
15003
15004 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
15005 let replies = handler
15006 .handle_control_frame(
15007 &client_ctx,
15008 route_open_frame_with_admission_facts(
15009 3,
15010 "target",
15011 unique_project_root("admission-facts"),
15012 Some(ConsumerIdentity {
15013 module_id: "opener".to_string(),
15014 launch_nonce: "opener-nonce".to_string(),
15015 }),
15016 None,
15017 ),
15018 )
15019 .await
15020 .expect("denied route.open returns a typed frame");
15021 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15022 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15023 assert!(
15024 target_rx.try_recv().is_err(),
15025 "forbidden route.open must not relay route.bind"
15026 );
15027 }
15028
15029 #[tokio::test]
15030 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
15031 let registry = Arc::new(Registry::default());
15032 let forwarding = Arc::new(ForwardingTable::default());
15033 let supervisor = SupervisorHandle::new();
15034 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15035 let handler =
15036 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15037 .with_supervisor(supervisor);
15038 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
15039 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
15040 register_capability_manifest(
15041 &handler,
15042 &target_ctx,
15043 &mut target_rx,
15044 capability_manifest("target", &["credentials-provider/v1"], &[]),
15045 1,
15046 )
15047 .await;
15048 register_capability_manifest(
15049 &handler,
15050 &old_opener_ctx,
15051 &mut old_opener_rx,
15052 capability_manifest("opener", &[], &[]),
15053 2,
15054 )
15055 .await;
15056 let (mut client_rx, _) = open_route_for_capability_test(
15057 &handler,
15058 &target_ctx,
15059 &mut target_rx,
15060 712,
15061 3,
15062 "target",
15063 Some(ConsumerIdentity {
15064 module_id: "opener".to_string(),
15065 launch_nonce: "opener-nonce".to_string(),
15066 }),
15067 )
15068 .await;
15069 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15070
15071 handler
15072 .cleanup_connection(old_opener_ctx.connection_id)
15073 .expect("old opener registration cleans up");
15074 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
15075 register_capability_manifest(
15076 &handler,
15077 &new_opener_ctx,
15078 &mut new_opener_rx,
15079 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15080 4,
15081 )
15082 .await;
15083
15084 let closed_channel = assert_capability_denied_push(
15085 client_rx
15086 .try_recv()
15087 .expect("HELLO deny addition must emit route.closed")
15088 .frame,
15089 "target",
15090 );
15091 let goodbye = client_rx
15092 .try_recv()
15093 .expect("HELLO deny addition must send the client a route GOODBYE")
15094 .frame;
15095 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15096 assert_eq!(goodbye.header.channel, closed_channel);
15097 assert_ne!(goodbye.header.channel, 0);
15098 assert_ne!(goodbye.header.epoch, 0);
15099 assert!(client_rx.try_recv().is_err());
15100 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15101 assert!(matches!(
15102 target_rx.try_recv(),
15103 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15104 ));
15105 }
15106
15107 #[tokio::test]
15108 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
15109 let registry = Arc::new(Registry::default());
15110 let forwarding = Arc::new(ForwardingTable::default());
15111 let supervisor = SupervisorHandle::new();
15112 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15113 let handler =
15114 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15115 .with_supervisor(supervisor);
15116 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
15117 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
15118 register_capability_manifest(
15119 &handler,
15120 &target_ctx,
15121 &mut target_rx,
15122 capability_manifest("target", &[], &[]),
15123 1,
15124 )
15125 .await;
15126 register_capability_manifest(
15127 &handler,
15128 &opener_ctx,
15129 &mut opener_rx,
15130 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15131 2,
15132 )
15133 .await;
15134 let (mut client_rx, _) = open_route_for_capability_test(
15135 &handler,
15136 &target_ctx,
15137 &mut target_rx,
15138 722,
15139 3,
15140 "target",
15141 Some(ConsumerIdentity {
15142 module_id: "opener".to_string(),
15143 launch_nonce: "opener-nonce".to_string(),
15144 }),
15145 )
15146 .await;
15147 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15148
15149 let replies = handler
15150 .handle_control_frame(
15151 &target_ctx,
15152 catalog_update_with_capabilities_frame(
15153 4,
15154 CapabilityDeclarations {
15155 provides: vec!["credentials-provider/v1".to_string()],
15156 requires: Vec::new(),
15157 must_never_reach: Vec::new(),
15158 },
15159 ),
15160 )
15161 .await
15162 .expect("claim catalog.update succeeds");
15163 assert!(matches!(
15164 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
15165 Ok(ModuleControlResponseToModule::CatalogUpdate {})
15166 ));
15167 let closed_channel = assert_capability_denied_push(
15168 client_rx
15169 .try_recv()
15170 .expect("claim addition must emit route.closed")
15171 .frame,
15172 "target",
15173 );
15174 let goodbye = client_rx
15175 .try_recv()
15176 .expect("claim addition must send the client a route GOODBYE")
15177 .frame;
15178 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15179 assert_eq!(goodbye.header.channel, closed_channel);
15180 assert_ne!(goodbye.header.channel, 0);
15181 assert_ne!(goodbye.header.epoch, 0);
15182 assert!(client_rx.try_recv().is_err());
15183 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15184 assert!(matches!(
15185 target_rx.try_recv(),
15186 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15187 ));
15188 }
15189
15190 #[tokio::test]
15191 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
15192 let registry = Arc::new(Registry::default());
15193 let forwarding = Arc::new(ForwardingTable::default());
15194 let supervisor = SupervisorHandle::new();
15195 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15196 let handler =
15197 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15198 .with_supervisor(supervisor);
15199 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
15200 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
15201 register_capability_manifest(
15202 &handler,
15203 &target_ctx,
15204 &mut target_rx,
15205 capability_manifest("target", &["credentials-provider/v1"], &[]),
15206 1,
15207 )
15208 .await;
15209 register_capability_manifest(
15210 &handler,
15211 &opener_ctx,
15212 &mut opener_rx,
15213 capability_manifest("opener", &[], &[]),
15214 2,
15215 )
15216 .await;
15217 let (mut client_rx, _) = open_route_for_capability_test(
15218 &handler,
15219 &target_ctx,
15220 &mut target_rx,
15221 732,
15222 3,
15223 "target",
15224 Some(ConsumerIdentity {
15225 module_id: "opener".to_string(),
15226 launch_nonce: "opener-nonce".to_string(),
15227 }),
15228 )
15229 .await;
15230 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15231
15232 handler
15233 .handle_control_frame(
15234 &target_ctx,
15235 catalog_update_with_capabilities_frame(
15236 4,
15237 CapabilityDeclarations {
15238 provides: Vec::new(),
15239 requires: Vec::new(),
15240 must_never_reach: Vec::new(),
15241 },
15242 ),
15243 )
15244 .await
15245 .expect("claim removal catalog.update succeeds");
15246 assert_eq!(
15247 forwarding.active_binding_count().unwrap(),
15248 1,
15249 "removing an attested target claim must leave the route census unchanged"
15250 );
15251 assert!(
15252 client_rx.try_recv().is_err(),
15253 "claim removal must not emit route.closed or a client route GOODBYE"
15254 );
15255 assert!(
15256 target_rx.try_recv().is_err(),
15257 "claim removal must not send the target a route GOODBYE"
15258 );
15259 }
15260
15261 #[tokio::test]
15264 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
15265 let registry = Arc::new(Registry::default());
15266 let forwarding = Arc::new(ForwardingTable::default());
15267 let supervisor = SupervisorHandle::new();
15268 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15269 let handler =
15270 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15271 .with_supervisor(supervisor);
15272 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
15273 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
15274 register_capability_manifest(
15275 &handler,
15276 &target_ctx,
15277 &mut target_rx,
15278 capability_manifest("target", &["credentials-provider/v1"], &[]),
15279 1,
15280 )
15281 .await;
15282 register_capability_manifest(
15283 &handler,
15284 &opener_ctx,
15285 &mut opener_rx,
15286 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15287 2,
15288 )
15289 .await;
15290
15291 let (_client_rx, bind) = open_route_for_capability_test(
15292 &handler,
15293 &target_ctx,
15294 &mut target_rx,
15295 742,
15296 3,
15297 "target",
15298 None,
15299 )
15300 .await;
15301 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
15302 panic!("direct scope-honesty route must bind");
15303 };
15304 assert_eq!(principal, Some(Principal::Direct));
15305 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15306 }
15307
15308 #[tokio::test]
15311 async fn must_never_reach_self_route_is_capability_forbidden() {
15312 let registry = Arc::new(Registry::default());
15313 let forwarding = Arc::new(ForwardingTable::default());
15314 let supervisor = SupervisorHandle::new();
15315 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
15316 let handler =
15317 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15318 .with_supervisor(supervisor);
15319 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
15320 register_capability_manifest(
15321 &handler,
15322 &self_ctx,
15323 &mut self_rx,
15324 capability_manifest(
15325 "self-provider",
15326 &["credentials-provider/v1"],
15327 &["credentials-provider/v1"],
15328 ),
15329 1,
15330 )
15331 .await;
15332
15333 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
15334 let replies = handler
15335 .handle_control_frame(
15336 &client_ctx,
15337 route_open_frame_with_admission_facts(
15338 2,
15339 "self-provider",
15340 unique_project_root("admission-facts"),
15341 Some(ConsumerIdentity {
15342 module_id: "self-provider".to_string(),
15343 launch_nonce: "self-nonce".to_string(),
15344 }),
15345 None,
15346 ),
15347 )
15348 .await
15349 .expect("self-route refusal returns a typed frame");
15350 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15351 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15352 assert!(
15353 self_rx.try_recv().is_err(),
15354 "self denial must not relay route.bind"
15355 );
15356 }
15357
15358 #[test]
15359 fn unsupported_channel_zero_frame_returns_error() {
15360 let handler = ControlHandler::default();
15361 let request = Frame::build(
15362 FrameType::Request,
15363 control_flags(),
15364 0,
15365 0,
15366 21,
15367 b"opaque".to_vec(),
15368 )
15369 .unwrap();
15370
15371 let response = handler
15372 .handle_control(ConnectionId::new(1), request)
15373 .unwrap();
15374
15375 assert_eq!(response[0].header.ty, FrameType::Error);
15376 assert_eq!(
15377 parse_error(&response[0])["code"],
15378 "unsupported_control_frame"
15379 );
15380 }
15381
15382 mod swap {
15387 use super::*;
15388
15389 const INCUMBENT: ConnectionId = ConnectionId::new(30);
15390 const CANDIDATE: ConnectionId = ConnectionId::new(40);
15391
15392 struct Swap {
15393 registry: Arc<Registry>,
15394 forwarding: Arc<ForwardingTable>,
15395 handler: ControlHandler,
15396 incumbent_ctx: RouteCtx,
15397 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15398 candidate_ctx: RouteCtx,
15399 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15400 }
15401
15402 async fn swap_with_incumbent() -> Swap {
15403 let registry = Arc::new(Registry::default());
15404 let forwarding = Arc::new(ForwardingTable::default());
15405 let handler =
15406 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15407 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
15408 hello_via_sink(
15409 &handler,
15410 &incumbent_ctx,
15411 &mut incumbent_rx,
15412 hello_frame("aft", PROTOCOL_VERSION, 7),
15413 )
15414 .await;
15415 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
15416 Swap {
15417 registry,
15418 forwarding,
15419 handler,
15420 incumbent_ctx,
15421 incumbent_rx,
15422 candidate_ctx,
15423 candidate_rx,
15424 }
15425 }
15426
15427 fn register_candidate(swap: &Swap, ready: Option<bool>) {
15428 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
15429 candidate_manifest.ready = ready;
15430 let registration = swap
15431 .registry
15432 .register_candidate_with_control_ops(
15433 candidate_manifest,
15434 PROTOCOL_VERSION,
15435 CANDIDATE,
15436 module_baseline_control_ops(),
15437 )
15438 .unwrap();
15439 swap.forwarding
15440 .register_candidate_module_connection(
15441 CANDIDATE,
15442 "aft".to_string(),
15443 PROTOCOL_VERSION,
15444 manifest_concurrency(®istration.manifest),
15445 swap.candidate_ctx.egress.clone(),
15446 )
15447 .unwrap();
15448 }
15449
15450 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
15451 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
15452 swap.registry.promote_candidate("aft").unwrap().unwrap();
15453 cutover.incumbent.unwrap()
15454 }
15455
15456 #[tokio::test]
15465 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
15466 let mut swap = swap_with_incumbent().await;
15467 let hold = Arc::new(std::sync::Barrier::new(2));
15468 let handler = swap
15469 .handler
15470 .clone()
15471 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
15472 handler
15473 .supervisor
15474 .open_swap("aft", "candidate-nonce".to_string());
15475 let hello = {
15476 let handler = handler.clone();
15477 let candidate_ctx = swap.candidate_ctx.clone();
15478 std::thread::spawn(move || {
15479 handler
15480 .handle_hello(
15481 candidate_ctx.connection_id,
15482 Some(candidate_ctx.egress.clone()),
15483 hello_frame_with_nonce(
15484 "aft",
15485 PROTOCOL_VERSION,
15486 8,
15487 Some("candidate-nonce"),
15488 ),
15489 )
15490 .unwrap()
15491 })
15492 };
15493
15494 hold.wait();
15495 assert!(swap
15498 .registry
15499 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
15500 .unwrap()
15501 .is_some());
15502 assert!(swap
15503 .registry
15504 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15505 .unwrap()
15506 .is_none());
15507 assert!(swap
15508 .forwarding
15509 .module_endpoint_for_connection(CANDIDATE)
15510 .unwrap()
15511 .is_none());
15512
15513 hold.wait();
15514 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
15515 assert_eq!(
15516 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
15517 FrameType::HelloAck
15518 );
15519 let published = swap
15520 .registry
15521 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15522 .unwrap()
15523 .expect("candidate is published once its HELLO completes");
15524 assert_eq!(published.connection_id, CANDIDATE);
15525 assert!(swap
15526 .forwarding
15527 .module_endpoint_for_connection(CANDIDATE)
15528 .unwrap()
15529 .is_some());
15530 }
15531
15532 fn keyed_total(counters: &Value, key: &str) -> u64 {
15533 counters[key]
15534 .as_object()
15535 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
15536 .unwrap_or(0)
15537 }
15538
15539 #[tokio::test]
15540 async fn replacement_logs_old_end_and_new_admission_once() {
15541 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15542 let mut swap = swap_with_incumbent().await;
15543 swap.handler
15544 .supervisor
15545 .open_swap("aft", "candidate-nonce".to_string());
15546 hello_via_sink(
15547 &swap.handler,
15548 &swap.candidate_ctx,
15549 &mut swap.candidate_rx,
15550 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
15551 )
15552 .await;
15553 cutover(&swap);
15554 swap.handler.cleanup_connection(INCUMBENT).unwrap();
15555
15556 let captured = crate::router::test_log::captured_logs(&logs);
15557 println!("captured replacement registry logs:\n{captured}");
15558 let new_admissions: Vec<_> = captured
15559 .lines()
15560 .filter(|line| {
15561 line.contains("connection_id=40")
15562 && line.contains("swap candidate registered; not routable until cutover")
15563 })
15564 .collect();
15565 assert_eq!(
15566 new_admissions.len(),
15567 1,
15568 "unexpected admission logs: {captured}"
15569 );
15570 assert!(new_admissions[0].contains("module_id=aft"));
15571 assert!(new_admissions[0].contains("connection_id=40"));
15572 assert!(
15573 new_admissions[0].contains("swap candidate registered; not routable until cutover")
15574 );
15575 assert!(new_admissions[0].contains("ready=true"));
15576
15577 let old_admissions: Vec<_> = captured
15578 .lines()
15579 .filter(|line| {
15580 line.contains("module registered module_id=aft ")
15581 && line.contains("connection_id=30")
15582 })
15583 .collect();
15584 assert_eq!(
15585 old_admissions.len(),
15586 1,
15587 "unexpected incumbent admission: {captured}"
15588 );
15589
15590 let old_ends: Vec<_> = captured
15591 .lines()
15592 .filter(|line| {
15593 line.contains("connection_id=30") && line.contains("module registration ended")
15594 })
15595 .collect();
15596 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
15597 assert!(old_ends[0].contains("module_id=aft"));
15598 assert!(old_ends[0].contains("connection_id=30"));
15599 assert!(old_ends[0].contains("reason=replaced"));
15600 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
15601 assert_eq!(
15602 captured
15603 .lines()
15604 .filter(|line| line.contains("module registration promoted"))
15605 .count(),
15606 1,
15607 "unexpected promotion logs: {captured}"
15608 );
15609 }
15610
15611 #[tokio::test]
15617 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
15618 let mut swap = swap_with_incumbent().await;
15619 let handler = swap.handler.clone();
15620
15621 let cotenant = ConnectionId::new(31);
15623 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
15624 let (cotenant_task, cotenant_bind) = relay_route_open(
15625 &handler,
15626 cotenant,
15627 &cotenant_ctx.egress,
15628 &mut swap.incumbent_rx,
15629 100,
15630 "aft",
15631 "swap-cotenant",
15632 )
15633 .await;
15634 handler
15635 .handle_control_frame(
15636 &swap.incumbent_ctx,
15637 route_bind_ack(cotenant_bind.header.corr),
15638 )
15639 .await
15640 .unwrap();
15641 assert!(cotenant_task.await.unwrap().is_empty());
15642 let (cotenant_channel, cotenant_epoch) =
15643 published_route(&cotenant_rx.recv().await.unwrap());
15644
15645 let caller = ConnectionId::new(32);
15647 let (caller_ctx, mut caller_rx) = route_ctx(caller);
15648 let (caller_task, caller_bind) = relay_route_open(
15649 &handler,
15650 caller,
15651 &caller_ctx.egress,
15652 &mut swap.incumbent_rx,
15653 101,
15654 "aft",
15655 "swap-caller",
15656 )
15657 .await;
15658 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
15659
15660 register_candidate(&swap, None);
15661 cutover(&swap);
15662
15663 let ack = handler
15665 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
15666 .await;
15667 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
15668 if module_loop_error.is_some() {
15669 handler.cleanup_connection(INCUMBENT).unwrap();
15672 }
15673
15674 assert!(
15676 cotenant_rx.try_recv().is_err(),
15677 "the co-tenant route on the incumbent was torn down by one late ack: \
15678 {module_loop_error:?}"
15679 );
15680 assert!(matches!(
15681 swap.forwarding
15682 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
15683 .unwrap(),
15684 DataRoute::Client(DataRouteState::Bound(_))
15685 ));
15686 assert_eq!(module_loop_error, None);
15687 assert!(swap
15688 .registry
15689 .get_module_by_connection(INCUMBENT)
15690 .unwrap()
15691 .is_some());
15692
15693 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
15695 .await
15696 .expect("the incumbent is told to drop the abandoned binding")
15697 .unwrap()
15698 .frame;
15699 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15700 assert_eq!(goodbye.header.channel, abandoned_channel);
15701 assert_eq!(goodbye.header.epoch, abandoned_epoch);
15702 assert!(swap.incumbent_rx.try_recv().is_err());
15703
15704 let response = caller_task.await.unwrap();
15706 assert_eq!(response.len(), 1);
15707 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15708 assert!(caller_rx.try_recv().is_err());
15709
15710 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
15714 let counters = handler.counters().snapshot();
15715 assert_eq!(
15716 keyed_total(&counters, "route_open_accepted_by_principal"),
15717 1
15718 );
15719 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
15720 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
15721 }
15722
15723 #[tokio::test]
15727 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
15728 let mut swap = swap_with_incumbent().await;
15729 register_candidate(&swap, None);
15730 let incumbent = cutover(&swap);
15731 swap.forwarding
15732 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
15733 .unwrap()
15734 .expect("the incumbent is still registered");
15735
15736 let client = ConnectionId::new(33);
15737 let (client_ctx, mut client_rx) = route_ctx(client);
15738 let route_handler = swap.handler.clone();
15739 let open_ctx = RouteCtx {
15740 connection_id: client,
15741 egress: client_ctx.egress.clone(),
15742 };
15743 let mut route_task = tokio::spawn(async move {
15744 route_handler
15745 .handle_control_frame(
15746 &open_ctx,
15747 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
15748 )
15749 .await
15750 .unwrap()
15751 });
15752 let bind = tokio::select! {
15753 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
15754 response = &mut route_task => {
15755 let response = response.unwrap();
15756 panic!(
15757 "post-cutover route.open was refused instead of relayed to the candidate: {}",
15758 parse_error(&response[0])["code"]
15759 );
15760 }
15761 };
15762 swap.handler
15763 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
15764 .await
15765 .unwrap();
15766 assert!(route_task.await.unwrap().is_empty());
15767 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
15768 match swap
15769 .forwarding
15770 .lookup_data_route(client, channel, epoch)
15771 .unwrap()
15772 {
15773 DataRoute::Client(DataRouteState::Bound(route)) => {
15774 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
15775 }
15776 other => panic!("expected a bound route on the candidate, got {other:?}"),
15777 }
15778 assert!(swap.incumbent_rx.try_recv().is_err());
15779 }
15780
15781 #[tokio::test]
15786 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
15787 let swap = swap_with_incumbent().await;
15788 register_candidate(&swap, Some(false));
15789 let update = Frame::build(
15790 FrameType::Request,
15791 control_flags(),
15792 0,
15793 0,
15794 55,
15795 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
15796 provides: manifest("aft", PROTOCOL_VERSION).provides,
15797 capabilities: None,
15798 ready: Some(true),
15799 })
15800 .unwrap(),
15801 )
15802 .unwrap();
15803
15804 let replies = swap
15805 .handler
15806 .handle_control_frame(&swap.candidate_ctx, update)
15807 .await
15808 .unwrap();
15809
15810 assert_eq!(replies.len(), 1);
15811 assert_eq!(
15812 replies[0].header.ty,
15813 FrameType::Response,
15814 "candidate catalog.update was refused: {:?}",
15815 serde_json::from_slice::<Value>(&replies[0].body).ok()
15816 );
15817 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
15818 assert_eq!(
15819 swap.registry
15820 .get_module("aft")
15821 .unwrap()
15822 .unwrap()
15823 .connection_id,
15824 INCUMBENT
15825 );
15826 }
15827 }
15828
15829 mod swap_admission {
15833 use super::*;
15834
15835 const INCUMBENT_NONCE: &str = "incumbent-nonce";
15836 const CANDIDATE_NONCE: &str = "candidate-nonce";
15837
15838 fn handler_with_incumbent(
15839 module_id: &str,
15840 reserved: bool,
15841 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
15842 let registry = Arc::new(Registry::default());
15843 let supervisor = SupervisorHandle::new();
15844 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
15845 if reserved {
15846 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
15847 }
15848 let handler =
15849 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
15850 let incumbent = handler
15851 .handle_control(
15852 ConnectionId::new(1),
15853 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
15854 )
15855 .unwrap();
15856 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
15857 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
15858 (registry, supervisor, handler)
15859 }
15860
15861 #[test]
15867 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
15868 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
15869
15870 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
15871 let replies = handler
15872 .handle_control(
15873 ConnectionId::new(connection),
15874 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
15875 )
15876 .unwrap();
15877 assert_eq!(replies[0].header.ty, FrameType::Error);
15878 assert_eq!(
15879 parse_error(&replies[0])["code"],
15880 "swap_token_invalid",
15881 "nonce {nonce:?}"
15882 );
15883 }
15884 assert!(registry.get_candidate("aft").unwrap().is_none());
15885 assert_eq!(
15886 registry.get_module("aft").unwrap().unwrap().connection_id,
15887 ConnectionId::new(1)
15888 );
15889
15890 let admitted = handler
15893 .handle_control(
15894 ConnectionId::new(4),
15895 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
15896 )
15897 .unwrap();
15898 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
15899 assert_eq!(
15900 registry
15901 .get_candidate("aft")
15902 .unwrap()
15903 .unwrap()
15904 .connection_id,
15905 ConnectionId::new(4)
15906 );
15907 assert_eq!(
15908 registry.get_module("aft").unwrap().unwrap().connection_id,
15909 ConnectionId::new(1),
15910 "the candidate must not take the active slot"
15911 );
15912 let replayed = handler
15913 .handle_control(
15914 ConnectionId::new(5),
15915 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
15916 )
15917 .unwrap();
15918 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
15919
15920 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
15924 let squatter = handler
15925 .handle_control(
15926 ConnectionId::new(6),
15927 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
15928 )
15929 .unwrap();
15930 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
15931 assert!(
15932 registry.get_module("aft").unwrap().is_none(),
15933 "a squatter took the active slot of an id being swapped"
15934 );
15935 }
15936
15937 #[test]
15942 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
15943 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
15944
15945 let replies = handler
15946 .handle_control(
15947 ConnectionId::new(2),
15948 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15949 )
15950 .unwrap();
15951
15952 assert_eq!(
15953 replies[0].header.ty,
15954 FrameType::HelloAck,
15955 "reserved candidate refused: {:?}",
15956 serde_json::from_slice::<Value>(&replies[0].body).ok()
15957 );
15958 assert_eq!(
15959 registry
15960 .get_candidate("vault")
15961 .unwrap()
15962 .unwrap()
15963 .connection_id,
15964 ConnectionId::new(2)
15965 );
15966 }
15967
15968 #[test]
15971 fn without_an_open_swap_the_ordinary_gates_decide() {
15972 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
15973 supervisor.close_swap("vault");
15974
15975 let candidate = handler
15976 .handle_control(
15977 ConnectionId::new(2),
15978 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15979 )
15980 .unwrap();
15981 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
15982 let duplicate = handler
15983 .handle_control(
15984 ConnectionId::new(3),
15985 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
15986 )
15987 .unwrap();
15988 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15989 assert!(registry.get_candidate("vault").unwrap().is_none());
15990 }
15991 }
15992
15993 mod scopes {
15997 mod idle_tests {
15998 use super::*;
15999 include!("scope_idle_tests.rs");
16000 }
16001 mod route_open_timings {
16002 use super::*;
16003 include!("route_open_timing_tests.rs");
16004 }
16005 mod route_open_refusal_log {
16006 use super::*;
16007 include!("route_open_refusal_log_tests.rs");
16008 }
16009 mod activity {
16010 include!("activity_scope_tests.rs");
16011 }
16012 mod delta {
16013 include!("scope_handler_delta_tests.rs");
16014 }
16015 mod call_modes {
16016 include!("scope_call_mode_tests.rs");
16017 }
16018 #[tokio::test]
16019 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
16020 let rig = rig().await;
16021 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
16022 "s".into(),
16023 1,
16024 "tool".into(),
16025 ))
16026 .unwrap();
16027 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
16028 let (direct, _rx) = wide_ctx(999);
16029 for (ctx, expected) in [
16030 (&rig.owner, FrameType::Response),
16031 (&direct, FrameType::Error),
16032 ] {
16033 let frame =
16034 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
16035 .unwrap();
16036 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
16037 assert_eq!(replies[0].header.ty, expected);
16038 if expected == FrameType::Response {
16039 let reply: subc_control::ScopeServedByReply =
16040 serde_json::from_slice(&replies[0].body).unwrap();
16041 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
16042 } else {
16043 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
16044 }
16045 }
16046 let reply = call(
16047 &rig.handler,
16048 &rig.owner,
16049 &ModuleControlRequestFromModule::ScopeDescribe {
16050 owner: Principal::Reserved {
16051 module_id: OWNER.into(),
16052 },
16053 scope_ref: "s".into(),
16054 },
16055 )
16056 .await;
16057 assert!(matches!(
16058 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
16059 ModuleControlResponseToModule::ScopeDescribe { .. }
16060 ));
16061 }
16062
16063 use subc_protocol::scope::{
16064 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
16065 ScopeStatus,
16066 };
16067
16068 use super::*;
16069
16070 const OWNER: &str = "prefrontal-core";
16071
16072 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
16073 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
16074 }
16075
16076 async fn call(
16077 handler: &ControlHandler,
16078 ctx: &RouteCtx,
16079 request: &ModuleControlRequestFromModule,
16080 ) -> Frame {
16081 let body = serde_json::to_vec(request).unwrap();
16082 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
16083 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
16084 assert_eq!(replies.len(), 1, "{replies:?}");
16085 replies.pop().unwrap()
16086 }
16087
16088 async fn sync(
16089 handler: &ControlHandler,
16090 ctx: &RouteCtx,
16091 generation: u64,
16092 scopes: Vec<ScopeRecord>,
16093 ) -> Result<ModuleControlResponseToModule, String> {
16094 let reply = call(
16095 handler,
16096 ctx,
16097 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
16098 )
16099 .await;
16100 match reply.header.ty {
16101 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
16102 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
16103 }
16104 }
16105
16106 async fn describe(
16107 handler: &ControlHandler,
16108 ctx: &RouteCtx,
16109 owner: &str,
16110 scope_ref: &str,
16111 ) -> ModuleControlResponseToModule {
16112 let reply = call(
16113 handler,
16114 ctx,
16115 &ModuleControlRequestFromModule::ScopeDescribe {
16116 owner: Principal::Reserved {
16117 module_id: owner.to_string(),
16118 },
16119 scope_ref: scope_ref.to_string(),
16120 },
16121 )
16122 .await;
16123 assert_eq!(
16124 reply.header.ty,
16125 FrameType::Response,
16126 "{:?}",
16127 parse_error(&reply)
16128 );
16129 serde_json::from_slice(&reply.body).unwrap()
16130 }
16131
16132 #[tokio::test]
16133 async fn scope_describe_never_echoes_bind_proof_or_hash() {
16134 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16135 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16136 let mut rig = rig().await;
16137 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16138 .await;
16139 let _bound = rig
16140 .bound(
16141 Some(AFT),
16142 PLEXUS,
16143 Some(
16144 rig.selector("s", Some(1))
16145 .with_bind_proof(Some(PROOF.into())),
16146 ),
16147 )
16148 .await;
16149 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
16150 let output = serde_json::to_string(&reply).unwrap();
16151 assert!(output.contains("\"status\":\"live\""));
16152 assert!(!output.contains(PROOF));
16153 assert!(!output.contains(HASH));
16154 }
16155
16156 async fn module(
16158 handler: &ControlHandler,
16159 connection: u64,
16160 module_id: &str,
16161 nonce: Option<&str>,
16162 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16163 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
16164 hello_via_sink(
16165 handler,
16166 &ctx,
16167 &mut rx,
16168 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
16169 )
16170 .await;
16171 (ctx, rx)
16172 }
16173
16174 #[tokio::test]
16177 async fn a_client_connection_cannot_sync_or_describe() {
16178 let handler = ControlHandler::new(Arc::new(Registry::default()));
16179 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
16180 for request in [
16181 ModuleControlRequestFromModule::ScopeSync {
16182 generation: 1,
16183 scopes: vec![head("s", 1)],
16184 },
16185 ModuleControlRequestFromModule::ScopeDescribe {
16186 owner: Principal::Direct,
16187 scope_ref: "s".to_string(),
16188 },
16189 ] {
16190 let reply = call(&handler, &ctx, &request).await;
16191 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
16192 }
16193 assert!(
16194 !handler
16195 .scopes
16196 .read()
16197 .unwrap()
16198 .describe(
16199 &Principal::Reserved {
16200 module_id: OWNER.to_string()
16201 },
16202 "s"
16203 )
16204 .owner_synced
16205 );
16206 }
16207
16208 #[tokio::test]
16211 async fn a_module_without_a_supervised_launch_cannot_sync() {
16212 let handler = ControlHandler::new(Arc::new(Registry::default()));
16213 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
16214 assert_eq!(
16215 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
16216 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16217 );
16218 }
16219
16220 #[tokio::test]
16221 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
16222 let supervisor = SupervisorHandle::new();
16223 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16224 let handler = ControlHandler::new(Arc::new(Registry::default()))
16225 .with_supervisor(supervisor.clone());
16226 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16227 sync(&handler, &incumbent, 1, vec![head("s", 1)])
16228 .await
16229 .expect("the current launch syncs");
16230
16231 supervisor.open_swap(OWNER, "n2".to_string());
16234 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
16235 assert_eq!(
16236 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16237 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16238 );
16239 sync(&handler, &incumbent, 2, vec![head("s", 1)])
16240 .await
16241 .expect("the serving owner syncs during the swap");
16242
16243 supervisor.close_swap(OWNER);
16246 assert_eq!(
16247 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16248 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16249 );
16250 sync(&handler, &incumbent, 3, vec![head("s", 1)])
16251 .await
16252 .expect("the serving owner syncs after the rollback");
16253 handler.cleanup_connection(candidate.connection_id).unwrap();
16254
16255 supervisor.open_swap(OWNER, "n3".to_string());
16260 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
16261 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
16262 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
16263 .await
16264 .expect("the promoted launch takes authority");
16265 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16266 panic!("unexpected reply {reply:?}");
16267 };
16268 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
16269 assert_eq!(
16270 sync(&handler, &incumbent, 4, Vec::new()).await,
16271 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16272 );
16273 }
16274
16275 #[tokio::test]
16278 async fn closing_the_authority_connection_frees_sync_authority() {
16279 let supervisor = SupervisorHandle::new();
16280 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16281 let handler = ControlHandler::new(Arc::new(Registry::default()))
16282 .with_supervisor(supervisor.clone());
16283 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16284 sync(&handler, &first, 10, vec![head("s", 1)])
16285 .await
16286 .unwrap();
16287 handler.cleanup_connection(first.connection_id).unwrap();
16288
16289 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
16290 sync(&handler, &second, 1, vec![head("s", 1)])
16291 .await
16292 .expect("the next connection takes the released authority");
16293 }
16294
16295 #[tokio::test]
16296 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
16297 let supervisor = SupervisorHandle::new();
16298 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16299 let handler =
16300 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
16301 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
16302 sync(&handler, &first, 10, vec![head("s", 1)])
16303 .await
16304 .unwrap();
16305 handler
16306 .handle_control_frame(
16307 &first,
16308 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
16309 )
16310 .await
16311 .unwrap();
16312 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
16313 sync(&handler, &second, 1, vec![head("s", 1)])
16314 .await
16315 .expect("GOODBYE releases authority even if the old socket remains open");
16316 }
16317
16318 #[tokio::test]
16319 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
16320 let registry = Arc::new(Registry::default());
16321 let supervisor_handle = SupervisorHandle::new();
16322 let supervisor =
16323 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
16324 .with_handle(supervisor_handle.clone())
16325 .with_daemon_incarnation("incarnation-7".to_string());
16326 supervisor
16329 .supervise_configured(
16330 ModuleSpec {
16331 module_id: OWNER.to_string(),
16332 program: PathBuf::from("/nonexistent/prefrontal-core"),
16333 args: Vec::new(),
16334 env: Vec::new(),
16335 reserved: false,
16336 reserved_prefixes: Vec::new(),
16337 protocol: ModuleProtocol::Subc,
16338 overlap: Default::default(),
16339 },
16340 false,
16341 )
16342 .unwrap();
16343 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
16344 let handler =
16345 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
16346 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
16347
16348 let ModuleControlResponseToModule::ScopeDescribe {
16350 status,
16351 daemon_incarnation,
16352 owner_synced,
16353 owner_configured,
16354 scope,
16355 ..
16356 } = describe(&handler, &reader, OWNER, "s").await
16357 else {
16358 panic!("not a describe reply");
16359 };
16360 assert_eq!(status, ScopeStatus::NotLive);
16361 assert_eq!(daemon_incarnation, "incarnation-7");
16362 assert!(!owner_synced);
16363 assert!(owner_configured);
16364 assert!(scope.is_none());
16365
16366 let ModuleControlResponseToModule::ScopeDescribe {
16369 status,
16370 owner_configured,
16371 ..
16372 } = describe(&handler, &reader, "ghost", "s").await
16373 else {
16374 panic!("not a describe reply");
16375 };
16376 assert_eq!(status, ScopeStatus::NotLive);
16377 assert!(!owner_configured);
16378
16379 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
16381 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
16382 let ModuleControlResponseToModule::ScopeDescribe {
16383 status,
16384 scope_epoch,
16385 owner_synced,
16386 scope,
16387 ..
16388 } = describe(&handler, &reader, OWNER, "s").await
16389 else {
16390 panic!("not a describe reply");
16391 };
16392 assert_eq!(status, ScopeStatus::Live);
16393 assert_eq!(scope_epoch, Some(4));
16394 assert!(owner_synced);
16395 let stamp = scope.expect("a live scope carries its stamp");
16396 assert!(
16397 stamp.owner_authorized,
16398 "prefrontal-core is the default authority"
16399 );
16400 assert_eq!(stamp.kind, ScopeKind::Head);
16401 }
16402
16403 #[tokio::test]
16404 async fn scope_authority_owners_decides_owner_authorized() {
16405 let supervisor = SupervisorHandle::new();
16406 supervisor.set_spawn_nonce("broca", "b1".to_string());
16407 let handler = ControlHandler::new(Arc::new(Registry::default()))
16408 .with_supervisor(supervisor)
16409 .with_scope_authority_owners(vec!["broca".to_string()]);
16410 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
16411 let mut gated = head("s", 1);
16412 gated.attributes.agent_id = Some("agent".to_string());
16413 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
16414 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
16415 describe(&handler, &broca, "broca", "s").await
16416 else {
16417 panic!("not a describe reply");
16418 };
16419 assert!(scope.unwrap().owner_authorized);
16420 }
16421
16422 #[tokio::test]
16426 async fn scope_ops_and_the_scopes_capability_are_advertised() {
16427 let handler = ControlHandler::new(Arc::new(Registry::default()));
16428 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
16429 let ack = hello_via_sink(
16430 &handler,
16431 &ctx,
16432 &mut rx,
16433 hello_frame("m", PROTOCOL_VERSION, 1),
16434 )
16435 .await;
16436 let ack = parse_ack(&ack);
16437 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
16438 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
16439 }
16440 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
16441
16442 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
16443 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16444 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16445 let reply = handler
16446 .handle_control_frame(&client, frame)
16447 .await
16448 .unwrap()
16449 .pop()
16450 .unwrap();
16451 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16452 serde_json::from_slice(&reply.body).unwrap()
16453 else {
16454 panic!("not a server.describe reply");
16455 };
16456 assert!(
16457 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
16458 "{capabilities:?}"
16459 );
16460 }
16461
16462 const PLEXUS: &str = "plexus";
16465 const OTHER: &str = "other";
16466 const AFT: &str = "aft";
16467 const BROCA: &str = "broca";
16468 const MAGIC: &str = "magic-context";
16469
16470 fn nonce(module_id: &str) -> String {
16471 format!("nonce-{module_id}")
16472 }
16473
16474 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16475 let (tx, rx) = mpsc::channel(64);
16476 (
16477 RouteCtx {
16478 connection_id: ConnectionId::new(connection),
16479 egress: FrameSink::new(tx),
16480 },
16481 rx,
16482 )
16483 }
16484
16485 struct Rig {
16489 handler: ControlHandler,
16490 forwarding: Arc<ForwardingTable>,
16491 owner: RouteCtx,
16492 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16493 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
16494 generation: u64,
16495 next_connection: u64,
16496 _supervisor: Supervisor,
16497 }
16498
16499 async fn rig() -> Rig {
16500 rig_with_flow_support(true).await
16501 }
16502
16503 async fn rig_with_flow_support(flow_support: bool) -> Rig {
16504 rig_with_scope_capabilities(flow_support, false).await
16505 }
16506
16507 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
16508 let mut provides = Vec::new();
16509 if flow_support {
16510 provides.push("flow-scopes/v1");
16511 }
16512 if run_support {
16513 provides.push("agent-run-scopes/v1");
16514 }
16515 rig_with_target_capabilities(&provides).await
16516 }
16517
16518 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
16519 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
16520 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16521 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16524 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
16525 Frame::build(
16526 FrameType::Hello,
16527 control_flags(),
16528 0,
16529 0,
16530 connection,
16531 serde_json::to_vec(&body).unwrap(),
16532 )
16533 .unwrap()
16534 }
16535
16536 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
16537 let registry = Arc::new(Registry::default());
16538 let forwarding = Arc::new(ForwardingTable::default());
16539 let supervisor_handle = SupervisorHandle::new();
16540 let supervisor =
16541 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
16542 .with_handle(supervisor_handle.clone());
16543 supervisor
16544 .supervise_configured(
16545 ModuleSpec {
16546 module_id: OWNER.to_string(),
16547 program: PathBuf::from("/nonexistent/prefrontal-core"),
16548 args: Vec::new(),
16549 env: Vec::new(),
16550 reserved: false,
16551 reserved_prefixes: Vec::new(),
16552 protocol: ModuleProtocol::Subc,
16553 overlap: Default::default(),
16554 },
16555 false,
16556 )
16557 .unwrap();
16558 for module_id in [OWNER, AFT, BROCA, MAGIC] {
16559 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
16560 }
16561 let handler =
16562 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16563 .with_supervisor(supervisor_handle);
16564 let (owner, mut owner_rx) = wide_ctx(1);
16565 hello_via_sink(
16566 &handler,
16567 &owner,
16568 &mut owner_rx,
16569 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
16570 )
16571 .await;
16572 let mut modules = BTreeMap::new();
16573 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
16574 let (ctx, mut rx) = wide_ctx(connection);
16575 let hello = hello_with_capabilities(module_id, connection, provides);
16576 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
16577 modules.insert(module_id.to_string(), (ctx, rx));
16578 }
16579 Rig {
16580 handler,
16581 forwarding,
16582 owner,
16583 _owner_rx: owner_rx,
16584 modules,
16585 generation: 0,
16586 next_connection: 100,
16587 _supervisor: supervisor,
16588 }
16589 }
16590
16591 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
16592 ScopeCarrier::new(Principal::Reserved {
16593 module_id: module_id.to_string(),
16594 })
16595 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
16596 }
16597
16598 fn session(scope_epoch: u64) -> ScopeRecord {
16601 let mut record = head("s", scope_epoch);
16602 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
16603 record.attributes.agent_id = Some("agent-1".to_string());
16604 record.attributes.delegates = true;
16605 record
16606 }
16607
16608 impl Rig {
16609 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
16610 self.generation += 1;
16611 sync(&self.handler, &self.owner, self.generation, scopes)
16612 .await
16613 .expect("the owner's sync is accepted");
16614 }
16615
16616 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16617 ScopeSelector::new(
16618 Principal::Reserved {
16619 module_id: OWNER.to_string(),
16620 },
16621 scope_ref.to_string(),
16622 scope_epoch,
16623 )
16624 }
16625
16626 fn open_frame(
16627 &mut self,
16628 opener: Option<&str>,
16629 target: &str,
16630 scope: Option<ScopeSelector>,
16631 ) -> (
16632 RouteCtx,
16633 mpsc::Receiver<crate::router::OutboundFrame>,
16634 Frame,
16635 ) {
16636 self.next_connection += 1;
16637 let (ctx, rx) = wide_ctx(self.next_connection);
16638 let root = unique_project_root("scoped-open");
16639 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
16640 target: RouteTarget::ToolProvider {
16641 module_id: target.to_string(),
16642 },
16643 identity: BindIdentity::new(
16644 root.path().to_path_buf(),
16645 "unit".to_string(),
16646 "session".to_string(),
16647 ),
16648 consumer_identity: opener.map(|module_id| ConsumerIdentity {
16649 module_id: module_id.to_string(),
16650 launch_nonce: nonce(module_id),
16651 }),
16652 consumer_capabilities: None,
16653 role_versions: None,
16654 admission_facts: None,
16655 scope,
16656 })
16657 .unwrap();
16658 let frame = Frame::build(
16659 FrameType::Request,
16660 control_flags(),
16661 0,
16662 0,
16663 self.next_connection,
16664 body,
16665 )
16666 .unwrap();
16667 (ctx, rx, frame)
16668 }
16669
16670 async fn refused(
16672 &mut self,
16673 opener: Option<&str>,
16674 target: &str,
16675 scope: Option<ScopeSelector>,
16676 ) -> String {
16677 self.refusal_body(opener, target, scope).await["code"]
16678 .as_str()
16679 .unwrap()
16680 .to_string()
16681 }
16682
16683 async fn refusal_body(
16684 &mut self,
16685 opener: Option<&str>,
16686 target: &str,
16687 scope: Option<ScopeSelector>,
16688 ) -> Value {
16689 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
16690 let replies = tokio::time::timeout(
16691 Duration::from_secs(2),
16692 self.handler.handle_control_frame(&ctx, frame),
16693 )
16694 .await
16695 .expect("the open must be refused before waiting for a bind ack")
16696 .unwrap();
16697 assert_eq!(replies.len(), 1, "{replies:?}");
16698 assert_eq!(replies[0].header.ty, FrameType::Error);
16699 let (_, module_rx) = self.modules.get_mut(target).unwrap();
16700 assert!(
16701 module_rx.try_recv().is_err(),
16702 "a refused open relays nothing"
16703 );
16704 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
16705 parse_error(&replies[0])
16706 }
16707
16708 async fn relayed(
16710 &mut self,
16711 opener: Option<&str>,
16712 target: &str,
16713 scope: Option<ScopeSelector>,
16714 ) -> Relayed {
16715 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
16716 let handler = self.handler.clone();
16717 let task_ctx = ctx.clone();
16718 let task = tokio::spawn(async move {
16719 handler
16720 .handle_control_frame(&task_ctx, frame)
16721 .await
16722 .unwrap()
16723 });
16724 let (_, module_rx) = self.modules.get_mut(target).unwrap();
16725 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
16726 .await
16727 .expect("the target receives the relayed route.bind")
16728 .unwrap()
16729 .frame;
16730 Relayed {
16731 target: target.to_string(),
16732 client: ctx,
16733 client_rx: rx,
16734 task,
16735 bind,
16736 }
16737 }
16738
16739 async fn ack(&self, relayed: &Relayed) {
16740 let (module, _) = &self.modules[&relayed.target];
16741 self.handler
16742 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
16743 .await
16744 .unwrap();
16745 }
16746
16747 async fn bound(
16749 &mut self,
16750 opener: Option<&str>,
16751 target: &str,
16752 scope: Option<ScopeSelector>,
16753 ) -> Bound {
16754 let relayed = self.relayed(opener, target, scope).await;
16755 self.ack(&relayed).await;
16756 let Relayed {
16757 target,
16758 client,
16759 mut client_rx,
16760 task,
16761 bind,
16762 } = relayed;
16763 assert!(
16764 task.await.unwrap().is_empty(),
16765 "the open is answered by commit"
16766 );
16767 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
16768 Bound {
16769 target,
16770 client,
16771 client_rx,
16772 channel,
16773 epoch,
16774 bind,
16775 }
16776 }
16777
16778 fn live(&self, route: &Bound) -> bool {
16779 matches!(
16780 self.forwarding
16781 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16782 .unwrap(),
16783 DataRoute::Client(DataRouteState::Bound(_))
16784 )
16785 }
16786 }
16787
16788 struct Relayed {
16789 target: String,
16790 client: RouteCtx,
16791 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16792 task: tokio::task::JoinHandle<Vec<Frame>>,
16793 bind: Frame,
16794 }
16795
16796 struct Bound {
16797 target: String,
16798 client: RouteCtx,
16799 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16800 channel: u16,
16801 epoch: u32,
16802 bind: Frame,
16803 }
16804
16805 impl Bound {
16806 fn closed_reason(&mut self) -> RouteCloseReason {
16809 let mut reason = None;
16810 let mut goodbye = false;
16811 while let Ok(outbound) = self.client_rx.try_recv() {
16812 let frame = outbound.frame;
16813 match frame.header.ty {
16814 FrameType::Goodbye => {
16815 assert_eq!(
16816 (frame.header.channel, frame.header.epoch),
16817 (self.channel, self.epoch)
16818 );
16819 goodbye = true;
16820 }
16821 FrameType::Push => {
16822 let ClientControlPush::RouteClosed {
16823 reason: r,
16824 module_id,
16825 ..
16826 } = serde_json::from_slice(&frame.body).unwrap()
16827 else {
16828 panic!("unexpected push");
16829 };
16830 assert_eq!(module_id, self.target);
16831 reason = Some(r);
16832 }
16833 other => panic!("unexpected frame {other:?}"),
16834 }
16835 }
16836 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
16837 reason.expect("the client is told why the route closed")
16838 }
16839
16840 fn untouched(&mut self) -> bool {
16841 self.client_rx.try_recv().is_err()
16842 }
16843
16844 fn stamp(&self) -> Option<ScopeStamp> {
16845 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
16846 ModuleControlRequest::RouteBind { scope, .. } => scope,
16847 other => panic!("expected a route.bind, got {other:?}"),
16848 }
16849 }
16850 }
16851
16852 #[tokio::test]
16853 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
16854 ) {
16855 let mut rig = rig().await;
16856 let mut record = session(1);
16857 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
16858 record.child_owners = vec![Principal::Reserved {
16859 module_id: MAGIC.to_string(),
16860 }];
16861 rig.sync(vec![record]).await;
16862 let scope = || Some(rig_selector("s", Some(1)));
16863
16864 rig.bound(Some(OWNER), PLEXUS, scope()).await;
16867 rig.bound(Some(AFT), OTHER, scope()).await;
16868 rig.bound(Some(BROCA), PLEXUS, scope()).await;
16869
16870 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
16874 assert_eq!(
16875 rig.refused(opener, target, scope()).await,
16876 error_codes::SCOPE_NOT_CARRIER,
16877 "{opener:?} -> {target}"
16878 );
16879 }
16880 }
16881
16882 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16883 ScopeSelector::new(
16884 Principal::Reserved {
16885 module_id: OWNER.to_string(),
16886 },
16887 scope_ref.to_string(),
16888 scope_epoch,
16889 )
16890 }
16891
16892 #[tokio::test]
16893 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
16894 let mut rig = rig().await;
16895 rig.sync(vec![session(1)]).await;
16896 for opener in [OWNER, AFT] {
16897 assert_eq!(
16898 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
16899 .await,
16900 error_codes::SCOPE_EPOCH_REQUIRED,
16901 "{opener}"
16902 );
16903 }
16904 }
16905
16906 #[tokio::test]
16907 async fn admission_separates_not_synced_not_live_and_ended() {
16908 let mut rig = rig().await;
16909 let code = rig
16911 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16912 .await;
16913 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
16914 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
16915
16916 let ghost = ScopeSelector::new(
16918 Principal::Reserved {
16919 module_id: "ghost".to_string(),
16920 },
16921 "s".to_string(),
16922 Some(1),
16923 );
16924 assert_eq!(
16925 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
16926 error_codes::SCOPE_NOT_LIVE
16927 );
16928
16929 rig.sync(vec![session(2)]).await;
16930 assert_eq!(
16931 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
16932 .await,
16933 error_codes::SCOPE_NOT_LIVE
16934 );
16935 for epoch in [1, 3] {
16936 assert_eq!(
16937 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
16938 .await,
16939 error_codes::SCOPE_ENDED,
16940 "epoch {epoch}"
16941 );
16942 }
16943 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
16945 .await;
16946 }
16947
16948 #[tokio::test]
16949 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
16950 let mut rig = rig().await;
16951 rig.sync(vec![session(1)]).await;
16952 let route = rig
16953 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16954 .await;
16955 let stamp = route.stamp().expect("a scoped bind carries the stamp");
16956 assert_eq!(stamp.scope_ref, "s");
16957 assert_eq!(stamp.scope_epoch, 1);
16958 assert_eq!(stamp.kind, ScopeKind::Head);
16959 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
16960 assert!(stamp.attributes.delegates);
16961 assert!(stamp.owner_authorized);
16962 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16963 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
16964
16965 let (broca, mut broca_rx) = wide_ctx(50);
16968 hello_via_sink(
16969 &rig.handler,
16970 &broca,
16971 &mut broca_rx,
16972 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
16973 )
16974 .await;
16975 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
16976 .await
16977 .unwrap();
16978 let own = ScopeSelector::new(
16979 Principal::Reserved {
16980 module_id: BROCA.to_string(),
16981 },
16982 "b".to_string(),
16983 Some(1),
16984 );
16985 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
16986 assert!(!route.stamp().unwrap().owner_authorized);
16987 }
16988
16989 #[tokio::test]
16990 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
16991 let mut rig = rig().await;
16992 let mut record = head("s", 1);
16993 record.carriers = vec![carrier(AFT, None)];
16994 let flow_id = "Flow:run-7/step_2!~";
16995 record.attributes.flow_id = Some(flow_id.to_string());
16996 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
16997 .await
16998 .unwrap();
16999 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17000 panic!("not a sync reply");
17001 };
17002 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
17003 let route = rig
17004 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17005 .await;
17006 assert!(rig.live(&route), "the stamped bind committed");
17007 let stamp = route.stamp().expect("a flow scope carries a stamp");
17008 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
17009 assert_eq!(stamp.attributes.agent_id, None);
17010 assert!(!stamp.attributes.delegates);
17011 assert!(stamp.owner_authorized);
17012 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17013 assert_eq!(unscoped.stamp(), None);
17014 }
17015
17016 #[tokio::test]
17017 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
17018 let mut rig = rig_with_flow_support(false).await;
17019 let mut record = session(1);
17020 record.attributes.flow_id = Some("flow:7".to_string());
17021 rig.sync(vec![record]).await;
17022 for opener in [OWNER, AFT] {
17023 let body = rig
17024 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17025 .await;
17026 assert_eq!(body["code"], "target_flow_unsupported");
17027 let message = body["message"].as_str().unwrap();
17028 for required in [PLEXUS, "flow-scopes/v1"] {
17029 assert!(message.contains(required), "{message}");
17030 }
17031 }
17032 }
17033
17034 #[tokio::test]
17035 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
17036 let mut rig = rig_with_flow_support(true).await;
17037 let mut record = session(1);
17038 record.attributes.flow_id = Some("flow:7".to_string());
17039 rig.sync(vec![record]).await;
17040 for opener in [OWNER, AFT] {
17041 let route = rig
17042 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17043 .await;
17044 assert!(rig.live(&route));
17045 assert_eq!(
17046 route.stamp().unwrap().attributes.flow_id.as_deref(),
17047 Some("flow:7")
17048 );
17049 }
17050 }
17051
17052 #[tokio::test]
17053 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
17054 use std::future::Future;
17055
17056 let mut rig = rig_with_flow_support(true).await;
17057 let mut record = session(1);
17058 record.attributes.flow_id = Some("flow:7".to_string());
17059 rig.sync(vec![record]).await;
17060 let (client, mut client_rx, frame) =
17061 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
17062 for _ in 0..64 {
17065 client.egress.try_send(route_bind_ack(1)).unwrap();
17066 }
17067 let handler = rig.handler.clone();
17068 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17069 std::future::poll_fn(|cx| {
17070 assert!(open.as_mut().poll(cx).is_pending());
17071 std::task::Poll::Ready(())
17072 })
17073 .await;
17074 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17075
17076 let old_connection = rig.modules[PLEXUS].0.connection_id;
17077 rig.handler.cleanup_connection(old_connection).unwrap();
17078 let (replacement, mut replacement_rx) = wide_ctx(200);
17079 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
17080 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17081 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17082 let hello = Frame::build(
17083 FrameType::Hello,
17084 control_flags(),
17085 0,
17086 0,
17087 200,
17088 serde_json::to_vec(&body).unwrap(),
17089 )
17090 .unwrap();
17091 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
17092
17093 client_rx.try_recv().unwrap();
17094 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17095 .await
17096 .expect("the replacement is refused without waiting for a bind ack")
17097 .unwrap();
17098 assert_eq!(replies.len(), 1);
17099 let body = parse_error(&replies[0]);
17100 assert_eq!(body["code"], "target_flow_unsupported");
17101 for required in [PLEXUS, "flow-scopes/v1"] {
17102 assert!(body["message"].as_str().unwrap().contains(required));
17103 }
17104 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
17105 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17106 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17107 assert!(rig
17108 .handler
17109 .registry
17110 .get_module_by_connection(replacement.connection_id)
17111 .unwrap()
17112 .is_some());
17113 }
17114
17115 #[tokio::test]
17116 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
17117 use std::future::Future;
17118 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17119 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17120 const NEXT_HASH: &str =
17121 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17122 let mut rig = rig().await;
17123 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
17124 record.carriers.push(ScopeCarrier::new(Principal::Direct));
17125 rig.sync(vec![record.clone()]).await;
17126 let (client, mut rx, frame) = rig.open_frame(
17127 None,
17128 PLEXUS,
17129 Some(
17130 rig.selector("s", Some(1))
17131 .with_bind_proof(Some(PROOF.into())),
17132 ),
17133 );
17134 for _ in 0..64 {
17137 client.egress.try_send(route_bind_ack(1)).unwrap();
17138 }
17139 let handler = rig.handler.clone();
17140 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17141 std::future::poll_fn(|cx| {
17142 assert!(open.as_mut().poll(cx).is_pending());
17143 std::task::Poll::Ready(())
17144 })
17145 .await;
17146 record.bind_proof_sha256 = Some(NEXT_HASH.into());
17147 rig.sync(vec![record]).await;
17148 rx.try_recv().unwrap();
17149 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17150 .await
17151 .unwrap()
17152 .unwrap();
17153 assert_eq!(
17154 parse_error(&replies[0])["code"],
17155 error_codes::SCOPE_BIND_PROOF_MISMATCH
17156 );
17157 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
17158 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17159 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17160 }
17161
17162 #[tokio::test]
17163 async fn scope_bind_proof_reserved_openers_ignore_proof() {
17164 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17165 let mut rig = rig().await;
17166 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17167 .await;
17168 for opener in [OWNER, AFT] {
17169 for proof in [None, Some("malformed+proof=")] {
17170 let route = rig
17171 .bound(
17172 Some(opener),
17173 PLEXUS,
17174 Some(
17175 rig.selector("s", Some(1))
17176 .with_bind_proof(proof.map(str::to_string)),
17177 ),
17178 )
17179 .await;
17180 assert!(rig.live(&route));
17181 }
17182 }
17183 }
17184
17185 #[tokio::test]
17186 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
17187 use std::future::Future;
17188 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17189 const NEXT_HASH: &str =
17190 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17191 let mut rig = rig().await;
17192 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17193 .await;
17194 let (client, mut rx, frame) =
17195 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
17196 for _ in 0..64 {
17197 client.egress.try_send(route_bind_ack(1)).unwrap();
17198 }
17199 let handler = rig.handler.clone();
17200 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17201 std::future::poll_fn(|cx| {
17202 assert!(open.as_mut().poll(cx).is_pending());
17203 std::task::Poll::Ready(())
17204 })
17205 .await;
17206 rig.sync(vec![
17207 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
17208 ])
17209 .await;
17210 rx.try_recv().unwrap();
17211 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17212 let bind = std::future::poll_fn(|cx| {
17213 assert!(open.as_mut().poll(cx).is_pending());
17214 module_rx.poll_recv(cx)
17215 })
17216 .await
17217 .unwrap()
17218 .frame;
17219 handler
17220 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
17221 .await
17222 .unwrap();
17223 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17224 .await
17225 .unwrap()
17226 .unwrap();
17227 assert_eq!(replies.len(), 1);
17228 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
17231 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17232 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
17233 let route = rig
17234 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
17235 .await;
17236 assert!(rig.live(&route));
17237 }
17238
17239 #[tokio::test]
17240 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
17241 {
17242 let mut rig = rig_with_flow_support(false).await;
17243 rig.sync(vec![session(1)]).await;
17244 let route = rig
17245 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17246 .await;
17247 assert!(rig.live(&route));
17248 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
17249 let mut record = session(1);
17250 record.attributes.flow_id = Some("flow:7".to_string());
17251 rig.sync(vec![record]).await;
17252 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
17253 assert!(rig.live(&unscoped));
17254 assert_eq!(unscoped.stamp(), None);
17255 }
17256
17257 #[tokio::test]
17258 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
17259 let mut rig = rig().await;
17260 let mut record = session(1);
17261 record.attributes.flow_id = Some("flow:7".to_string());
17262 rig.sync(vec![record.clone()]).await;
17263 let before = rig.forwarding.published_scope_tag(OWNER, "s");
17264 let mut owner_route = rig
17265 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17266 .await;
17267 let mut carrier_route = rig
17268 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
17269 .await;
17270 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17271 rig.sync(vec![record.clone()]).await;
17272 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17273 assert!(rig.live(&owner_route) && owner_route.untouched());
17274 assert!(rig.live(&carrier_route) && carrier_route.untouched());
17275
17276 record.attributes.flow_id = Some("flow:8".to_string());
17277 rig.sync(vec![record]).await;
17278 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
17279 let before = before.unwrap();
17280 assert_eq!(after.scope_epoch, before.scope_epoch);
17281 assert!(after.version > before.version);
17282 for route in [&mut owner_route, &mut carrier_route] {
17283 assert!(!rig.live(route));
17284 assert_eq!(
17285 route.closed_reason(),
17286 RouteCloseReason::ScopeDelegationChanged
17287 );
17288 }
17289 assert!(rig.live(&unscoped) && unscoped.untouched());
17290 for target in [PLEXUS, OTHER] {
17293 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
17294 let goodbye = module_rx
17295 .try_recv()
17296 .expect("the provider sees the drain")
17297 .frame;
17298 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17299 assert!(module_rx.try_recv().is_err());
17300 }
17301 let rebound = rig
17302 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17303 .await;
17304 assert_eq!(
17305 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
17306 Some("flow:8")
17307 );
17308 }
17309
17310 #[tokio::test]
17315 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
17316 let mut rig = rig().await;
17317 rig.sync(vec![session(1)]).await;
17318 let mut cotenant = rig
17319 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17320 .await;
17321
17322 let mut changed = session(1);
17323 changed.child_owners.push(Principal::Reserved {
17324 module_id: MAGIC.to_string(),
17325 });
17326 let mut ended = None;
17327 for (code, next) in [
17328 (error_codes::SCOPE_CHANGED, vec![changed]),
17329 (error_codes::SCOPE_ENDED, Vec::new()),
17330 ] {
17331 let relayed = rig
17332 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17333 .await;
17334 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
17335 ended = Some(next.is_empty());
17336 rig.sync(next).await;
17337 rig.ack(&relayed).await;
17338 let replies = relayed.task.await.unwrap();
17339 assert_eq!(replies.len(), 1, "{replies:?}");
17340 assert_eq!(parse_error(&replies[0])["code"], code);
17341 assert_eq!(
17342 subc_protocol::error_codes::is_retryable_route_open(code),
17343 code == error_codes::SCOPE_CHANGED
17344 );
17345 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17346 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17348 let mut goodbyes = Vec::new();
17351 while let Ok(outbound) = plexus_rx.try_recv() {
17352 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
17353 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
17354 }
17355 assert!(
17356 goodbyes.contains(&(bind_channel, bind_epoch)),
17357 "{goodbyes:?}"
17358 );
17359 assert!(rig
17360 .handler
17361 .registry
17362 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
17363 .unwrap()
17364 .is_some());
17365 }
17366 assert_eq!(ended, Some(true));
17367 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
17370 }
17371
17372 #[tokio::test]
17375 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
17376 struct Case {
17377 name: &'static str,
17378 change: fn(&mut ScopeRecord),
17379 closed: [Option<RouteCloseReason>; 4],
17382 }
17383 use RouteCloseReason::*;
17384 let cases = [
17385 Case {
17386 name: "a carrier entry removed",
17387 change: |r| {
17388 r.carriers.retain(|c| {
17389 c.principal
17390 != Principal::Reserved {
17391 module_id: AFT.to_string(),
17392 }
17393 })
17394 },
17395 closed: [None, Some(ScopeCarrierRemoved), None, None],
17396 },
17397 Case {
17398 name: "a target removed from a carrier",
17399 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
17400 closed: [None, None, None, Some(ScopeCarrierRemoved)],
17401 },
17402 Case {
17403 name: "a bare carrier narrowed to targets",
17404 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
17405 closed: [None, Some(ScopeCarrierRemoved), None, None],
17406 },
17407 Case {
17408 name: "delegates turned off",
17409 change: |r| r.attributes.delegates = false,
17410 closed: [Some(ScopeDelegationChanged); 4],
17411 },
17412 Case {
17413 name: "agent_id changed",
17414 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
17415 closed: [Some(ScopeDelegationChanged); 4],
17416 },
17417 Case {
17418 name: "a carrier added, child owners changed, the record re-sent",
17419 change: |r| {
17420 r.carriers.push(carrier(MAGIC, None));
17421 r.child_owners.push(Principal::Reserved {
17422 module_id: MAGIC.to_string(),
17423 });
17424 },
17425 closed: [None; 4],
17426 },
17427 Case {
17428 name: "a target added",
17429 change: |r| {
17430 r.carriers[1]
17431 .targets
17432 .as_mut()
17433 .unwrap()
17434 .push("third".to_string())
17435 },
17436 closed: [None; 4],
17437 },
17438 Case {
17439 name: "delegates turned on",
17440 change: |r| r.attributes.delegates = true,
17441 closed: [None; 4],
17442 },
17443 ];
17444 for case in cases {
17445 let mut rig = rig().await;
17446 rig.sync(vec![session(1)]).await;
17447 let scope = || Some(rig_selector("s", Some(1)));
17448 let mut routes = [
17449 rig.bound(Some(OWNER), PLEXUS, scope()).await,
17450 rig.bound(Some(AFT), PLEXUS, scope()).await,
17451 rig.bound(Some(BROCA), PLEXUS, scope()).await,
17452 rig.bound(Some(BROCA), OTHER, scope()).await,
17453 ];
17454 let mut record = session(1);
17455 (case.change)(&mut record);
17456 rig.sync(vec![record]).await;
17457 for (index, expected) in case.closed.iter().enumerate() {
17458 let route = &mut routes[index];
17459 match expected {
17460 Some(reason) => {
17461 assert!(!rig.live(route), "{}: route {index} still live", case.name);
17462 assert_eq!(
17463 route.closed_reason(),
17464 *reason,
17465 "{}: route {index}",
17466 case.name
17467 );
17468 }
17469 None => {
17470 assert!(rig.live(route), "{}: route {index} closed", case.name);
17471 assert!(
17472 route.untouched(),
17473 "{}: route {index} was told something",
17474 case.name
17475 );
17476 }
17477 }
17478 }
17479 }
17480 }
17481
17482 #[tokio::test]
17483 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
17484 for next in [Vec::new(), vec![session(2)]] {
17486 let mut rig = rig().await;
17487 rig.sync(vec![session(1)]).await;
17488 let mut route = rig
17489 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17490 .await;
17491 rig.sync(next).await;
17492 assert!(!rig.live(&route));
17493 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
17494 }
17495
17496 let mut rig = rig().await;
17499 let mut child = session(1);
17500 child.scope_ref = "child".to_string();
17501 child.kind = ScopeKind::Worker;
17502 child.parent = Some(ScopeParent::new(
17503 Principal::Reserved {
17504 module_id: OWNER.to_string(),
17505 },
17506 "s".to_string(),
17507 1,
17508 ));
17509 rig.sync(vec![session(1), child.clone()]).await;
17510 let mut child_route = rig
17511 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
17512 .await;
17513 assert_eq!(
17514 child_route.stamp().unwrap().parent_state,
17515 Some(ParentState::Linked)
17516 );
17517 rig.sync(vec![child]).await;
17518 assert!(!rig.live(&child_route));
17519 assert_eq!(
17520 child_route.closed_reason(),
17521 RouteCloseReason::ScopeParentEnded
17522 );
17523 }
17524
17525 #[tokio::test]
17526 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
17527 ) {
17528 let mut rig = rig().await;
17529 rig.sync(vec![session(1)]).await;
17530 let mut route = rig
17531 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17532 .await;
17533 let before = rig.forwarding.published_scope_tag(OWNER, "s");
17534 rig.sync(vec![session(1)]).await;
17535 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17536 assert!(rig.live(&route) && route.untouched());
17537
17538 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
17542 .forwarding
17543 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17544 .unwrap()
17545 else {
17546 panic!("the route is bound");
17547 };
17548 binding.flow.acquire_tagged(9, false).await.unwrap();
17549 let mut widened = session(1);
17550 widened.carriers.push(carrier(MAGIC, None));
17551 rig.sync(vec![widened]).await;
17552 assert!(rig.live(&route) && route.untouched());
17553 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17554 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
17555 assert_eq!(binding.flow.in_flight(), 1);
17558 binding
17559 .flow
17560 .acquire_tagged(10, false)
17561 .await
17562 .expect("the flow is still open");
17563 }
17564
17565 #[tokio::test]
17568 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
17569 let mut rig = rig().await;
17570 rig.sync(vec![session(1)]).await;
17571 let mut on_incumbent = rig
17572 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17573 .await;
17574
17575 let (candidate, _candidate_rx) = wide_ctx(9);
17578 let registration = rig
17579 .handler
17580 .registry
17581 .register_candidate_with_control_ops(
17582 manifest(PLEXUS, PROTOCOL_VERSION),
17583 PROTOCOL_VERSION,
17584 candidate.connection_id,
17585 module_baseline_control_ops(),
17586 )
17587 .unwrap();
17588 rig.forwarding
17589 .register_candidate_module_connection(
17590 candidate.connection_id,
17591 PLEXUS.to_string(),
17592 PROTOCOL_VERSION,
17593 manifest_concurrency(®istration.manifest),
17594 candidate.egress.clone(),
17595 )
17596 .unwrap();
17597 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
17598 rig.handler
17599 .registry
17600 .promote_candidate(PLEXUS)
17601 .unwrap()
17602 .unwrap();
17603 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
17604
17605 rig.sync(Vec::new()).await;
17606 assert!(!rig.live(&on_incumbent));
17607 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
17608 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17609 let goodbye = incumbent_rx
17610 .try_recv()
17611 .expect("the superseded endpoint is told")
17612 .frame;
17613 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17614 }
17615 }
17616}
17617
17618#[cfg(test)]
17619mod concurrency_default_exposure_tests {
17620 use super::*;
17621
17622 fn hello_body(role_json: &str) -> Vec<u8> {
17623 format!(
17624 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
17625 )
17626 .into_bytes()
17627 }
17628
17629 fn manifest_from(body: &[u8]) -> ModuleManifest {
17630 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
17631 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
17632 .expect("manifest parses")
17633 }
17634
17635 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
17636
17637 #[test]
17638 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
17639 let body = hello_body(&format!(
17640 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
17641 ));
17642 let manifest = manifest_from(&body);
17643 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17646 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
17647 }
17648
17649 #[test]
17650 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
17651 let body = hello_body(&format!(
17652 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
17653 ));
17654 let manifest = manifest_from(&body);
17655 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17656 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17657 }
17658
17659 #[test]
17660 fn non_management_roles_are_never_reported() {
17661 let body = hello_body(
17662 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
17663 );
17664 let manifest = manifest_from(&body);
17665 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17666 }
17667}