1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_event_declarations,
25 validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
26 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
27 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
28 },
29 scope::{
30 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
31 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
32 SCOPE_SYNC_OP,
33 },
34 session::{
35 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
36 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
37 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
38 ROLE_VERSIONS_FIELD,
39 },
40 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
41 Priority, RouteTarget, PROTOCOL_VERSION,
42};
43use tokio::time::{timeout_at, Instant};
44use tracing::{debug, info, warn};
45
46use crate::{
47 caller_identity::CallerIdentity,
48 capability_requirements::{
49 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
50 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
51 RuntimeModule,
52 },
53 child_roster::DaemonShutdownFlag,
54 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
55 forwarding::{
56 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
57 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
58 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
59 },
60 observability::{
61 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
62 },
63 privileged_prompt::{prompt_text, PrivilegedOperation},
64 provenance::{
65 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
66 },
67 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
68 router::{RouteCtx, RouterError},
69 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
70 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
71 stderr_tail::{CaptureState, TailEntry},
72 supervise::{
73 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
74 SupervisorHandle, SwapHelloAdmission,
75 },
76 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
77};
78
79pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
85
86const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
87const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
88const CAP_PING_PONG: &str = "ping_pong_v1";
89const CAP_SESSION_ATTACH: &str = "session_attach_v1";
90const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
91
92const SUBC_CONTROL_OPS: &[&str] = &[
93 ops::ACTIVITY_RECORD,
94 ops::ACTIVITY_QUERY,
95 ops::ACTIVITY_SUBSCRIBE,
96 ops::SERVER_DESCRIBE,
97 ops::CATALOG_LIST,
98 ops::ROUTE_OPEN,
99 ops::ROUTE_POLL,
100 ops::ROUTE_CLOSING,
101 ops::ROUTE_CLOSED,
102 ops::SUPERVISOR_LIST,
103 ops::SUPERVISOR_RESTART,
104 ops::SUPERVISOR_SWAP,
105 ops::SUPERVISOR_RELOAD,
106 ops::SUPERVISOR_RESCAN,
107 ops::SUPERVISOR_RELEASE_RESERVED,
108 ops::SUPERVISOR_SET_ENABLED,
109 ops::SUPERVISOR_HEALTH_PROBE,
110 ops::SUPERVISOR_HEALTH,
111 ops::SUPERVISOR_STDERR_TAIL,
112 ops::SUPERVISOR_TERMINALS,
113 ops::SUPERVISOR_ROUTES,
114 ops::SUPERVISOR_PROVENANCE,
115 ops::SUPERVISOR_SPAWN_SNAPSHOT,
116 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
117];
118
119const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
126
127mod rescan_checked;
128use rescan_checked::{RescanMode, RescanPlan};
129
130const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
131 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
132 "supervisor.live_roots",
133 SCOPE_SYNC_OP,
134 SCOPE_APPLY_OP,
135 SCOPE_DESCRIBE_OP,
136 "operator.confirm",
137];
138
139const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
143
144const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
145
146pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
154
155pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
170
171pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
189
190const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
191const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
192
193fn reload_verdict(
194 configured: &Path,
195 spawned_from: Option<&Path>,
196 image: subc_control::RunningImageAgreement,
197) -> PendingReloadVerdict {
198 let path = match spawned_from {
199 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
200 ReloadPathAgreement::Match
201 }
202 Some(spawned_from) => ReloadPathAgreement::Mismatch {
203 configured: configured.to_path_buf(),
204 spawned_from: spawned_from.to_path_buf(),
205 },
206 None => ReloadPathAgreement::Unavailable {
207 reason: if matches!(
208 image,
209 subc_control::RunningImageAgreement::Unavailable {
210 reason: subc_control::RunningImageUnavailableReason::NotRunning
211 }
212 ) {
213 ReloadPathUnavailableReason::NotRunning
214 } else {
215 ReloadPathUnavailableReason::SpawnedPathUnavailable
216 },
217 },
218 };
219 PendingReloadVerdict {
220 path,
221 image,
222 spec: None,
223 }
224}
225
226fn launch_spec_agreement(
227 configured: &crate::supervise::ModuleSpec,
228 running: bool,
229 spawned: Option<&crate::supervise::LaunchSpecDigest>,
230) -> subc_control::LaunchSpecAgreement {
231 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
232 if !running {
233 return Agreement::Unavailable {
234 reason: Reason::NotRunning,
235 };
236 }
237 match spawned {
238 None => Agreement::Unavailable {
239 reason: Reason::SpawnedSpecUnavailable,
240 },
241 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
242 Agreement::Match
243 }
244 Some(_) => Agreement::Mismatch,
245 }
246}
247
248fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
249 if configured == spawned_from {
250 return true;
251 }
252 #[cfg(windows)]
253 {
254 match (
260 std::fs::canonicalize(configured),
261 std::fs::canonicalize(spawned_from),
262 ) {
263 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
264 _ => false,
265 }
266 }
267 #[cfg(not(windows))]
268 {
269 false
270 }
271}
272
273#[derive(Clone)]
274struct DaemonProvenanceFacts {
275 build: DaemonBuildProvenance,
276 pid: Option<u32>,
277 started_at_ms: Option<u64>,
278 start_clock: Option<crate::clock::StartClock>,
279 executable_path: Option<PathBuf>,
280 executable_identity: Option<SpawnedFileIdentity>,
281 process_start_time: Option<u64>,
282 probe: ExecutableIdentityProbe,
283}
284
285impl Default for DaemonProvenanceFacts {
286 fn default() -> Self {
287 Self {
288 build: DaemonBuildProvenance {
289 build_git_sha: None,
290 build_lock_digest: None,
291 },
292 pid: None,
293 started_at_ms: None,
294 start_clock: None,
295 executable_path: None,
296 executable_identity: None,
297 process_start_time: None,
298 probe: ExecutableIdentityProbe::default(),
299 }
300 }
301}
302
303#[derive(Debug, Clone)]
304struct SupervisorRescanContext {
305 supervisor: Supervisor,
306 config_path: PathBuf,
307 configured_port: Option<u16>,
308 storage_config: Option<crate::daemon_config::StorageConfig>,
309 admission_facts_carrier_module_id: Option<String>,
310 admission_facts_targets: Option<Vec<String>>,
311 scope_authority_owners: Vec<String>,
312 privileged_operator_prompt: PrivilegedOperatorPrompt,
313 generation: Arc<std::sync::atomic::AtomicU64>,
319}
320
321const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
342 "reloading",
343 "supervisor_not_live",
344 "registration_not_active",
345 "no_forwarding_connection",
346 "relay_send_failed",
347];
348
349#[derive(Clone)]
351pub struct ControlHandler {
352 registry: Arc<Registry>,
353 activity: Option<crate::activity::ActivityLedger>,
354 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
355 forwarding: Arc<ForwardingTable>,
356 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
357 supervisor: SupervisorHandle,
358 subc_capabilities: Arc<[String]>,
359 route_bind_relay_timeout: Duration,
363 route_bind_relay_timeouts: BTreeMap<String, Duration>,
367 route_bind_breakers: RouteBindBreakers,
370 route_bind_concurrency: RouteBindConcurrency,
373 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
378 route_bind_breaker_threshold: u32,
380 route_bind_breaker_cooldown: Duration,
382 health_probe_timeout: Duration,
383 storage_config: Option<crate::daemon_config::StorageConfig>,
386 machine_id: Option<crate::machine_id::MachineId>,
390 admission_facts_carrier_module_id: Option<String>,
391 admission_facts_targets: Option<Vec<String>>,
392 scopes: Arc<RwLock<ScopeTable>>,
396 scope_deadline_added: Arc<tokio::sync::Notify>,
400 scope_authority_owners: Vec<String>,
403 privileged_operator_prompt: PrivilegedOperatorPrompt,
405 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
408 #[cfg(test)]
409 caller_lookup: Option<TestCallerLookup>,
410 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
413 rescan: Option<SupervisorRescanContext>,
414 connected_clients: ConnectedClients,
415 counters: DaemonCounters,
416 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
417 daemon_shutdown: DaemonShutdownFlag,
422 daemon_provenance: DaemonProvenanceFacts,
423 #[cfg(test)]
424 control_dispatch_delay: Option<Duration>,
425 #[cfg(test)]
426 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
427 #[cfg(test)]
431 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
432 #[cfg(test)]
438 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
439}
440
441impl fmt::Debug for ControlHandler {
442 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
443 f.debug_struct("ControlHandler")
444 .field("registry", &self.registry)
445 .field("forwarding", &self.forwarding)
446 .field("process_liveness", &self.process_liveness.is_some())
447 .field("supervisor", &self.supervisor)
448 .field("subc_capabilities", &self.subc_capabilities)
449 .finish()
450 }
451}
452
453struct RouteOpenRequest {
454 target: RouteTarget,
455 identity: BindIdentity,
456 consumer_identity: Option<ConsumerIdentity>,
457 consumer_capabilities: Option<Vec<String>>,
458 role_versions: Option<BTreeMap<String, String>>,
459 admission_facts: Option<serde_json::Value>,
460 scope: Option<ScopeSelector>,
461}
462
463#[derive(Clone, Copy)]
464enum RouteOpenPhase {
465 Admission,
466 ProjectRoot,
467 EgressReservation,
468 BindSend,
469 BindAckWait,
470}
471
472struct RouteOpenTimings {
476 module_id: String,
477 principal: String,
478 admission: Duration,
479 project_root: Duration,
480 egress_reservation: Duration,
481 bind_send: Duration,
482 bind_ack_wait: Duration,
483 phase: Option<RouteOpenPhase>,
484 phase_started_at: Instant,
485}
486
487impl RouteOpenTimings {
488 fn new() -> Self {
489 Self {
490 module_id: String::new(),
491 principal: "unresolved".to_string(),
493 admission: Duration::ZERO,
494 project_root: Duration::ZERO,
495 egress_reservation: Duration::ZERO,
496 bind_send: Duration::ZERO,
497 bind_ack_wait: Duration::ZERO,
498 phase: Some(RouteOpenPhase::Admission),
499 phase_started_at: Instant::now(),
500 }
501 }
502
503 fn finish(&mut self) {
504 let elapsed = self.phase_started_at.elapsed();
505 match self.phase.take() {
506 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
507 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
508 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
509 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
510 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
511 None => {}
512 }
513 }
514
515 fn enter(&mut self, phase: RouteOpenPhase) {
516 self.finish();
517 self.phase = Some(phase);
518 self.phase_started_at = Instant::now();
519 }
520}
521
522struct RouteBindReservationGuard {
523 forwarding: Arc<ForwardingTable>,
524 endpoint: ModuleEndpointId,
525 relay_corr: u64,
526 armed: bool,
527}
528
529struct ModuleControlRpcGuard {
530 forwarding: Arc<ForwardingTable>,
531 endpoint: ModuleEndpointId,
532 corr: u64,
533 armed: bool,
534}
535
536impl ModuleControlRpcGuard {
537 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
538 Self {
539 forwarding,
540 endpoint,
541 corr,
542 armed: true,
543 }
544 }
545
546 fn disarm(&mut self) {
547 self.armed = false;
548 }
549}
550
551impl Drop for ModuleControlRpcGuard {
552 fn drop(&mut self) {
553 if self.armed {
554 let _ = self
555 .forwarding
556 .cancel_module_control_rpc(self.endpoint, self.corr);
557 }
558 }
559}
560
561impl RouteBindReservationGuard {
562 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
563 Self {
564 forwarding,
565 endpoint,
566 relay_corr,
567 armed: true,
568 }
569 }
570
571 fn release_and_disarm(&mut self) {
572 if !self.armed {
573 return;
574 }
575 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
576 self.endpoint,
577 self.relay_corr,
578 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
579 ) {
580 send_goodbye_target_best_effort(
581 &self.forwarding.counters(),
582 &target,
583 "canceled route.bind",
584 );
585 }
586 self.armed = false;
587 }
588
589 fn disarm(&mut self) {
590 self.armed = false;
591 }
592}
593
594impl Drop for RouteBindReservationGuard {
595 fn drop(&mut self) {
596 self.release_and_disarm();
597 }
598}
599
600#[derive(Debug, Clone, Default)]
630pub(crate) struct RouteBindBreakers {
631 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
632}
633
634#[derive(Debug, Clone, Default)]
635pub(crate) struct RouteBindConcurrency {
636 modules: Arc<Mutex<HashMap<String, usize>>>,
637}
638
639struct RouteBindConcurrencyGuard {
640 concurrency: RouteBindConcurrency,
641 module_id: String,
642}
643
644impl RouteBindConcurrency {
645 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
648 let mut modules = self
649 .modules
650 .lock()
651 .expect("route.bind concurrency mutex poisoned");
652 let in_flight = modules.entry(module_id.to_string()).or_default();
653 if *in_flight >= limit {
654 return Err(*in_flight);
655 }
656 *in_flight += 1;
657 Ok(RouteBindConcurrencyGuard {
658 concurrency: self.clone(),
659 module_id: module_id.to_string(),
660 })
661 }
662}
663
664impl Drop for RouteBindConcurrencyGuard {
665 fn drop(&mut self) {
666 let mut modules = self
667 .concurrency
668 .modules
669 .lock()
670 .expect("route.bind concurrency mutex poisoned");
671 let remove = {
672 let in_flight = modules
673 .get_mut(&self.module_id)
674 .expect("admitted route.bind has a concurrency entry");
675 *in_flight -= 1;
676 *in_flight == 0
677 };
678 if remove {
679 modules.remove(&self.module_id);
680 }
681 }
682}
683
684#[derive(Debug, Default)]
685struct ModuleBreakerState {
686 consecutive_timeouts: u32,
688 cooldown_until: Option<Instant>,
691 probe_in_flight: Option<Arc<()>>,
696}
697
698enum RouteBindAdmission<'a> {
700 Admitted {
701 guard: RouteBindBreakerGuard<'a>,
702 probe: bool,
705 },
706 Refused {
707 consecutive_timeouts: u32,
708 retry_in: Duration,
712 probe_in_flight: bool,
713 },
714}
715
716struct RouteBindBreakerGuard<'a> {
723 breakers: RouteBindBreakers,
724 module_id: &'a str,
725 probe_token: Option<Arc<()>>,
726 settled: bool,
727}
728
729impl RouteBindBreakerGuard<'_> {
730 fn record_accepted(&mut self) -> bool {
734 self.settled = true;
735 self.breakers.record_accepted(self.module_id)
736 }
737
738 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
741 self.settled = true;
742 self.breakers.record_timeout(
743 self.module_id,
744 self.probe_token.as_ref(),
745 threshold,
746 cooldown,
747 )
748 }
749
750 fn record_inconclusive(&mut self) {
759 self.settled = true;
760 self.breakers
761 .record_inconclusive(self.module_id, self.probe_token.as_ref());
762 }
763}
764
765impl Drop for RouteBindBreakerGuard<'_> {
766 fn drop(&mut self) {
767 if !self.settled {
768 self.breakers
769 .record_inconclusive(self.module_id, self.probe_token.as_ref());
770 }
771 }
772}
773
774struct BreakerOpened {
778 consecutive_timeouts: u32,
779 reopened_after_probe: bool,
782}
783
784impl RouteBindBreakers {
785 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
786 self.modules
787 .lock()
788 .expect("route.bind breaker mutex poisoned")
789 }
790
791 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
794 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
795 probe: probe_token.is_some(),
796 guard: RouteBindBreakerGuard {
797 breakers: self.clone(),
798 module_id,
799 probe_token,
800 settled: false,
801 },
802 };
803
804 let mut modules = self.lock();
805 let Some(state) = modules.get_mut(module_id) else {
806 return admitted(None);
807 };
808 let Some(cooldown_until) = state.cooldown_until else {
809 return admitted(None);
810 };
811 if state.probe_in_flight.is_some() {
812 return RouteBindAdmission::Refused {
813 consecutive_timeouts: state.consecutive_timeouts,
814 retry_in: Duration::ZERO,
815 probe_in_flight: true,
816 };
817 }
818 let now = Instant::now();
819 if now < cooldown_until {
820 return RouteBindAdmission::Refused {
821 consecutive_timeouts: state.consecutive_timeouts,
822 retry_in: cooldown_until - now,
823 probe_in_flight: false,
824 };
825 }
826 let token = Arc::new(());
827 state.probe_in_flight = Some(Arc::clone(&token));
828 admitted(Some(token))
829 }
830
831 fn record_accepted(&self, module_id: &str) -> bool {
832 self.lock()
833 .remove(module_id)
834 .is_some_and(|state| state.cooldown_until.is_some())
835 }
836
837 fn record_timeout(
838 &self,
839 module_id: &str,
840 probe_token: Option<&Arc<()>>,
841 threshold: u32,
842 cooldown: Duration,
843 ) -> Option<BreakerOpened> {
844 let mut modules = self.lock();
845 let state = modules.entry(module_id.to_string()).or_default();
846 let was_open = state.cooldown_until.is_some();
847 let was_probe = Self::owns_probe(state, probe_token);
848 if was_probe {
849 state.probe_in_flight = None;
850 }
851 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
852 if state.consecutive_timeouts < threshold {
853 return None;
854 }
855 state.cooldown_until = Some(Instant::now() + cooldown);
856 Some(BreakerOpened {
857 consecutive_timeouts: state.consecutive_timeouts,
858 reopened_after_probe: was_open && was_probe,
859 })
860 }
861
862 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
863 state
867 .probe_in_flight
868 .as_ref()
869 .zip(token)
870 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
871 }
872
873 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
874 if let Some(state) = self.lock().get_mut(module_id) {
875 if Self::owns_probe(state, probe_token) {
876 state.probe_in_flight = None;
877 }
878 }
879 }
880
881 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
896 self.lock()
897 .remove(module_id)
898 .map(|state| state.consecutive_timeouts)
899 .filter(|discarded| *discarded > 0)
900 }
901
902 fn open_snapshot(&self) -> Option<serde_json::Value> {
909 let now = Instant::now();
910 let modules = self.lock();
911 let open = modules
912 .iter()
913 .filter_map(|(module_id, state)| {
914 let cooldown_until = state.cooldown_until?;
915 Some((
916 module_id.clone(),
917 serde_json::json!({
918 "consecutive_timeouts": state.consecutive_timeouts,
919 "cooldown_remaining_ms":
920 cooldown_until.saturating_duration_since(now).as_millis() as u64,
921 "probe_in_flight": state.probe_in_flight.is_some(),
922 }),
923 ))
924 })
925 .collect::<serde_json::Map<String, serde_json::Value>>();
926 (!open.is_empty()).then_some(serde_json::Value::Object(open))
927 }
928}
929
930impl ControlHandler {
931 pub fn new(registry: Arc<Registry>) -> Self {
932 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
933 }
934
935 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
936 let counters = forwarding.counters();
937 let route_bind_breakers = forwarding.route_bind_breakers();
941 let route_bind_concurrency = forwarding.route_bind_concurrency();
942 let route_outages = forwarding.route_outages();
943 Self {
944 registry,
945 activity: None,
946 activity_streams: Arc::new(Mutex::new(HashMap::new())),
947 forwarding,
948 process_liveness: None,
949 supervisor: SupervisorHandle::new(),
950 subc_capabilities: Arc::from([
951 CAP_MANIFEST_REGISTRATION.to_string(),
952 CAP_CHANNEL_LIFECYCLE.to_string(),
953 CAP_PING_PONG.to_string(),
954 CAP_SESSION_ATTACH.to_string(),
955 CAP_ADMISSION_FACTS_RELAY.to_string(),
956 CAP_SCOPES_V1.to_string(),
957 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
958 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
959 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
960 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
961 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
962 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
963 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
964 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
965 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
966 ]),
967 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
968 route_bind_relay_timeouts: BTreeMap::new(),
969 route_bind_breakers,
970 route_bind_concurrency,
971 route_outages,
972 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
973 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
974 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
975 storage_config: None,
976 machine_id: None,
977 admission_facts_carrier_module_id: None,
978 admission_facts_targets: None,
979 scopes: Arc::new(RwLock::new(ScopeTable::new(
980 crate::daemon_config::default_scope_authority_owners(),
981 ))),
982 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
983 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
984 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
985 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
986 #[cfg(test)]
987 caller_lookup: None,
988 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
989 rescan: None,
990 connected_clients: ConnectedClients::new(),
991 counters,
992 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
993 daemon_shutdown: DaemonShutdownFlag::default(),
994 daemon_provenance: DaemonProvenanceFacts::default(),
995 #[cfg(test)]
996 control_dispatch_delay: None,
997 #[cfg(test)]
998 provenance_probe_override: None,
999 #[cfg(test)]
1000 route_bind_relay_sent_hold: None,
1001 #[cfg(test)]
1002 hello_registry_to_forwarding_hold: None,
1003 }
1004 }
1005
1006 pub fn with_storage_config(
1009 mut self,
1010 storage_config: Option<crate::daemon_config::StorageConfig>,
1011 ) -> Self {
1012 self.storage_config = storage_config;
1013 self
1014 }
1015
1016 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1019 self.activity = Some(crate::activity::ActivityLedger::open(
1020 run_dir.join("activity.jsonl"),
1021 incarnation,
1022 ));
1023 self.supervisor.set_activity(self.activity.clone());
1024 self
1025 }
1026
1027 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1030 self.machine_id = machine_id;
1031 self
1032 }
1033
1034 pub fn with_admission_facts_config(
1038 mut self,
1039 carrier_module_id: Option<String>,
1040 targets: Option<Vec<String>>,
1041 ) -> Self {
1042 self.admission_facts_carrier_module_id = carrier_module_id;
1043 self.admission_facts_targets = targets;
1044 self
1045 }
1046
1047 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1051 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1052 self.scope_authority_owners = owners;
1053 self
1054 }
1055
1056 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1058 self.privileged_operator_prompt = mode;
1059 self
1060 }
1061
1062 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1065 self.route_bind_relay_timeout = timeout;
1066 self
1067 }
1068
1069 pub fn with_route_bind_relay_timeouts(
1075 mut self,
1076 timeouts: impl IntoIterator<Item = (String, Duration)>,
1077 ) -> Self {
1078 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1079 self
1080 }
1081
1082 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1088 self.route_bind_relay_timeouts
1089 .get(module_id)
1090 .copied()
1091 .unwrap_or(self.route_bind_relay_timeout)
1092 }
1093
1094 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1102 self.route_bind_breaker_threshold = threshold.max(1);
1103 self.route_bind_breaker_cooldown = cooldown;
1104 self
1105 }
1106
1107 #[cfg(test)]
1108 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1109 self.health_probe_timeout = timeout;
1110 self
1111 }
1112
1113 #[cfg(test)]
1114 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1115 self.control_dispatch_delay = Some(delay);
1116 self
1117 }
1118
1119 #[cfg(test)]
1123 pub(crate) fn with_route_bind_relay_sent_hold(
1124 mut self,
1125 hold: Arc<tokio::sync::Barrier>,
1126 ) -> Self {
1127 self.route_bind_relay_sent_hold = Some(hold);
1128 self
1129 }
1130
1131 #[cfg(test)]
1136 pub(crate) fn with_hello_registry_to_forwarding_hold(
1137 mut self,
1138 hold: Arc<std::sync::Barrier>,
1139 ) -> Self {
1140 self.hello_registry_to_forwarding_hold = Some(hold);
1141 self
1142 }
1143
1144 #[cfg(test)]
1145 fn hold_between_registry_and_forwarding(&self) {
1146 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1147 hold.wait();
1148 hold.wait();
1149 }
1150 }
1151
1152 pub fn with_process_liveness(
1153 mut self,
1154 process_liveness: Arc<dyn ModuleProcessLiveness>,
1155 ) -> Self {
1156 self.process_liveness = Some(process_liveness);
1157 self
1158 }
1159
1160 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1161 self.supervisor = supervisor;
1162 self.supervisor.set_activity(self.activity.clone());
1163 self
1164 }
1165
1166 pub fn with_daemon_provenance(
1167 mut self,
1168 pid: u32,
1169 started_at_ms: u64,
1170 executable_path: Option<PathBuf>,
1171 build_git_sha: Option<String>,
1172 build_lock_digest: Option<String>,
1173 ) -> Self {
1174 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1175 let process_start_time = process_start_time(pid);
1176 self.daemon_provenance = DaemonProvenanceFacts {
1177 build: DaemonBuildProvenance {
1178 build_git_sha,
1179 build_lock_digest,
1180 },
1181 pid: Some(pid),
1182 started_at_ms: Some(started_at_ms),
1183 start_clock: None,
1184 executable_path,
1185 executable_identity,
1186 process_start_time,
1187 probe: ExecutableIdentityProbe::default(),
1188 };
1189 self
1190 }
1191
1192 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1193 self.daemon_provenance.start_clock = Some(clock);
1194 self
1195 }
1196
1197 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1199 self.scopes
1200 .write()
1201 .unwrap_or_else(|poisoned| poisoned.into_inner())
1202 .set_wall_clock(Arc::new(clock));
1203 self
1204 }
1205
1206 #[cfg(test)]
1207 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1208 self.provenance_probe_override = Some(result);
1209 self
1210 }
1211
1212 pub fn with_capability_config(
1216 self,
1217 modules: impl IntoIterator<Item = (String, bool)>,
1218 reserved_capabilities: BTreeMap<String, String>,
1219 ) -> Self {
1220 self.capability_evaluator
1221 .configure(modules, reserved_capabilities);
1222 self
1223 }
1224
1225 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1229 self.daemon_shutdown = flag;
1230 self
1231 }
1232
1233 pub fn with_supervisor_rescan(
1234 mut self,
1235 supervisor: Supervisor,
1236 config_path: impl Into<PathBuf>,
1237 configured_port: Option<u16>,
1238 ) -> Self {
1239 self.rescan = Some(SupervisorRescanContext {
1240 supervisor,
1241 config_path: config_path.into(),
1242 configured_port,
1243 storage_config: self.storage_config.clone(),
1244 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1245 admission_facts_targets: self.admission_facts_targets.clone(),
1246 scope_authority_owners: self.scope_authority_owners.clone(),
1247 privileged_operator_prompt: self.privileged_operator_prompt,
1248 generation: Arc::default(),
1249 });
1250 self
1251 }
1252
1253 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1254 self.connected_clients = connected_clients;
1255 self
1256 }
1257
1258 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1259 Arc::clone(&self.forwarding)
1260 }
1261
1262 pub(crate) fn counters(&self) -> DaemonCounters {
1263 self.counters.clone()
1264 }
1265
1266 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1269 tokio::spawn(async move {
1270 loop {
1271 self.capability_evaluator
1272 .wait_for_change_or_deadline()
1273 .await;
1274 self.refresh_capability_requirements();
1275 }
1276 });
1277 }
1278
1279 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1285 tokio::spawn(async move {
1286 loop {
1287 let has_deadlines = self
1288 .scopes
1289 .read()
1290 .unwrap_or_else(|poisoned| poisoned.into_inner())
1291 .has_deadlines();
1292 if !has_deadlines {
1293 self.scope_deadline_added.notified().await;
1294 continue;
1295 }
1296 tokio::time::sleep(Duration::from_secs(1)).await;
1297 if let Err(error) = self.sweep_expired_scopes() {
1298 warn!(%error, "scope expiry sweep failed");
1299 }
1300 }
1301 });
1302 }
1303
1304 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1305 let mut table = self
1306 .scopes
1307 .write()
1308 .unwrap_or_else(|poisoned| poisoned.into_inner());
1309 let swept = table.sweep_expired();
1310 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1311 drop(table);
1312 self.close_scope_drained_routes(drained);
1313 Ok(())
1314 }
1315
1316 fn publish_scope_changes(
1317 &self,
1318 changes: &[crate::scopes::ScopeTagChange],
1319 expired: &[crate::scopes::ScopeExpired],
1320 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1321 if changes.is_empty() {
1322 return Ok(Vec::new());
1323 }
1324 let drained = self
1325 .forwarding
1326 .publish_scope_changes(changes)
1327 .map_err(RouterError::Forwarding)?;
1328 let mut counts = HashMap::new();
1329 for route in &drained {
1330 *counts
1331 .entry((
1332 route.scope.owner.as_str(),
1333 route.scope.scope_ref.as_str(),
1334 route.scope.tag.scope_epoch,
1335 ))
1336 .or_insert(0usize) += 1;
1337 }
1338 for scope in expired {
1339 let routes_closed = counts
1340 .get(&(
1341 scope.owner.as_str(),
1342 scope.scope_ref.as_str(),
1343 scope.scope_epoch,
1344 ))
1345 .copied()
1346 .unwrap_or(0);
1347 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1348 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1349 routes_closed, "scope expired");
1350 }
1351 Ok(drained)
1352 }
1353
1354 fn runtime_capability_snapshot(
1355 &self,
1356 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1357 let runtime = self
1358 .supervisor
1359 .list()
1360 .into_iter()
1361 .map(|module| {
1362 let status = module.status().map_err(|err| {
1363 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1364 })?;
1365 Ok(RuntimeModule {
1366 module_id: status.module_id,
1367 state: status.state,
1368 enabled: status.enabled,
1369 })
1370 })
1371 .collect::<Result<Vec<_>, RouterError>>()?;
1372 let registrations = self.registry.list_registered_modules().map_err(|err| {
1378 RouterError::backend(
1379 0,
1380 0,
1381 format!("failed to list capability registrations: {err}"),
1382 )
1383 })?;
1384 let registrations = registrations
1385 .into_iter()
1386 .map(|registration| RegisteredModule {
1387 module_id: registration.manifest.module_id,
1388 module_version: registration.manifest.module_version,
1389 capabilities: registration.manifest.capabilities,
1390 })
1391 .collect();
1392 Ok((runtime, registrations))
1393 }
1394
1395 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1402 let cached_registration = RegisteredModule {
1403 module_id: registration.manifest.module_id.clone(),
1404 module_version: registration.manifest.module_version.clone(),
1405 capabilities: registration.manifest.capabilities.clone(),
1406 };
1407 if self.capability_evaluator.record_hello(&cached_registration) {
1408 warn!(
1409 module_id = %cached_registration.module_id,
1410 "capability claims drifted from the cached manifest"
1411 );
1412 }
1413 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1414 self.enforce_capability_denies();
1415 }
1416 self.refresh_capability_requirements();
1417 }
1418
1419 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1423 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1424 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1425 self.supervisor.set_swap_promotion_observer(observer);
1426 }
1427
1428 pub fn refresh_capability_requirements(&self) {
1429 match self.runtime_capability_snapshot() {
1430 Ok((runtime, registrations)) => {
1431 log_requirement_events(
1432 self.capability_evaluator
1433 .evaluate_now(&runtime, ®istrations),
1434 &self.daemon_shutdown,
1435 );
1436 }
1437 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1438 }
1439 }
1440
1441 fn enforce_capability_denies(&self) {
1445 let registrations = match self.registry.list_registered_modules() {
1446 Ok(snapshot) => snapshot,
1447 Err(err) => {
1448 warn!(error = %err, "failed to read registrations for capability deny census");
1449 return;
1450 }
1451 };
1452 let manifests = registrations
1453 .into_iter()
1454 .map(|registration| {
1455 (
1456 registration.manifest.module_id.clone(),
1457 registration.manifest,
1458 )
1459 })
1460 .collect::<BTreeMap<_, _>>();
1461 let census = match self.forwarding.route_census(None) {
1462 Ok(census) => census,
1463 Err(err) => {
1464 warn!(error = %err, "failed to read route census for capability deny enforcement");
1465 return;
1466 }
1467 };
1468
1469 for (target_module_id, routes) in census {
1470 let Some(target_manifest) = manifests.get(&target_module_id) else {
1471 continue;
1472 };
1473 let mut closed_routes = Vec::new();
1474 let mut module_goodbyes = Vec::new();
1475 for route in routes {
1476 let Principal::Reserved {
1477 module_id: opening_module_id,
1478 } = &route.principal
1479 else {
1480 continue;
1481 };
1482 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1483 continue;
1484 };
1485 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1486 continue;
1487 };
1488
1489 match self.forwarding.release_client_route(
1490 route.goodbye_target.connection_id,
1491 route.goodbye_target.channel,
1492 route.goodbye_target.epoch,
1493 ) {
1494 Ok(RouteRelease::Removed(module_goodbye)) => {
1495 warn!(
1496 opening_module_id,
1497 target_module_id,
1498 capability,
1499 "force-closing route because an attested capability deny edge now matches"
1500 );
1501 closed_routes.push(route);
1502 module_goodbyes.push(module_goodbye);
1503 }
1504 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1505 Err(err) => warn!(
1506 opening_module_id,
1507 target_module_id,
1508 capability,
1509 error = %err,
1510 "failed to force-close capability-denied route"
1511 ),
1512 }
1513 }
1514
1515 if closed_routes.is_empty() {
1516 continue;
1517 }
1518 send_route_control_pushes(
1519 &self.forwarding,
1520 closed_routes,
1521 ClientControlPush::RouteClosed {
1522 module_id: target_module_id,
1523 channels: Vec::new(),
1524 reason: RouteCloseReason::CapabilityDenied,
1525 drained: false,
1526 abandoned: 0,
1527 excluded_subscriptions: 0,
1528 terminal: Some(false),
1529 },
1530 );
1531 self.emit_route_goodbyes(module_goodbyes);
1532 }
1533 }
1534
1535 fn not_ready_reason(
1541 &self,
1542 registration: &crate::registry::ModuleRegistration,
1543 ) -> Option<NotReadyReason> {
1544 if !registration.ready {
1545 return Some(NotReadyReason {
1546 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1547 capability: None,
1548 });
1549 }
1550 self.first_unprovided_required_capability(registration)
1551 .map(|capability| NotReadyReason {
1552 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1553 capability: Some(capability),
1554 })
1555 }
1556
1557 fn first_unprovided_required_capability(
1575 &self,
1576 registration: &crate::registry::ModuleRegistration,
1577 ) -> Option<String> {
1578 let required = registration
1579 .manifest
1580 .capabilities
1581 .iter()
1582 .flat_map(|declarations| declarations.requires.iter())
1583 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1584 .map(|requirement| requirement.capability.as_str())
1585 .collect::<BTreeSet<_>>();
1586 if required.is_empty() {
1587 return None;
1588 }
1589 let module_id = registration.manifest.module_id.as_str();
1590 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1591 if required
1592 .iter()
1593 .any(|capability| verdict(capability).is_none())
1594 {
1595 self.refresh_capability_requirements();
1596 }
1597 required
1598 .into_iter()
1599 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1600 .map(str::to_string)
1601 }
1602
1603 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1604 self.capability_evaluator
1605 .statuses()
1606 .into_iter()
1607 .map(capability_requirement_status)
1608 .collect()
1609 }
1610
1611 fn deregister_connection(
1618 &self,
1619 connection_id: ConnectionId,
1620 reason: RegistrationEndReason,
1621 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1622 self.registry
1623 .deregister_connection_with_reason(connection_id, reason)
1624 }
1625
1626 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1627 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1628 return None;
1629 }
1630 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1631 parse_client_control_request(&frame.body)
1632 else {
1633 return None;
1634 };
1635 Some(target_module_id(&target).to_string())
1636 }
1637
1638 pub(crate) fn route_open_capacity_refusal(
1639 &self,
1640 ctx: &RouteCtx,
1641 frame: &Frame,
1642 target_module_id: &str,
1643 in_flight: usize,
1644 limit: usize,
1645 ) -> Result<Frame, RouterError> {
1646 let caller = self.route_open_caller_from_frame(frame);
1649 self.route_open_admission_refusal_frame(
1650 ctx,
1651 &caller,
1652 frame,
1653 target_module_id,
1654 "open_admission_full",
1655 (in_flight, limit),
1656 format!(
1657 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1658 ),
1659 )
1660 }
1661
1662 fn route_open_target_capacity_refusal(
1663 &self,
1664 ctx: &RouteCtx,
1665 caller: &RouteOpenCaller,
1666 frame: &Frame,
1667 target_module_id: &str,
1668 in_flight: usize,
1669 ) -> Result<Frame, RouterError> {
1670 self.route_open_admission_refusal_frame(
1671 ctx,
1672 caller,
1673 frame,
1674 target_module_id,
1675 "target_binds_full",
1676 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1677 format!(
1678 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1679 ),
1680 )
1681 }
1682
1683 #[allow(clippy::too_many_arguments)]
1691 fn route_open_admission_refusal_frame(
1692 &self,
1693 ctx: &RouteCtx,
1694 caller: &RouteOpenCaller,
1695 frame: &Frame,
1696 target_module_id: &str,
1697 reason: &'static str,
1698 (in_flight, limit): (usize, usize),
1699 message: impl Into<String>,
1700 ) -> Result<Frame, RouterError> {
1701 let code = error_codes::TARGET_UNAVAILABLE;
1702 self.counters.increment_route_open_refused(code);
1703 info!(
1704 target: "control",
1705 code,
1706 reason,
1707 module_id = ?target_module_id,
1708 connection_id = ctx.connection_id.get(),
1709 principal = %caller.principal,
1710 scope = %caller.scope,
1711 harness = %caller.harness,
1712 in_flight,
1713 limit,
1714 "route.open refused"
1715 );
1716 control_error_frame(frame, code, message.into())
1717 }
1718
1719 #[cfg(test)]
1725 pub fn handle_control(
1726 &self,
1727 connection_id: ConnectionId,
1728 frame: Frame,
1729 ) -> Result<Vec<Frame>, RouterError> {
1730 match frame.header.ty {
1731 FrameType::Ping => Ok(vec![pong(&frame)?]),
1732 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1733 FrameType::Goodbye => self.handle_goodbye(connection_id),
1734 ty => Ok(vec![control_error_frame(
1735 &frame,
1736 "unsupported_control_frame",
1737 format!("unsupported channel-0 frame {ty:?}"),
1738 )?]),
1739 }
1740 }
1741
1742 pub async fn handle_control_frame(
1743 &self,
1744 ctx: &RouteCtx,
1745 frame: Frame,
1746 ) -> Result<Vec<Frame>, RouterError> {
1747 self.handle_control_frame_timed(ctx, frame, None).await
1748 }
1749
1750 pub(crate) fn record_caller_endpoints(
1751 &self,
1752 connection: ConnectionId,
1753 endpoints: Option<(SocketAddr, SocketAddr)>,
1754 ) {
1755 let mut callers = self
1756 .caller_endpoints
1757 .lock()
1758 .unwrap_or_else(|p| p.into_inner());
1759 callers.remove(&connection);
1760 if let Some(endpoints) = endpoints {
1761 callers.insert(connection, endpoints);
1762 }
1763 }
1764
1765 #[cfg(test)]
1766 pub(crate) fn with_caller_lookup_for_test(
1767 mut self,
1768 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1769 ) -> Self {
1770 self.caller_lookup = Some(Arc::new(lookup));
1771 self
1772 }
1773
1774 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1775 let endpoints = self
1776 .caller_endpoints
1777 .lock()
1778 .unwrap_or_else(|p| p.into_inner())
1779 .get(&connection)
1780 .copied();
1781 #[cfg(test)]
1782 let lookup = self.caller_lookup.clone();
1783 tokio::task::spawn_blocking(move || {
1784 #[cfg(test)]
1785 if let Some(lookup) = lookup {
1786 return lookup(endpoints);
1787 }
1788 endpoints
1789 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1790 .unwrap_or_default()
1791 })
1792 .await
1793 .unwrap_or_default()
1794 }
1795
1796 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1797 frame.header.channel == 0
1798 && frame.header.ty == FrameType::Request
1799 && self
1800 .forwarding
1801 .module_endpoint_for_connection(ctx.connection_id)
1802 .is_ok_and(|endpoint| endpoint.is_none())
1803 && parse_client_control_request(&frame.body)
1804 .ok()
1805 .is_some_and(|request| privileged_operation(&request).is_some())
1806 }
1807
1808 pub(crate) async fn handle_control_frame_timed(
1809 &self,
1810 ctx: &RouteCtx,
1811 frame: Frame,
1812 dispatch_started_at: Option<StdInstant>,
1813 ) -> Result<Vec<Frame>, RouterError> {
1814 match frame.header.ty {
1815 FrameType::Ping => Ok(vec![pong(&frame)?]),
1816 FrameType::Hello => {
1817 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1818 }
1819 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1820 FrameType::Cancel => {
1821 if self
1824 .forwarding
1825 .operator_confirms()
1826 .cancel(ctx.connection_id, frame.header.corr)
1827 || self
1828 .supervisor
1829 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1830 || self.cancel_activity_subscription(ctx, &frame)?
1831 {
1832 Ok(Vec::new())
1833 } else {
1834 Ok(vec![control_error_frame(
1835 &frame,
1836 "unknown_subscription",
1837 "no subscription has this correlation id",
1838 )?])
1839 }
1840 }
1841 FrameType::Request => {
1842 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1845 if op
1846 .as_ref()
1847 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1848 {
1849 return self.handle_activity_record(ctx, frame).await;
1850 }
1851 if op.as_ref().is_some_and(|probe| {
1852 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1853 }) {
1854 return self.handle_activity_read(ctx, frame).await;
1855 }
1856 if op
1857 .as_ref()
1858 .is_some_and(|probe| probe.op == "operator.confirm")
1859 {
1860 return self.handle_operator_confirm(ctx, frame);
1861 }
1862 if op
1863 .as_ref()
1864 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1865 {
1866 return self.handle_scope_served_by(ctx.connection_id, frame);
1867 }
1868 if self
1869 .forwarding
1870 .module_endpoint_for_connection(ctx.connection_id)
1871 .map_err(RouterError::Forwarding)?
1872 .is_some()
1873 {
1874 if !is_known_module_request_op(&frame.body) {
1875 return Ok(vec![control_error_frame(
1876 &frame,
1877 "unsupported_control_frame",
1878 "module-originated channel-0 REQUEST is not supported",
1879 )?]);
1880 }
1881 let request = match parse_module_control_request_from_module(&frame.body) {
1882 Ok(request) => request,
1883 Err((err, ControlRequestBodyError::UnknownOp)) => {
1884 return Ok(vec![control_error_frame(
1885 &frame,
1886 "unsupported_control_frame",
1887 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1888 )?])
1889 }
1890 Err((err, ControlRequestBodyError::InvalidBody)) => {
1891 return Ok(vec![control_error_frame(
1892 &frame,
1893 "invalid_control_body",
1894 format!("malformed module control body: {err}"),
1895 )?])
1896 }
1897 };
1898 let op = module_control_request_op(&request);
1899 let corr = frame.header.corr;
1900 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1901 let result =
1902 self.handle_module_control_request(ctx.connection_id, frame, request);
1903 log_slow_control_dispatch(
1904 dispatch_started_at,
1905 op,
1906 ctx.connection_id,
1907 corr,
1908 None,
1909 );
1910 return result;
1911 }
1912
1913 if is_known_module_request_op(&frame.body) {
1914 return Ok(vec![control_error_frame(
1915 &frame,
1916 "not_registered",
1917 "catalog.update requires an active module registration owned by this connection",
1918 )?]);
1919 }
1920
1921 let request = match parse_client_control_request(&frame.body) {
1922 Ok(request) => request,
1923 Err((err, ControlRequestBodyError::UnknownOp)) => {
1924 return Ok(vec![control_error_frame(
1925 &frame,
1926 "unknown_control_op",
1927 format!("unknown client control op: {err}"),
1928 )?])
1929 }
1930 Err((err, ControlRequestBodyError::InvalidBody)) => {
1931 return Ok(vec![control_error_frame(
1932 &frame,
1933 "invalid_control_body",
1934 format!("malformed client control body: {err}"),
1935 )?])
1936 }
1937 };
1938 let op = client_control_request_op(&request);
1939 let corr = frame.header.corr;
1940 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1941 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1942 #[cfg(test)]
1943 if let Some(delay) = self.control_dispatch_delay {
1944 tokio::time::sleep(delay).await;
1945 }
1946 let result = self
1947 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1948 .await;
1949 if let Some(timings) = route_open_timings.as_mut() {
1950 timings.finish();
1951 }
1952 log_slow_control_dispatch(
1953 dispatch_started_at,
1954 op,
1955 ctx.connection_id,
1956 corr,
1957 route_open_timings.as_ref(),
1958 );
1959 result
1960 }
1961 FrameType::Push => {
1962 let Some(endpoint) = self
1963 .forwarding
1964 .module_endpoint_for_connection(ctx.connection_id)
1965 .map_err(RouterError::Forwarding)?
1966 else {
1967 return Ok(vec![control_error_frame(
1968 &frame,
1969 "unsupported_control_frame",
1970 "client-originated channel-0 PUSH is not supported",
1971 )?]);
1972 };
1973 self.handle_status_update(endpoint, frame)
1974 }
1975 FrameType::Response | FrameType::Error
1976 if self
1977 .forwarding
1978 .module_endpoint_for_connection(ctx.connection_id)
1979 .map_err(RouterError::Forwarding)?
1980 .is_some() =>
1981 {
1982 self.handle_module_relay_response(ctx.connection_id, frame)
1983 }
1984 ty => Ok(vec![control_error_frame(
1985 &frame,
1986 "unsupported_control_frame",
1987 format!("unsupported channel-0 frame {ty:?}"),
1988 )?]),
1989 }
1990 }
1991
1992 pub fn cleanup_connection(
1993 &self,
1994 connection_id: ConnectionId,
1995 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1996 self.caller_endpoints
1997 .lock()
1998 .unwrap_or_else(|p| p.into_inner())
1999 .remove(&connection_id);
2000 self.forwarding
2001 .operator_confirms()
2002 .connection_closed(connection_id);
2003 self.cleanup_connection_with_end_reason(
2004 connection_id,
2005 RegistrationEndReason::ConnectionClosed,
2006 )
2007 }
2008
2009 fn cleanup_connection_with_end_reason(
2010 &self,
2011 connection_id: ConnectionId,
2012 requested_reason: RegistrationEndReason,
2013 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2014 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2015 self.registry
2016 .get_module_by_connection(connection_id)?
2017 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2018 .and_then(|module| module.registration_end_reason().ok().flatten())
2019 .unwrap_or(requested_reason)
2020 } else {
2021 requested_reason
2022 };
2023 let crash_closed = self
2024 .registry
2025 .get_module_by_connection(connection_id)?
2026 .and_then(|registration| {
2027 self.forwarding
2028 .module_endpoint_for_connection(connection_id)
2029 .ok()
2030 .flatten()
2031 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2032 .map(|routes| (registration.manifest.module_id, routes))
2033 });
2034 let crash_closed = crash_closed.map(|(module_id, routes)| {
2035 let terminal = match self.supervisor.get(&module_id) {
2036 None => false,
2037 Some(module) => match module.will_recover_after_connection_loss() {
2038 Ok(will_recover) => !will_recover,
2039 Err(err) => {
2040 warn!(
2041 %module_id,
2042 error = %err,
2043 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2044 );
2045 false
2046 }
2047 },
2048 };
2049 let (reason, drained) = match self
2053 .forwarding
2054 .daemon_drain_result_for_connection(connection_id)
2055 {
2056 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2057 Ok(None) => (RouteCloseReason::Crash, false),
2058 Err(err) => {
2059 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2060 (RouteCloseReason::Crash, false)
2061 }
2062 };
2063 (module_id, routes, reason, drained, terminal)
2064 });
2065 let registrations = self.deregister_connection(connection_id, end_reason);
2066 let cleanup = if crash_closed.is_some() {
2067 self.forwarding.cleanup_connection_counted(connection_id)
2068 } else {
2069 self.forwarding
2075 .cleanup_connection(connection_id)
2076 .map(|released| crate::forwarding::ConnectionCleanup {
2077 released,
2078 abandoned_relays: 0,
2079 })
2080 };
2081 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2086 let abandoned = cleanup
2087 .as_ref()
2088 .map_or(0, |cleanup| cleanup.abandoned_relays);
2089 send_route_control_pushes(
2090 &self.forwarding,
2091 routes,
2092 ClientControlPush::RouteClosed {
2093 module_id,
2094 channels: Vec::new(),
2095 reason,
2096 drained,
2097 abandoned,
2098 excluded_subscriptions: 0,
2099 terminal: Some(terminal),
2100 },
2101 );
2102 }
2103 if let Ok(cleanup) = cleanup {
2104 self.emit_route_goodbyes(cleanup.released);
2105 }
2106 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2110 crate::supervise::notify_registration_release();
2111 self.capability_evaluator.wake_deadline_loop();
2112 self.refresh_capability_requirements();
2113 }
2114 self.supervisor.remove_spawn_subscribers(connection_id);
2115 self.activity_streams
2116 .lock()
2117 .unwrap_or_else(|p| p.into_inner())
2118 .retain(|(id, _), task| {
2119 if *id == connection_id {
2120 task.abort();
2121 false
2122 } else {
2123 true
2124 }
2125 });
2126 self.hello_launch_nonces
2129 .lock()
2130 .unwrap_or_else(|poisoned| poisoned.into_inner())
2131 .forget(connection_id);
2132 self.scopes
2133 .write()
2134 .unwrap_or_else(|poisoned| poisoned.into_inner())
2135 .release_connection(connection_id);
2136 registrations
2137 }
2138
2139 pub(crate) fn handle_route_goodbye(
2140 &self,
2141 connection_id: ConnectionId,
2142 route_channel: u16,
2143 route_epoch: u32,
2144 ) -> Result<bool, RouterError> {
2145 debug!(
2146 connection_id = connection_id.get(),
2147 route_channel, route_epoch, "handling route GOODBYE"
2148 );
2149 let RouteRelease::Removed(released_route) = self
2150 .forwarding
2151 .release_client_route(connection_id, route_channel, route_epoch)
2152 .map_err(RouterError::Forwarding)?
2153 else {
2154 return Ok(false);
2155 };
2156 self.emit_route_goodbyes(vec![released_route]);
2157 Ok(true)
2158 }
2159
2160 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2161 for released in released_routes {
2162 let frame = match Frame::build_with_version(
2163 released.negotiated_ver,
2164 FrameType::Goodbye,
2165 control_flags(),
2166 released.channel,
2167 released.epoch,
2168 0,
2169 Vec::new(),
2170 ) {
2171 Ok(frame) => frame,
2172 Err(err) => {
2173 warn!(
2174 route_channel = released.channel,
2175 error = %err,
2176 "failed to build route GOODBYE frame"
2177 );
2178 continue;
2179 }
2180 };
2181 if !released.close_on_delivery_failure() {
2182 crate::forwarding::send_module_route_goodbye(
2183 &self.counters,
2184 &released.sink,
2185 frame,
2186 released.module_id.as_deref(),
2187 "client route released",
2188 );
2189 continue;
2190 }
2191 if let Err(err) = released.sink.try_send(frame) {
2192 warn!(
2193 target_connection_id = released.connection_id.get(),
2194 route_channel = released.channel,
2195 error = %err,
2196 "route GOODBYE was not delivered to client; closing target connection"
2197 );
2198 if self
2199 .forwarding
2200 .escalate_client_delivery_failure(
2201 released.connection_id,
2202 released.channel,
2203 released.epoch,
2204 CloseReason::new(
2205 "route_goodbye_delivery_failed",
2206 format!(
2207 "failed to enqueue route GOODBYE for channel {}: {err}",
2208 released.channel
2209 ),
2210 ),
2211 crate::forwarding::UndeliveredFrame {
2212 module_id: released.module_id.as_deref(),
2213 sink: &released.sink,
2214 },
2215 )
2216 .unwrap_or(false)
2217 {
2218 self.counters.increment_goodbye_relay_client_failed();
2219 }
2220 }
2221 }
2222 }
2223
2224 fn send_abandoned_route_bind_goodbye(
2236 &self,
2237 module_sink: &crate::FrameSink,
2238 negotiated_ver: u8,
2239 module_channel: u16,
2240 module_epoch: u32,
2241 ) {
2242 let frame = match Frame::build_with_version(
2243 negotiated_ver,
2244 FrameType::Goodbye,
2245 control_flags(),
2246 module_channel,
2247 module_epoch,
2248 0,
2249 Vec::new(),
2250 ) {
2251 Ok(frame) => frame,
2252 Err(err) => {
2253 warn!(
2254 route_channel = module_channel,
2255 error = %err,
2256 "failed to build GOODBYE for abandoned route.bind"
2257 );
2258 return;
2259 }
2260 };
2261 crate::forwarding::send_module_route_goodbye(
2262 &self.counters,
2263 module_sink,
2264 frame,
2265 None,
2266 "abandoned route.bind",
2267 );
2268 }
2269
2270 fn handle_hello(
2271 &self,
2272 connection_id: ConnectionId,
2273 sink: Option<crate::FrameSink>,
2274 frame: Frame,
2275 ) -> Result<Vec<Frame>, RouterError> {
2276 debug!(
2277 connection_id = connection_id.get(),
2278 corr = frame.header.corr,
2279 "handling HELLO"
2280 );
2281 if self
2285 .registry
2286 .get_module_by_connection(connection_id)
2287 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2288 .is_some()
2289 {
2290 return Ok(vec![control_error_frame(
2291 &frame,
2292 "invalid_hello",
2293 "connection is already registered as a module",
2294 )?]);
2295 }
2296 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2297 Ok(value) => value,
2298 Err(err) => {
2299 return Ok(vec![control_error_frame(
2300 &frame,
2301 "invalid_hello",
2302 format!("malformed HELLO body: {err}"),
2303 )?])
2304 }
2305 };
2306 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2307 return Ok(vec![control_error_frame(
2308 &frame,
2309 "invalid_capability_grammar",
2310 err.to_string(),
2311 )?]);
2312 }
2313 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2314 return Ok(vec![control_error_frame(
2315 &frame,
2316 "invalid_manifest",
2317 err.to_string(),
2318 )?]);
2319 }
2320 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2321 return Ok(vec![control_error_frame(
2322 &frame,
2323 "invalid_event_declaration",
2324 err.to_string(),
2325 )?]);
2326 }
2327 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2328 return Ok(vec![control_error_frame(
2329 &frame,
2330 "invalid_manifest",
2331 err.to_string(),
2332 )?]);
2333 }
2334 if let Some(provenance) = hello_value
2335 .get("manifest")
2336 .and_then(|manifest| manifest.get("provenance"))
2337 {
2338 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2339 return Ok(vec![control_error_frame(
2340 &frame,
2341 "invalid_manifest",
2342 format!("malformed manifest provenance: {err}"),
2343 )?]);
2344 }
2345 }
2346 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2347 let module_id = hello_value["manifest"]["module_id"]
2348 .as_str()
2349 .unwrap_or("<unknown>");
2350 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2351 }
2352 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2353 Ok(hello) => hello,
2354 Err(err) => {
2355 return Ok(vec![control_error_frame(
2356 &frame,
2357 "invalid_hello",
2358 format!("malformed HELLO body: {err}"),
2359 )?])
2360 }
2361 };
2362
2363 if hello.protocol_ver != hello.manifest.protocol_ver {
2364 return Ok(vec![control_error_frame(
2365 &frame,
2366 "invalid_manifest",
2367 format!(
2368 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2369 hello.protocol_ver, hello.manifest.protocol_ver
2370 ),
2371 )?]);
2372 }
2373
2374 if hello.manifest.module_id.trim().is_empty() {
2375 return Ok(vec![control_error_frame(
2376 &frame,
2377 "invalid_manifest",
2378 "manifest module_id must not be empty",
2379 )?]);
2380 }
2381
2382 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2383 Ok(negotiated_ver) => negotiated_ver,
2384 Err(message) => {
2385 return Ok(vec![control_error_frame(
2386 &frame,
2387 "version_unsupported",
2388 message,
2389 )?])
2390 }
2391 };
2392
2393 let swap_admission = self
2401 .supervisor
2402 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2403 if swap_admission == SwapHelloAdmission::Refused {
2404 warn!(
2405 module_id = %hello.manifest.module_id,
2406 connection_id = connection_id.get(),
2407 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2408 );
2409 return Ok(vec![control_error_frame(
2410 &frame,
2411 "swap_token_invalid",
2412 format!(
2413 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2414 hello.manifest.module_id
2415 ),
2416 )?]);
2417 }
2418 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2419
2420 let nonce_verified = if swap_candidate {
2425 true
2426 } else {
2427 match self.supervisor.reserved_hello_verification(
2428 &hello.manifest.module_id,
2429 hello.launch_nonce.as_deref(),
2430 ) {
2431 Ok(verified) => verified,
2432 Err(rejection) => {
2433 let message = match rejection {
2434 ReservedHelloRejection::Exact { module_id } => format!(
2435 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2436 ),
2437 ReservedHelloRejection::Prefix {
2438 prefix,
2439 owner_module_id,
2440 } => format!(
2441 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2442 hello.manifest.module_id
2443 ),
2444 };
2445 return Ok(vec![control_error_frame(
2446 &frame,
2447 "reserved_module",
2448 message,
2449 )?]);
2450 }
2451 }
2452 };
2453
2454 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2455 &hello.manifest.module_id,
2456 hello.manifest.capabilities.as_ref(),
2457 );
2458 if let Some(refusal) = reserved_capability_refusals.first() {
2459 let capability = refusal.capability.clone();
2460 let bound_module = refusal.claimants[0].clone();
2461 log_duplicate_claim_events(reserved_capability_refusals);
2462 return Ok(vec![control_error_frame(
2463 &frame,
2464 "reserved_capability",
2465 format!(
2466 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2467 capability, bound_module, hello.manifest.module_id
2468 ),
2469 )?]);
2470 }
2471
2472 if self
2475 .forwarding
2476 .connection_has_client_routes(connection_id)
2477 .map_err(RouterError::Forwarding)?
2478 {
2479 return Ok(vec![control_error_frame(
2480 &frame,
2481 "invalid_hello",
2482 "connection has open client routes and cannot also register as a module",
2483 )?]);
2484 }
2485
2486 self.hello_launch_nonces
2491 .lock()
2492 .unwrap_or_else(|poisoned| poisoned.into_inner())
2493 .record(connection_id, hello.launch_nonce.as_deref());
2494 let control_ops = effective_module_control_ops(hello.control_ops);
2495 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2498 if swap_candidate {
2499 return self.register_swap_candidate(
2500 connection_id,
2501 sink,
2502 &frame,
2503 hello.manifest,
2504 negotiated_ver,
2505 control_ops,
2506 hello_ack,
2507 );
2508 }
2509 let registration = match self
2516 .registry
2517 .register_unpublished_verified_with_control_ops(
2518 hello.manifest,
2519 negotiated_ver,
2520 connection_id,
2521 control_ops,
2522 nonce_verified,
2523 ) {
2524 Ok(registration) => registration,
2525 Err(RegistryError::DuplicateModuleId { module_id }) => {
2526 return Ok(vec![control_error_frame(
2527 &frame,
2528 "duplicate_module_id",
2529 format!(
2530 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2531 ),
2532 )?])
2533 }
2534 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2535 return Ok(vec![control_error_frame(
2536 &frame,
2537 "invalid_module_id",
2538 err.to_string(),
2539 )?])
2540 }
2541 Err(err) => {
2542 return Ok(vec![control_error_frame(
2543 &frame,
2544 "registry_error",
2545 err.to_string(),
2546 )?])
2547 }
2548 };
2549 #[cfg(test)]
2550 self.hold_between_registry_and_forwarding();
2551
2552 let reply = if let Some(sink) = sink {
2553 let concurrency = manifest_concurrency(®istration.manifest);
2569 if let Err(err) = self.forwarding.register_module_connection_acked(
2570 connection_id,
2571 registration.manifest.module_id.clone(),
2572 negotiated_ver,
2573 concurrency,
2574 sink,
2575 hello_ack,
2576 ) {
2577 if matches!(
2582 self.deregister_connection(
2583 connection_id,
2584 RegistrationEndReason::RegistrationFailed,
2585 ),
2586 Ok(r) if !r.is_empty()
2587 ) {
2588 crate::supervise::notify_registration_release();
2589 }
2590 return Ok(vec![control_error_frame(
2591 &frame,
2592 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2593 "invalid_hello"
2594 } else {
2595 forwarding_error_code(&err)
2596 },
2597 err.to_string(),
2598 )?]);
2599 }
2600 Vec::new()
2601 } else {
2602 vec![hello_ack]
2605 };
2606
2607 if !self
2612 .registry
2613 .publish_registration(connection_id)
2614 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2615 {
2616 debug!(
2617 module_id = %registration.manifest.module_id,
2618 connection_id = connection_id.get(),
2619 "registration was removed before it could be published"
2620 );
2621 }
2622
2623 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2632 info!(
2633 module_id = %registration.manifest.module_id,
2634 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2635 );
2636 }
2637
2638 self.apply_registration_capabilities(®istration);
2639
2640 self.supervisor
2641 .record_activity_registration(®istration.manifest.module_id);
2642
2643 info!(
2644 module_id = %registration.manifest.module_id,
2645 module_version = %registration.manifest.module_version,
2646 negotiated_ver,
2647 routable_provider = manifest_provides_routable_role(®istration.manifest),
2648 connection_id = connection_id.get(),
2649 "module registered"
2650 );
2651
2652 Ok(reply)
2653 }
2654
2655 #[allow(clippy::too_many_arguments)]
2672 fn register_swap_candidate(
2673 &self,
2674 connection_id: ConnectionId,
2675 sink: Option<crate::FrameSink>,
2676 frame: &Frame,
2677 manifest: ModuleManifest,
2678 negotiated_ver: u8,
2679 control_ops: Vec<String>,
2680 hello_ack: Frame,
2681 ) -> Result<Vec<Frame>, RouterError> {
2682 let module_id = manifest.module_id.clone();
2683 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2684 manifest,
2685 negotiated_ver,
2686 connection_id,
2687 control_ops,
2688 ) {
2689 Ok(registration) => registration,
2690 Err(RegistryError::DuplicateModuleId { module_id }) => {
2691 return Ok(vec![control_error_frame(
2692 frame,
2693 "duplicate_module_id",
2694 format!(
2695 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2696 ),
2697 )?])
2698 }
2699 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2700 return Ok(vec![control_error_frame(
2701 frame,
2702 "invalid_module_id",
2703 err.to_string(),
2704 )?])
2705 }
2706 Err(err) => {
2707 return Ok(vec![control_error_frame(
2708 frame,
2709 "registry_error",
2710 err.to_string(),
2711 )?])
2712 }
2713 };
2714 #[cfg(test)]
2715 self.hold_between_registry_and_forwarding();
2716 let reply = if let Some(sink) = sink {
2717 let concurrency = manifest_concurrency(®istration.manifest);
2721 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2722 connection_id,
2723 module_id.clone(),
2724 negotiated_ver,
2725 concurrency,
2726 sink,
2727 hello_ack,
2728 ) {
2729 if matches!(
2730 self.deregister_connection(
2731 connection_id,
2732 RegistrationEndReason::RegistrationFailed,
2733 ),
2734 Ok(r) if !r.is_empty()
2735 ) {
2736 crate::supervise::notify_registration_release();
2737 }
2738 return Ok(vec![control_error_frame(
2739 frame,
2740 forwarding_error_code(&err),
2741 err.to_string(),
2742 )?]);
2743 }
2744 Vec::new()
2745 } else {
2746 vec![hello_ack]
2747 };
2748 if !self
2749 .registry
2750 .publish_registration(connection_id)
2751 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2752 {
2753 debug!(
2754 module_id = %module_id,
2755 connection_id = connection_id.get(),
2756 "swap candidate registration was removed before it could be published"
2757 );
2758 }
2759 self.supervisor.mark_swap_candidate_admitted(&module_id);
2760 info!(
2761 module_id = %module_id,
2762 module_version = %registration.manifest.module_version,
2763 negotiated_ver,
2764 ready = registration.ready,
2765 connection_id = connection_id.get(),
2766 "swap candidate registered; not routable until cutover"
2767 );
2768 Ok(reply)
2769 }
2770
2771 fn build_hello_ack(
2772 &self,
2773 frame: &Frame,
2774 negotiated_ver: u8,
2775 module_id: &str,
2776 ) -> Result<Frame, RouterError> {
2777 let ack = ModuleHelloAckBody {
2778 negotiated_ver,
2779 subc_ops: module_subc_ops(),
2780 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2781 storage: self
2782 .storage_config
2783 .as_ref()
2784 .map(|cfg| cfg.descriptor_for(module_id)),
2785 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2786 };
2787 let body = serde_json::to_vec(&ack).map_err(|err| {
2788 RouterError::backend(
2789 0,
2790 frame.header.corr,
2791 format!("failed to encode HELLO_ACK: {err}"),
2792 )
2793 })?;
2794
2795 Frame::build_with_version(
2796 negotiated_ver,
2797 FrameType::HelloAck,
2798 control_flags(),
2799 0,
2800 0,
2801 frame.header.corr,
2802 body,
2803 )
2804 .map_err(RouterError::FrameBuild)
2805 }
2806
2807 async fn handle_client_control_request(
2808 &self,
2809 ctx: &RouteCtx,
2810 frame: Frame,
2811 request: ClientControlRequest,
2812 route_open_timings: &mut Option<RouteOpenTimings>,
2813 ) -> Result<Vec<Frame>, RouterError> {
2814 if let Some(operation) = privileged_operation(&request) {
2815 let caller = self.resolve_caller(ctx.connection_id).await;
2816 let op = client_control_request_op(&request);
2817 let text = prompt_text(operation, &caller);
2818 match self.privileged_operator_prompt {
2819 PrivilegedOperatorPrompt::Log => {
2820 info!(target: "subc_daemon::privileged_prompt", operation = op,
2821 target = ?operation, caller = %caller.description(), summary = %text,
2822 "privileged request would prompt the operator");
2823 }
2824 PrivilegedOperatorPrompt::Enforce => {
2825 let key =
2826 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2827 let outcome = self
2828 .forwarding
2829 .operator_confirms()
2830 .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2831 .await;
2832 if outcome.code != "confirmed" {
2833 return Ok(vec![outcome.frame(&frame)]);
2834 }
2835 }
2836 }
2837 }
2838 match request {
2839 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2840 ClientControlRequest::CatalogList { module_id } => {
2841 self.handle_catalog_list(frame, module_id)
2842 }
2843 ClientControlRequest::RouteOpen {
2844 target,
2845 identity,
2846 consumer_identity,
2847 consumer_capabilities,
2848 role_versions,
2849 admission_facts,
2850 scope,
2851 } => {
2852 self.handle_route_open(
2853 ctx,
2854 frame,
2855 RouteOpenRequest {
2856 target,
2857 identity,
2858 consumer_identity,
2859 consumer_capabilities,
2860 role_versions,
2861 admission_facts,
2862 scope,
2863 },
2864 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2865 )
2866 .await
2867 }
2868 ClientControlRequest::RoutePoll {
2869 route_channel,
2870 route_epoch,
2871 kind,
2872 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2873 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2874 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2875 self.handle_supervisor_spawn_snapshot(frame)
2876 }
2877 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2878 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2879 }
2880 ClientControlRequest::SupervisorRestart {
2881 module_id,
2882 drain_timeout_ms,
2883 } => {
2884 self.log_supervisor_request_received(
2885 ctx,
2886 frame.header.corr,
2887 ops::SUPERVISOR_RESTART,
2888 Some(&module_id),
2889 None,
2890 )?;
2891 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2892 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2893 .await
2894 }
2895 ClientControlRequest::SupervisorSwap {
2896 module_id,
2897 ready_timeout_ms,
2898 } => {
2899 self.log_supervisor_request_received(
2900 ctx,
2901 frame.header.corr,
2902 ops::SUPERVISOR_SWAP,
2903 Some(&module_id),
2904 None,
2905 )?;
2906 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2907 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2908 .await
2909 }
2910 ClientControlRequest::SupervisorReload { module_id } => {
2911 self.log_supervisor_request_received(
2912 ctx,
2913 frame.header.corr,
2914 ops::SUPERVISOR_RELOAD,
2915 Some(&module_id),
2916 None,
2917 )?;
2918 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2919 self.handle_supervisor_reload(frame, module_id, requester)
2920 .await
2921 }
2922 ClientControlRequest::SupervisorRescan { preview } => {
2923 if !preview {
2924 self.log_supervisor_request_received(
2925 ctx,
2926 frame.header.corr,
2927 ops::SUPERVISOR_RESCAN,
2928 None,
2929 None,
2930 )?;
2931 }
2932 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2933 self.handle_supervisor_rescan(
2934 frame,
2935 if preview {
2936 RescanMode::Preview
2937 } else {
2938 RescanMode::Execute
2939 },
2940 requester,
2941 )
2942 .await
2943 }
2944 ClientControlRequest::SupervisorRescanChecked {
2945 preview,
2946 candidate,
2947 expect,
2948 } => {
2949 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
2950 .await
2951 }
2952 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2953 self.log_supervisor_request_received(
2954 ctx,
2955 frame.header.corr,
2956 ops::SUPERVISOR_RELEASE_RESERVED,
2957 Some(&module_id),
2958 None,
2959 )?;
2960 self.handle_supervisor_release_reserved(frame, module_id)
2961 .await
2962 }
2963 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2964 self.log_supervisor_request_received(
2965 ctx,
2966 frame.header.corr,
2967 ops::SUPERVISOR_SET_ENABLED,
2968 Some(&module_id),
2969 Some(enabled),
2970 )?;
2971 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2972 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2973 .await
2974 }
2975 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2976 self.handle_supervisor_health_probe(frame, module_id).await
2977 }
2978 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2979 ClientControlRequest::SupervisorRoutes { module_id } => {
2980 self.handle_supervisor_routes(frame, module_id)
2981 }
2982 ClientControlRequest::SupervisorProvenance { module_id } => {
2983 self.handle_supervisor_provenance(frame, module_id).await
2984 }
2985 ClientControlRequest::SupervisorStderrTail {
2986 module_id,
2987 max_lines,
2988 max_bytes,
2989 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2990 ClientControlRequest::SupervisorTerminals { module_id } => {
2991 self.handle_supervisor_terminals(frame, module_id).await
2992 }
2993 }
2994 }
2995
2996 fn log_supervisor_request_received(
2997 &self,
2998 ctx: &RouteCtx,
2999 corr: u64,
3000 op: &'static str,
3001 module_id: Option<&str>,
3002 enabled: Option<bool>,
3003 ) -> Result<(), RouterError> {
3004 let caller = self.supervisor_requester(ctx, corr)?;
3005 let caller = principal_label(&caller);
3006 info!(
3009 target: "control",
3010 op,
3011 module_id,
3012 enabled,
3013 connection_id = ctx.connection_id.get(),
3014 caller = %caller,
3015 "supervisor request received"
3016 );
3017 Ok(())
3018 }
3019
3020 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3021 Ok(self
3022 .registry
3023 .get_module_by_connection(ctx.connection_id)
3024 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3025 .map(|registration| Principal::Reserved {
3026 module_id: registration.manifest.module_id,
3027 })
3028 .unwrap_or(Principal::Direct))
3029 }
3030
3031 fn handle_module_control_request(
3032 &self,
3033 connection_id: ConnectionId,
3034 frame: Frame,
3035 request: ModuleControlRequestFromModule,
3036 ) -> Result<Vec<Frame>, RouterError> {
3037 match request {
3038 ModuleControlRequestFromModule::CatalogUpdate {
3039 provides,
3040 capabilities,
3041 ready,
3042 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3043 ModuleControlRequestFromModule::LiveRoots {} => {
3044 let registered = self
3045 .registry
3046 .get_module_by_connection(connection_id)
3047 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3048 let Some(registration) = registered else {
3049 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3050 };
3051 let response = self
3052 .forwarding
3053 .live_roots(®istration.manifest.module_id)
3054 .map_err(RouterError::Forwarding)?;
3055 Ok(vec![control_response_body_frame(
3056 &frame,
3057 &response,
3058 "ModuleControlResponseToModule::LiveRoots",
3059 )?])
3060 }
3061 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3062 self.handle_scope_sync(connection_id, frame, generation, scopes)
3063 }
3064 ModuleControlRequestFromModule::ScopeApply {
3065 generation,
3066 upsert,
3067 end,
3068 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3069 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3070 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3071 }
3072 }
3073 }
3074
3075 fn handle_scope_served_by(
3076 &self,
3077 connection_id: ConnectionId,
3078 frame: Frame,
3079 ) -> Result<Vec<Frame>, RouterError> {
3080 let registered = self
3081 .registry
3082 .get_module_by_connection(connection_id)
3083 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3084 let Some(registered) = registered else {
3085 return Ok(vec![control_error_frame(
3086 &frame,
3087 "not_registered",
3088 "scope.served_by requires an active module registration owned by this connection",
3089 )?]);
3090 };
3091 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3092 {
3093 Ok(request) => request,
3094 Err(err) => {
3095 return Ok(vec![control_error_frame(
3096 &frame,
3097 "invalid_control_body",
3098 format!("malformed scope.served_by body: {err}"),
3099 )?])
3100 }
3101 };
3102 let answer = self
3103 .scopes
3104 .read()
3105 .unwrap_or_else(|poisoned| poisoned.into_inner())
3106 .served_by(
3107 ®istered.manifest.module_id,
3108 &request.scope_ref,
3109 request.scope_epoch,
3110 &request.module_id,
3111 );
3112 let reply = subc_control::ScopeServedByReply::new(
3113 answer,
3114 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3115 );
3116 Ok(vec![control_response_body_frame(
3117 &frame,
3118 &reply,
3119 "ScopeServedByReply",
3120 )?])
3121 }
3122
3123 fn handle_operator_confirm(
3124 &self,
3125 ctx: &RouteCtx,
3126 frame: Frame,
3127 ) -> Result<Vec<Frame>, RouterError> {
3128 use crate::operator_confirm::{audit, Outcome};
3129 let registration = self
3130 .registry
3131 .get_module_by_connection(ctx.connection_id)
3132 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3133 let Some(registration) = registration else {
3134 let outcome = Outcome::refusal("not_registered");
3135 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3136 return Ok(vec![outcome.frame(&frame)]);
3137 };
3138 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3139 Ok(request) => request,
3140 Err(_) => {
3141 let outcome = Outcome::refusal("invalid_control_body");
3142 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3143 return Ok(vec![outcome.frame(&frame)]);
3144 }
3145 };
3146 let module_id = registration.manifest.module_id;
3147 let nonce = self
3151 .hello_launch_nonces
3152 .lock()
3153 .unwrap_or_else(|p| p.into_inner())
3154 .nonce(ctx.connection_id)
3155 .map(str::to_owned);
3156 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3157 self.supervisor
3158 .spawned_consumer_authorized(&module_id, nonce)
3159 });
3160 let confirms = self.forwarding.operator_confirms();
3161 self.forwarding
3162 .with_operator_route(
3163 ctx.connection_id,
3164 request.route_channel,
3165 request.route_epoch,
3166 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3167 )
3168 .map_err(RouterError::Forwarding)
3169 }
3170
3171 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3174 let mut scopes = ScopeRecords::from(scopes);
3175 for (index, record) in scopes.records.iter_mut().enumerate() {
3176 if let Some(refusal) = Self::canonicalize_workdir(record) {
3177 scopes.refusals.insert(index, refusal);
3178 }
3179 }
3180 scopes
3181 }
3182
3183 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3184 let workdir = record.workdir.as_mut()?;
3185 let path = std::path::Path::new(workdir);
3186 if !path.is_absolute() {
3187 return Some((
3188 error_codes::INVALID_WORKDIR,
3189 "workdir must be absolute".to_string(),
3190 ));
3191 }
3192 match ProjectRootId::from_path_allowing_missing(path) {
3193 Ok(root) => match root.as_path().to_str() {
3194 Some(canonical) => {
3195 *workdir = canonical.to_string();
3196 None
3197 }
3198 None => Some((
3199 error_codes::INVALID_WORKDIR,
3200 "canonical workdir is not UTF-8".to_string(),
3201 )),
3202 },
3203 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3204 }
3205 }
3206
3207 fn handle_scope_sync(
3211 &self,
3212 connection_id: ConnectionId,
3213 frame: Frame,
3214 generation: u64,
3215 scopes: Vec<ScopeRecord>,
3216 ) -> Result<Vec<Frame>, RouterError> {
3217 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3218 }
3219
3220 fn handle_scope_change(
3221 &self,
3222 connection_id: ConnectionId,
3223 frame: Frame,
3224 generation: u64,
3225 scopes: Vec<ScopeRecord>,
3226 end: Option<Vec<ScopeEnd>>,
3227 ) -> Result<Vec<Frame>, RouterError> {
3228 let is_apply = end.is_some();
3229 let Some(registration) = self
3230 .registry
3231 .get_module_by_connection(connection_id)
3232 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3233 else {
3234 return Ok(vec![control_error_frame(
3235 &frame,
3236 "not_registered",
3237 "scope.sync and scope.apply require an active module registration owned by this connection",
3238 )?]);
3239 };
3240 let owner = registration.manifest.module_id;
3241 let scopes = Self::prepare_scope_records(scopes);
3242 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3243 let is_current_launch = |connection: ConnectionId| {
3244 self.hello_launch_nonces
3245 .lock()
3246 .unwrap_or_else(|poisoned| poisoned.into_inner())
3247 .presented(connection, current_nonce.as_deref())
3248 };
3249 let mut table = self
3254 .scopes
3255 .write()
3256 .unwrap_or_else(|poisoned| poisoned.into_inner());
3257 let outcome = match end {
3258 Some(end) => table.apply(
3259 &owner,
3260 connection_id,
3261 is_current_launch,
3262 generation,
3263 scopes,
3264 end,
3265 ),
3266 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3267 };
3268 let drained = match &outcome {
3269 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3270 Err(_) => Vec::new(),
3271 };
3272 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3273 drop(table);
3274 if has_deadlines {
3275 self.scope_deadline_added.notify_one();
3276 }
3277 match outcome {
3278 Ok(applied) => {
3279 let counts = ScopeOutcomeCounts::of(&applied.results);
3280 info!(
3281 owner = %owner,
3282 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3283 generation,
3284 records = applied.results.len(),
3285 created = counts.created,
3286 replaced = counts.replaced,
3287 updated = counts.updated,
3288 unchanged = counts.unchanged,
3289 refused = counts.refused,
3290 ended = applied.ended.len(),
3291 tag_changes = applied.tag_changes.len(),
3292 routes_closed = drained.len(),
3293 "scope change accepted"
3294 );
3295 for refused in applied
3301 .results
3302 .iter()
3303 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3304 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3305 {
3306 warn!(
3307 owner = %owner,
3308 generation,
3309 scope_ref = %refused.scope_ref,
3310 scope_epoch = refused.scope_epoch,
3311 code = refused.code.as_deref().unwrap_or(""),
3312 "scope record refused"
3313 );
3314 }
3315 self.close_scope_drained_routes(drained);
3316 let response = if is_apply {
3317 ModuleControlResponseToModule::ScopeApply {
3318 generation,
3319 results: applied.results,
3320 end_results: applied.end_results,
3321 ended: applied.ended,
3322 }
3323 } else {
3324 ModuleControlResponseToModule::ScopeSync {
3325 generation,
3326 results: applied.results,
3327 ended: applied.ended,
3328 }
3329 };
3330 Ok(vec![control_response_body_frame(
3331 &frame,
3332 &response,
3333 "scope change response",
3334 )?])
3335 }
3336 Err(refusal) => {
3337 info!(
3338 owner = %owner,
3339 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3340 generation,
3341 code = refusal.code,
3342 "scope change refused"
3343 );
3344 Ok(vec![control_error_frame(
3345 &frame,
3346 refusal.code,
3347 refusal.message,
3348 )?])
3349 }
3350 }
3351 }
3352
3353 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3359 if drained.is_empty() {
3360 return;
3361 }
3362 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3363 BTreeMap::new();
3364 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3365 for route in drained {
3366 warn!(
3367 module_id = %route.module_id,
3368 reason = ?route.reason,
3369 client_connection_id = route.client.connection_id.get(),
3370 route_channel = route.client.channel,
3371 "closing route because its scope changed"
3372 );
3373 pushes
3374 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3375 .or_insert_with(|| (route.reason, Vec::new()))
3376 .1
3377 .push(EndpointRoute {
3378 goodbye_target: route.client.clone(),
3379 principal: Principal::Unverified,
3380 bound_at: Instant::now(),
3381 draining: false,
3382 drain_reason: None,
3383 });
3384 goodbyes.push(route.module);
3385 goodbyes.push(route.client);
3386 }
3387 for ((module_id, _), (reason, routes)) in pushes {
3388 send_route_control_pushes(
3389 &self.forwarding,
3390 routes,
3391 ClientControlPush::RouteClosed {
3392 module_id,
3393 channels: Vec::new(),
3394 reason,
3395 drained: false,
3396 abandoned: 0,
3397 excluded_subscriptions: 0,
3398 terminal: Some(false),
3399 },
3400 );
3401 }
3402 self.emit_route_goodbyes(goodbyes);
3403 }
3404
3405 fn handle_scope_describe(
3408 &self,
3409 connection_id: ConnectionId,
3410 frame: Frame,
3411 owner: Principal,
3412 scope_ref: String,
3413 ) -> Result<Vec<Frame>, RouterError> {
3414 let registered = self
3415 .registry
3416 .get_module_by_connection(connection_id)
3417 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3418 let Some(registered) = registered else {
3419 return Ok(vec![control_error_frame(
3420 &frame,
3421 "not_registered",
3422 "scope.describe requires an active module registration owned by this connection",
3423 )?]);
3424 };
3425 let mut description = self
3426 .scopes
3427 .read()
3428 .unwrap_or_else(|poisoned| poisoned.into_inner())
3429 .describe(&owner, &scope_ref);
3430 if !registered
3433 .manifest
3434 .capabilities
3435 .as_ref()
3436 .is_some_and(|caps| {
3437 caps.provides
3438 .iter()
3439 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3440 })
3441 {
3442 if let Some(stamp) = description.stamp.as_mut() {
3443 stamp.attributes.on_behalf_of_agent = None;
3444 }
3445 }
3446 let owner_configured = match &owner {
3447 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3452 _ => false,
3453 };
3454 let response = ModuleControlResponseToModule::ScopeDescribe {
3455 status: description.status,
3456 scope_epoch: description.scope_epoch,
3457 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3458 owner_synced: description.owner_synced,
3459 owner_configured,
3460 scope: description.stamp,
3461 };
3462 Ok(vec![control_response_body_frame(
3463 &frame,
3464 &response,
3465 "ModuleControlResponseToModule::ScopeDescribe",
3466 )?])
3467 }
3468
3469 fn handle_catalog_update(
3470 &self,
3471 connection_id: ConnectionId,
3472 frame: Frame,
3473 provides: Vec<ProviderRole>,
3474 capabilities: Option<CapabilityDeclarations>,
3475 ready: Option<bool>,
3476 ) -> Result<Vec<Frame>, RouterError> {
3477 self.refresh_capability_requirements();
3478 let Some(registration) = self
3479 .registry
3480 .get_module_by_connection(connection_id)
3481 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3482 else {
3483 return Ok(vec![control_error_frame(
3484 &frame,
3485 "not_registered",
3486 "catalog.update requires an active module registration owned by this connection",
3487 )?]);
3488 };
3489
3490 if let Some(message) =
3491 catalog_update_frozen_field_message(®istration.manifest, &provides)
3492 {
3493 return Ok(vec![control_error_frame(
3494 &frame,
3495 "catalog_update_frozen_field",
3496 message,
3497 )?]);
3498 }
3499
3500 let mut candidate = registration.manifest.clone();
3501 candidate.provides = provides.clone();
3502 candidate.capabilities = capabilities
3503 .clone()
3504 .or_else(|| registration.manifest.capabilities.clone());
3505 if let Some(rich_ui) = &candidate.rich_ui {
3506 if let Err(err) = rich_ui.validate() {
3507 return Ok(vec![control_error_frame(
3508 &frame,
3509 "invalid_manifest",
3510 err.to_string(),
3511 )?]);
3512 }
3513 }
3514 if let Err(err) = candidate.validate_capability_grammar() {
3515 return Ok(vec![control_error_frame(
3516 &frame,
3517 "invalid_capability_grammar",
3518 err.to_string(),
3519 )?]);
3520 }
3521
3522 let mut conflicts = self
3525 .capability_evaluator
3526 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3527 if let Some(conflict) = conflicts.first() {
3528 let message = format!(
3529 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3530 conflict.capability, conflict.claimants[0], candidate.module_id
3531 );
3532 for conflict in &mut conflicts {
3533 conflict.source = DuplicateClaimSource::CatalogUpdate;
3534 }
3535 log_duplicate_claim_events(conflicts);
3536 return Ok(vec![control_error_frame(
3537 &frame,
3538 "reserved_capability",
3539 message,
3540 )?]);
3541 }
3542
3543 let updated = self
3544 .registry
3545 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3546 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3547 if updated.is_none() {
3548 return Ok(vec![control_error_frame(
3549 &frame,
3550 "not_registered",
3551 "catalog.update requires an active module registration owned by this connection",
3552 )?]);
3553 }
3554 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3555 log_duplicate_claim_events(
3556 self.capability_evaluator
3557 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3558 );
3559 }
3560 if capability_census_trigger(
3561 registration.manifest.capabilities.as_ref(),
3562 updated
3563 .as_ref()
3564 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3565 ) {
3566 self.enforce_capability_denies();
3567 }
3568 self.refresh_capability_requirements();
3569
3570 let response = ModuleControlResponseToModule::CatalogUpdate {};
3571 control_response_body_frame(
3572 &frame,
3573 &response,
3574 "ModuleControlResponseToModule::CatalogUpdate",
3575 )
3576 .map(|frame| vec![frame])
3577 }
3578
3579 pub(crate) fn with_activity_ledger_config(
3580 mut self,
3581 run_dir: &Path,
3582 incarnation: String,
3583 config: crate::activity::ActivityConfig,
3584 ) -> Self {
3585 self.activity = Some(crate::activity::ActivityLedger::open_config(
3586 run_dir.join("activity.jsonl"),
3587 incarnation,
3588 config,
3589 ));
3590 self.supervisor.set_activity(self.activity.clone());
3591 self
3592 }
3593
3594 pub(crate) async fn configure_activity(
3595 &self,
3596 config: crate::activity::ActivityConfig,
3597 ) -> Result<(), &'static str> {
3598 match &self.activity {
3599 Some(ledger) => ledger.configure(config).await,
3600 None => Ok(()),
3601 }
3602 }
3603
3604 fn cancel_activity_subscription(
3605 &self,
3606 ctx: &RouteCtx,
3607 frame: &Frame,
3608 ) -> Result<bool, RouterError> {
3609 let task = self
3610 .activity_streams
3611 .lock()
3612 .unwrap_or_else(|p| p.into_inner())
3613 .remove(&(ctx.connection_id, frame.header.corr));
3614 let Some(task) = task else {
3615 return Ok(false);
3616 };
3617 task.abort();
3618 let end = Frame::build_with_version(
3619 response_version(frame),
3620 FrameType::StreamEnd,
3621 control_flags(),
3622 0,
3623 0,
3624 frame.header.corr,
3625 Vec::new(),
3626 )
3627 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3628 let sink = ctx.egress.clone();
3629 tokio::spawn(async move {
3630 let _ = sink.send(end).await;
3631 });
3632 Ok(true)
3633 }
3634
3635 async fn handle_activity_read(
3636 &self,
3637 ctx: &RouteCtx,
3638 frame: Frame,
3639 ) -> Result<Vec<Frame>, RouterError> {
3640 use subc_control::{
3641 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3642 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3643 };
3644 #[derive(Deserialize)]
3645 struct Identity {
3646 #[serde(default)]
3647 consumer_identity: Option<ConsumerIdentity>,
3648 }
3649 let principal = match self
3653 .forwarding
3654 .module_id_for_connection(ctx.connection_id)
3655 .map_err(RouterError::Forwarding)?
3656 {
3657 Some(module_id) => Ok(Principal::Reserved { module_id }),
3658 None => {
3659 let identity: Identity = match serde_json::from_slice(&frame.body) {
3660 Ok(identity) => identity,
3661 Err(_) => {
3662 return Ok(vec![control_error_frame(
3663 &frame,
3664 ACTIVITY_MALFORMED,
3665 "malformed activity read",
3666 )?])
3667 }
3668 };
3669 self.route_open_principal(&frame, identity.consumer_identity)?
3670 }
3671 };
3672 let admitted = matches!(principal, Ok(Principal::Direct))
3673 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3674 if !admitted {
3675 return Ok(vec![control_error_frame(
3676 &frame,
3677 ACTIVITY_READ_FORBIDDEN,
3678 "activity reads require an operator or trusted mirror principal",
3679 )?]);
3680 }
3681 let Some(ledger) = &self.activity else {
3682 return Ok(vec![control_error_frame(
3683 &frame,
3684 ACTIVITY_UNAVAILABLE,
3685 "activity ledger unavailable",
3686 )?]);
3687 };
3688 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3689 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3690 {
3691 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3692 Ok(request) => ledger.query(request).await,
3693 Err(_) => Err(ACTIVITY_MALFORMED),
3694 };
3695 return match result {
3696 Ok(reply) => Ok(vec![control_response_body_frame(
3697 &frame,
3698 &reply,
3699 ops::ACTIVITY_QUERY,
3700 )?]),
3701 Err(code) => Ok(vec![control_error_frame(
3702 &frame,
3703 code,
3704 "activity query refused",
3705 )?]),
3706 };
3707 }
3708 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3709 Ok(request) => request,
3710 Err(_) => {
3711 return Ok(vec![control_error_frame(
3712 &frame,
3713 ACTIVITY_MALFORMED,
3714 "malformed activity subscription",
3715 )?])
3716 }
3717 };
3718 let mut subscription = match ledger.subscribe(request.since).await {
3719 Ok(subscription) => subscription,
3720 Err(code) => {
3721 return Ok(vec![control_error_frame(
3722 &frame,
3723 code,
3724 "activity subscription refused",
3725 )?])
3726 }
3727 };
3728 let key = (ctx.connection_id, frame.header.corr);
3729 let sink = ctx.egress.clone();
3730 let mut streams = self
3731 .activity_streams
3732 .lock()
3733 .unwrap_or_else(|p| p.into_inner());
3734 streams.retain(|_, task| !task.is_finished());
3735 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3736 return Ok(vec![control_error_frame(
3737 &frame,
3738 "duplicate_subscription",
3739 "activity subscription correlation id already in use",
3740 )?]);
3741 }
3742 let task = tokio::spawn(async move {
3743 loop {
3744 let entry = if let Some(entry) = subscription.replay.pop_front() {
3745 Some(entry)
3746 } else {
3747 subscription.receiver.recv().await
3748 };
3749 let Some(entry) = entry else {
3750 break;
3751 };
3752 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3753 return;
3754 };
3755 let Ok(next) = Frame::build_with_version(
3756 response_version(&frame),
3757 FrameType::StreamData,
3758 control_flags(),
3759 0,
3760 0,
3761 frame.header.corr,
3762 body,
3763 ) else {
3764 return;
3765 };
3766 if sink.send(next).await.is_err() {
3767 return;
3768 }
3769 }
3770 if let Some(code) = subscription.terminal_code() {
3773 if let Ok(end) =
3774 control_error_frame(&frame, code, "activity subscriber fell behind")
3775 {
3776 let _ = sink.send(end).await;
3777 }
3778 }
3779 });
3780 streams.insert(key, task.abort_handle());
3781 Ok(Vec::new())
3782 }
3783
3784 async fn handle_activity_record(
3785 &self,
3786 ctx: &RouteCtx,
3787 frame: Frame,
3788 ) -> Result<Vec<Frame>, RouterError> {
3789 use subc_control::{
3790 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3791 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3792 };
3793 let result = async {
3794 let module = self
3795 .forwarding
3796 .module_id_for_connection(ctx.connection_id)
3797 .ok()
3798 .flatten()
3799 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3800 let request: ActivityRecordRequest =
3801 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3802 crate::activity::validate(&module, &request)?;
3803 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3804 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3805 let at_ms = std::time::SystemTime::now()
3806 .duration_since(std::time::UNIX_EPOCH)
3807 .unwrap_or_default()
3808 .as_millis() as u64;
3809 ledger.record(module, request, actor, at_ms).await
3810 }
3811 .await;
3812 match result {
3813 Ok(entry) => Ok(vec![control_response_body_frame(
3814 &frame,
3815 &ActivityRecordResponse { seq: entry.seq },
3816 "activity.record",
3817 )?]),
3818 Err(code) => Ok(vec![control_error_frame(
3819 &frame,
3820 code,
3821 "activity recording refused",
3822 )?]),
3823 }
3824 }
3825
3826 fn activity_actor(
3827 &self,
3828 connection: ConnectionId,
3829 module: &str,
3830 on: Option<&subc_control::ActivityOn>,
3831 ) -> Result<subc_control::ActivityActor, &'static str> {
3832 use crate::forwarding::{DataRoute, DataRouteState};
3833 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3834 let principal = |id: &str| Principal::Reserved {
3835 module_id: id.to_owned(),
3836 };
3837 let mut actor = ActivityActor {
3838 principal: principal(module),
3839 scope_owner: None,
3840 agent_id: None,
3841 run_id: None,
3842 flow_id: None,
3843 project_root: None,
3844 workdir: None,
3845 project_id: None,
3846 };
3847 let scope = match on {
3848 None => return Ok(actor),
3849 Some(ActivityOn::Scope(scope)) => {
3850 let selector = subc_protocol::scope::ScopeSelector::new(
3851 principal(module),
3852 scope.scope_ref.clone(),
3853 Some(scope.epoch),
3854 );
3855 self.scopes
3856 .read()
3857 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3858 .attribute(&principal(module), module, &selector, true)
3859 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3860 }
3861 Some(ActivityOn::Route(route)) => {
3862 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3863 .forwarding
3864 .lookup_data_route(connection, route.channel, route.epoch)
3865 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3866 else {
3867 return Err(ACTIVITY_ACTOR_UNKNOWN);
3868 };
3869 if binding.module_id != module {
3870 return Err(ACTIVITY_ACTOR_UNKNOWN);
3871 }
3872 actor.principal = binding.principal.clone();
3873 actor.project_root = binding
3874 .project_root
3875 .as_ref()
3876 .map(|root| root.as_path().to_path_buf());
3877 let Some(bound) = &binding.scope else {
3878 return Ok(actor);
3879 };
3880 let owner = principal(&bound.owner);
3881 let selector = subc_protocol::scope::ScopeSelector::new(
3882 owner.clone(),
3883 bound.scope_ref.clone(),
3884 Some(bound.tag.scope_epoch),
3885 );
3886 let admitted = self
3887 .scopes
3888 .read()
3889 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3890 .attribute(&owner, module, &selector, true)
3891 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3892 if admitted.tag != bound.tag {
3893 return Err(ACTIVITY_ACTOR_UNKNOWN);
3894 }
3895 admitted
3896 }
3897 };
3898 actor.scope_owner = Some(scope.stamp.owner);
3899 actor.agent_id = scope.stamp.attributes.agent_id;
3900 actor.run_id = scope.stamp.attributes.run_id;
3901 actor.flow_id = scope.stamp.attributes.flow_id;
3902 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
3903 actor.project_id = scope.stamp.project_id;
3904 Ok(actor)
3905 }
3906
3907 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3908 self.refresh_capability_requirements();
3909 let mut counters = self.counters.snapshot();
3915 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3916 obj.insert(
3917 "activity_unparseable_lines".into(),
3918 ledger.read_errors().into(),
3919 );
3920 obj.insert(
3921 "activity_lifecycle_append_failures".into(),
3922 ledger.lifecycle_failures().into(),
3923 );
3924 obj.insert(
3925 "activity_started_digest_skips".into(),
3926 ledger.lifecycle_digest_skips().into(),
3927 );
3928 }
3929 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3930 self.forwarding.client_route_concentration(),
3931 counters.as_object_mut(),
3932 ) {
3933 obj.insert(
3934 "client_connections_with_routes".into(),
3935 connections_with_routes.into(),
3936 );
3937 obj.insert("max_routes_on_one_connection".into(), max.into());
3938 }
3939 if let (Some(open_breakers), Some(obj)) = (
3945 self.route_bind_breakers.open_snapshot(),
3946 counters.as_object_mut(),
3947 ) {
3948 obj.insert("route_bind_breakers_open".into(), open_breakers);
3949 }
3950 let response = ClientControlResponse::ServerDescribe {
3951 protocol_ver: PROTOCOL_VERSION,
3952 subc_ops: subc_ops(),
3953 capabilities: self.describe_capabilities(),
3954 connected_clients: self.connected_clients.count(),
3955 counters: Some(counters),
3956 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3957 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3958 capability_requirements: self.capability_requirement_statuses(),
3959 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3960 config_path: self
3961 .rescan
3962 .as_ref()
3963 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
3964 };
3965 Ok(vec![control_response_body_frame(
3966 &frame,
3967 &response,
3968 "ClientControlResponse::ServerDescribe",
3969 )?])
3970 }
3971
3972 fn handle_catalog_list(
3973 &self,
3974 frame: Frame,
3975 module_id: Option<String>,
3976 ) -> Result<Vec<Frame>, RouterError> {
3977 let (generation, modules) = self
3978 .registry
3979 .list_modules_with_nonce_verification()
3980 .map_err(|err| {
3981 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3982 })?;
3983 let entries = modules
3984 .into_iter()
3985 .filter(|(registration, _)| {
3986 module_id
3987 .as_deref()
3988 .map(|wanted| registration.manifest.module_id == wanted)
3989 .unwrap_or(true)
3990 })
3991 .map(|(registration, nonce_verified)| {
3992 let not_ready = self.not_ready_reason(®istration);
3993 let roles = registration.manifest.provides;
3994 CatalogEntry::new(
3995 registration.manifest.module_id,
3996 roles,
3997 registration.control_ops,
3998 )
3999 .with_ready(not_ready.is_none())
4000 .with_not_ready(not_ready)
4001 .with_module_version(Some(registration.manifest.module_version))
4002 .with_capabilities(registration.manifest.capabilities)
4003 .with_self_signals(registration.manifest.self_signals)
4004 .with_events(registration.manifest.events)
4005 .with_operation_delivery(registration.manifest.operation_delivery)
4006 .with_operation_payload(registration.manifest.operation_payload)
4007 .with_rich_ui(registration.manifest.rich_ui)
4008 .with_role_declarations(registration.manifest.role_declarations)
4009 .with_nonce_verified(nonce_verified)
4010 })
4011 .collect();
4012 let response = ClientControlResponse::CatalogList {
4013 generation,
4014 modules: entries,
4015 subc_ops: subc_ops(),
4016 };
4017 Ok(vec![control_response_body_frame(
4018 &frame,
4019 &response,
4020 "ClientControlResponse::CatalogList",
4021 )?])
4022 }
4023
4024 fn attest_route_open_consumer(
4029 &self,
4030 consumer_identity: Option<&ConsumerIdentity>,
4031 ) -> Option<Principal> {
4032 let Some(consumer_identity) = consumer_identity else {
4033 return Some(Principal::Direct);
4034 };
4035 self.supervisor
4036 .spawned_consumer_authorized(
4037 &consumer_identity.module_id,
4038 &consumer_identity.launch_nonce,
4039 )
4040 .then(|| Principal::Reserved {
4041 module_id: consumer_identity.module_id.clone(),
4042 })
4043 }
4044
4045 fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4049 let Ok(ClientControlRequest::RouteOpen {
4050 identity,
4051 consumer_identity,
4052 scope,
4053 ..
4054 }) = parse_client_control_request(&frame.body)
4055 else {
4056 return RouteOpenCaller::unknown();
4057 };
4058 let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4059 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4060 }
4061
4062 fn route_open_principal(
4063 &self,
4064 frame: &Frame,
4065 consumer_identity: Option<ConsumerIdentity>,
4066 ) -> Result<Result<Principal, Frame>, RouterError> {
4067 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4068 self.route_open_principal_attested(frame, consumer_identity, attested)
4069 }
4070
4071 fn route_open_principal_attested(
4075 &self,
4076 frame: &Frame,
4077 consumer_identity: Option<ConsumerIdentity>,
4078 attested: Option<Principal>,
4079 ) -> Result<Result<Principal, Frame>, RouterError> {
4080 if let Some(principal) = attested {
4081 return Ok(Ok(principal));
4082 }
4083 let Some(consumer_identity) = consumer_identity else {
4084 return Ok(Ok(Principal::Direct));
4085 };
4086
4087 Ok(Err(control_error_frame(
4088 frame,
4089 "bad_consumer_identity",
4090 format!(
4091 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4092 consumer_identity.module_id
4093 ),
4094 )?))
4095 }
4096
4097 #[allow(clippy::too_many_arguments)]
4104 fn route_open_refusal_frame(
4105 &self,
4106 ctx: &RouteCtx,
4107 caller: &RouteOpenCaller,
4108 frame: &Frame,
4109 module_id: &str,
4110 reason: &'static str,
4111 code: &'static str,
4112 message: impl Into<String>,
4113 ) -> Result<Frame, RouterError> {
4114 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4115 control_error_frame(frame, code, message.into())
4116 }
4117
4118 #[allow(clippy::too_many_arguments)]
4136 fn route_open_breaker_refusal_frame(
4137 &self,
4138 ctx: &RouteCtx,
4139 caller: &RouteOpenCaller,
4140 frame: &Frame,
4141 module_id: &str,
4142 consecutive_timeouts: u32,
4143 retry_in: Duration,
4144 probe_in_flight: bool,
4145 ) -> Result<Frame, RouterError> {
4146 self.counters
4147 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4148 debug!(
4149 target: "control",
4150 code = "module_timeout",
4151 module_id = ?module_id,
4152 connection_id = ctx.connection_id.get(),
4153 principal = %caller.principal,
4154 scope = %caller.scope,
4155 harness = %caller.harness,
4156 consecutive_timeouts,
4157 retry_in_ms = retry_in.as_millis() as u64,
4158 probe_in_flight,
4159 "route.open refused by open bind-relay breaker"
4160 );
4161 let detail = if probe_in_flight {
4168 "one test route is already being tried; retry once it settles".to_string()
4169 } else {
4170 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4171 };
4172 control_error_frame(
4173 frame,
4174 "module_timeout",
4175 format!(
4176 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4177 timed out in a row); {detail}; its established routes are unaffected"
4178 ),
4179 )
4180 }
4181
4182 fn observe_route_open_refusal(
4195 &self,
4196 ctx: &RouteCtx,
4197 caller: &RouteOpenCaller,
4198 module_id: &str,
4199 reason: &'static str,
4200 code: &'static str,
4201 ) {
4202 self.counters.increment_route_open_refused(code);
4203 info!(
4204 target: "control",
4205 code,
4206 reason,
4207 module_id = ?module_id,
4208 connection_id = ctx.connection_id.get(),
4209 principal = %caller.principal,
4210 scope = %caller.scope,
4211 harness = %caller.harness,
4212 "route.open refused"
4213 );
4214 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4215 self.route_outages.record_not_serving(module_id, reason);
4216 }
4217 }
4218
4219 fn observe_route_open_accept(
4264 &self,
4265 ctx: &RouteCtx,
4266 module_id: &str,
4267 principal: &str,
4268 scope: &str,
4269 ) {
4270 self.route_outages.record_accepted(module_id);
4271 self.counters.increment_route_open_accepted(principal);
4272 info!(
4273 target: "control",
4274 principal,
4275 module_id,
4276 connection_id = ctx.connection_id.get(),
4277 scope = %scope,
4278 "route.open accepted"
4279 );
4280 }
4281
4282 fn supervised_absent_route_open_refusal_frame(
4283 &self,
4284 ctx: &RouteCtx,
4285 caller: &RouteOpenCaller,
4286 frame: &Frame,
4287 module_id: &str,
4288 code: &'static str,
4289 status: &crate::supervise::ModuleStatus,
4290 ) -> Result<Frame, RouterError> {
4291 self.counters.increment_route_open_refused(code);
4292 info!(
4293 target: "control",
4294 code,
4295 reason = "supervised_not_registered",
4296 module_id = ?module_id,
4297 connection_id = ctx.connection_id.get(),
4298 principal = %caller.principal,
4299 scope = %caller.scope,
4300 harness = %caller.harness,
4301 state = %status.state,
4302 enabled = status.enabled,
4303 live = status.live,
4304 "route.open refused"
4305 );
4306 self.route_outages
4310 .record_not_serving(module_id, "supervised_not_registered");
4311 let message = if let Some(pid) = status.waiting_for_previous_pid {
4312 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4313 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4314 })
4315 } else {
4316 format!(
4317 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4318 status.state, status.enabled, status.live
4319 )
4320 };
4321 control_error_frame(frame, code, message)
4322 }
4323
4324 async fn handle_route_open(
4325 &self,
4326 ctx: &RouteCtx,
4327 frame: Frame,
4328 request: RouteOpenRequest,
4329 timings: &mut RouteOpenTimings,
4330 ) -> Result<Vec<Frame>, RouterError> {
4331 let RouteOpenRequest {
4332 target,
4333 mut identity,
4334 consumer_identity,
4335 consumer_capabilities,
4336 role_versions,
4337 admission_facts,
4338 scope,
4339 } = request;
4340 let target_module_id = target_module_id(&target).to_string();
4341 timings.module_id.clone_from(&target_module_id);
4342 if self
4343 .registry
4344 .get_module_by_connection(ctx.connection_id)
4345 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4346 .is_some()
4347 {
4348 return Ok(vec![control_error_frame(
4349 &frame,
4350 "invalid_request",
4351 "module connections cannot open client routes",
4352 )?]);
4353 }
4354 debug!(
4355 connection_id = ctx.connection_id.get(),
4356 corr = frame.header.corr,
4357 module_id = %target_module_id,
4358 "handling route.open"
4359 );
4360
4361 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4366 let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4367
4368 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4374 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4375 self.observe_route_open_refusal(
4376 ctx,
4377 &caller,
4378 &target_module_id,
4379 "invalid_role_versions",
4380 error_codes::INVALID_REQUEST,
4381 );
4382 return Ok(vec![control_error_body_frame(
4383 &frame,
4384 ErrorBody {
4385 code: error_codes::INVALID_REQUEST.to_string(),
4386 message: error.to_string(),
4387 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4388 },
4389 )?]);
4390 }
4391
4392 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4393 if status.waiting_for_previous_pid.is_some() {
4394 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4395 ctx,
4396 &caller,
4397 &frame,
4398 &target_module_id,
4399 "module_warming",
4400 &status,
4401 )?]);
4402 }
4403 }
4404
4405 let Some(registration) = self
4424 .registry
4425 .get_module(&target_module_id)
4426 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4427 else {
4428 if let Some((status, warming)) =
4429 self.supervisor_status(&target_module_id, frame.header.corr)?
4430 {
4431 if status.protocol == ModuleProtocol::None {
4438 return Ok(vec![self.route_open_refusal_frame(
4439 ctx,
4440 &caller,
4441 &frame,
4442 &target_module_id,
4443 "protocol_none",
4444 error_codes::MODULE_NO_PROTOCOL,
4445 format!(
4446 "module_id '{target_module_id}' is declared protocol: none; \
4447 it speaks no subc wire and serves no routes"
4448 ),
4449 )?]);
4450 }
4451 let code = if warming {
4452 "module_warming"
4453 } else {
4454 "target_unavailable"
4455 };
4456 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4457 ctx,
4458 &caller,
4459 &frame,
4460 &target_module_id,
4461 code,
4462 &status,
4463 )?]);
4464 }
4465 if let Some(removed_ago_ms) =
4466 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4467 {
4468 return Ok(vec![self.route_open_refusal_frame(
4469 ctx,
4470 &caller,
4471 &frame,
4472 &target_module_id,
4473 "removed",
4474 error_codes::MODULE_REMOVED,
4475 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4476 )?]);
4477 }
4478 return Ok(vec![self.route_open_refusal_frame(
4479 ctx,
4480 &caller,
4481 &frame,
4482 &target_module_id,
4483 "not_registered",
4484 error_codes::UNKNOWN_MODULE,
4485 format!("module_id '{target_module_id}' is not registered"),
4486 )?]);
4487 };
4488
4489 if !registration.ready {
4493 self.counters
4494 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4495 info!(
4496 target: "control",
4497 code = error_codes::MODULE_WARMING,
4498 module_id = ?target_module_id,
4499 connection_id = ctx.connection_id.get(),
4500 principal = %caller.principal,
4501 scope = %caller.scope,
4502 harness = %caller.harness,
4503 reason = "declared_not_ready",
4504 "route.open refused"
4505 );
4506 self.route_outages
4509 .record_not_serving(&target_module_id, "declared_not_ready");
4510 return Ok(vec![control_error_body_frame(
4511 &frame,
4512 ErrorBody {
4513 code: error_codes::MODULE_WARMING.to_string(),
4514 message: format!(
4515 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4516 ),
4517 detail: Some(serde_json::json!({
4518 "reason": "declared_not_ready"
4519 })),
4520 },
4521 )?]);
4522 }
4523
4524 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4543 self.counters
4544 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4545 info!(
4546 target: "control",
4547 code = error_codes::MODULE_WARMING,
4548 module_id = ?target_module_id,
4549 connection_id = ctx.connection_id.get(),
4550 principal = %caller.principal,
4551 scope = %caller.scope,
4552 harness = %caller.harness,
4553 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4554 capability = %capability,
4555 "route.open refused"
4556 );
4557 return Ok(vec![control_error_body_frame(
4558 &frame,
4559 ErrorBody {
4560 code: error_codes::MODULE_WARMING.to_string(),
4561 message: format!(
4562 "module_id '{target_module_id}' requires capability '{capability}', \
4563 which no registered module provides; retry"
4564 ),
4565 detail: Some(serde_json::json!({
4566 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4567 "capability": capability,
4568 })),
4569 },
4570 )?]);
4571 }
4572
4573 if !target_has_required_role(&target, ®istration.manifest.provides) {
4574 return Ok(vec![self.route_open_refusal_frame(
4575 ctx,
4576 &caller,
4577 &frame,
4578 &target_module_id,
4579 "role_not_provided",
4580 "target_unavailable",
4581 format!("module_id '{target_module_id}' does not provide the requested target"),
4582 )?]);
4583 }
4584
4585 if registration.state != ChannelState::Active {
4586 return Ok(vec![self.route_open_refusal_frame(
4587 ctx,
4588 &caller,
4589 &frame,
4590 &target_module_id,
4591 "registration_not_active",
4592 "target_unavailable",
4593 format!("module_id '{target_module_id}' is not active"),
4594 )?]);
4595 }
4596
4597 if self
4598 .forwarding
4599 .module_is_draining(&target_module_id)
4600 .map_err(RouterError::Forwarding)?
4601 {
4602 return Ok(vec![self.route_open_refusal_frame(
4603 ctx,
4604 &caller,
4605 &frame,
4606 &target_module_id,
4607 "reloading",
4608 "module_reloading",
4609 format!("module_id '{target_module_id}' is reloading"),
4610 )?]);
4611 }
4612
4613 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4614 process_liveness.process_live(&target_module_id) == Some(false)
4615 }) {
4616 if process_liveness.process_replacing(&target_module_id) {
4623 return Ok(vec![self.route_open_refusal_frame(
4624 ctx,
4625 &caller,
4626 &frame,
4627 &target_module_id,
4628 "reloading",
4629 "module_reloading",
4630 format!("module_id '{target_module_id}' is reloading"),
4631 )?]);
4632 }
4633 return Ok(vec![self.route_open_refusal_frame(
4634 ctx,
4635 &caller,
4636 &frame,
4637 &target_module_id,
4638 "supervisor_not_live",
4639 "target_unavailable",
4640 format!("module_id '{target_module_id}' is not live"),
4641 )?]);
4642 }
4643
4644 if !self
4645 .forwarding
4646 .has_live_module_connection(&target_module_id)
4647 .map_err(RouterError::Forwarding)?
4648 {
4649 return Ok(vec![self.route_open_refusal_frame(
4650 ctx,
4651 &caller,
4652 &frame,
4653 &target_module_id,
4654 "no_forwarding_connection",
4655 "target_unavailable",
4656 format!("module_id '{target_module_id}' has no live forwarding connection"),
4657 )?]);
4658 }
4659
4660 if let Some(error) =
4661 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4662 {
4663 self.observe_route_open_refusal(
4664 ctx,
4665 &caller,
4666 &target_module_id,
4667 "op_not_allowed",
4668 "op_not_allowed",
4669 );
4670 return Ok(vec![error]);
4671 }
4672
4673 let principal =
4674 match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
4675 Ok(principal) => principal,
4676 Err(error) => {
4677 self.observe_route_open_refusal(
4678 ctx,
4679 &caller,
4680 &target_module_id,
4681 "bad_consumer_identity",
4682 "bad_consumer_identity",
4683 );
4684 return Ok(vec![error]);
4685 }
4686 };
4687 timings.principal = principal_label(&principal);
4688
4689 if let Principal::Reserved {
4693 module_id: opening_module_id,
4694 } = &principal
4695 {
4696 if let Some(opening_registration) = self
4697 .registry
4698 .get_module(opening_module_id)
4699 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4700 {
4701 if let Some(capability) =
4702 denied_capability(&opening_registration.manifest, ®istration.manifest)
4703 {
4704 warn!(
4705 opening_module_id,
4706 target_module_id,
4707 capability,
4708 "refusing route.open because an attested capability deny edge matches"
4709 );
4710 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4711 &frame,
4712 &target_module_id,
4713 "capability_deny_edge",
4714 "capability_forbidden",
4715 format!(
4716 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4717 ),
4718 )?]);
4719 }
4720 }
4721 }
4722
4723 if admission_facts.is_some() {
4724 let carrier_matches = matches!(
4725 &principal,
4726 Principal::Reserved { module_id }
4727 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4728 );
4729 if !carrier_matches {
4730 return Ok(vec![self.route_open_refusal_frame(
4731 ctx,
4732 &caller,
4733 &frame,
4734 &target_module_id,
4735 "admission_facts_carrier_not_permitted",
4736 "admission_facts_not_permitted",
4737 "admission facts may only be carried by the configured reserved module",
4738 )?]);
4739 }
4740
4741 let target_allowed = self
4742 .admission_facts_targets
4743 .as_ref()
4744 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4745 if !target_allowed {
4746 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4747 &frame,
4748 &target_module_id,
4749 "admission_facts_target_not_listed",
4750 "admission_facts_target_not_allowed",
4751 format!(
4752 "admission facts are not permitted for target module_id '{target_module_id}'"
4753 ),
4754 )?]);
4755 }
4756
4757 }
4761
4762 let (bound_scope, mut scope_stamp) = match &scope {
4768 None => (None, None),
4769 Some(selector) => {
4770 let owner_configured = match &selector.owner {
4771 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4777 _ => false,
4778 };
4779 let mut scopes = self
4780 .scopes
4781 .write()
4782 .unwrap_or_else(|poisoned| poisoned.into_inner());
4783 let admitted = scopes
4784 .admit(&principal, &target_module_id, selector, owner_configured)
4785 .and_then(|admission| {
4786 crate::scopes::check_target_flow_support(
4787 &admission.stamp,
4788 &target_module_id,
4789 registration.manifest.capabilities.as_ref(),
4790 )?;
4791 crate::scopes::check_target_agent_run_support(
4792 &admission.stamp,
4793 &target_module_id,
4794 registration.manifest.capabilities.as_ref(),
4795 )?;
4796 let bound = scopes.reserve_bind(
4797 &admission,
4798 &principal,
4799 ctx.connection_id,
4800 selector.hold_presence,
4801 );
4802 Ok((admission, bound))
4803 });
4804 match admitted {
4805 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4806 Err(refusal) => {
4807 return Ok(vec![self.route_open_refusal_frame(
4808 ctx,
4809 &caller,
4810 &frame,
4811 &target_module_id,
4812 refusal.code,
4813 refusal.code,
4814 refusal.message,
4815 )?]);
4816 }
4817 }
4818 }
4819 };
4820 if let Some(stamp) = &scope_stamp {
4823 caller.scope = route_open_scope_label(stamp);
4824 }
4825
4826 timings.enter(RouteOpenPhase::ProjectRoot);
4848 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4849 Ok(project_root) => project_root,
4850 Err(err) => {
4851 return Ok(vec![control_error_frame(
4852 &frame,
4853 "invalid_project_root",
4854 err.to_string(),
4855 )?])
4856 }
4857 };
4858 timings.enter(RouteOpenPhase::Admission);
4859 identity.project_root = project_root.as_path().to_path_buf();
4860
4861 let _concurrency_guard = match self
4872 .route_bind_concurrency
4873 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4874 {
4875 Ok(guard) => guard,
4876 Err(in_flight) => {
4877 return Ok(vec![self.route_open_target_capacity_refusal(
4878 ctx,
4879 &caller,
4880 &frame,
4881 &target_module_id,
4882 in_flight,
4883 )?]);
4884 }
4885 };
4886
4887 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4890 RouteBindAdmission::Admitted { guard, probe } => {
4891 if probe {
4892 info!(
4893 module_id = %target_module_id,
4894 connection_id = ctx.connection_id.get(),
4895 "route.bind breaker half-open: admitting one probe"
4896 );
4897 }
4898 guard
4899 }
4900 RouteBindAdmission::Refused {
4901 consecutive_timeouts,
4902 retry_in,
4903 probe_in_flight,
4904 } => {
4905 return Ok(vec![self.route_open_breaker_refusal_frame(
4906 ctx,
4907 &caller,
4908 &frame,
4909 &target_module_id,
4910 consecutive_timeouts,
4911 retry_in,
4912 probe_in_flight,
4913 )?]);
4914 }
4915 };
4916
4917 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4922 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4923 timings.enter(RouteOpenPhase::EgressReservation);
4924 let pending = match self
4925 .forwarding
4926 .begin_route_bind_relay_for(
4927 ctx.connection_id,
4928 ctx.egress.clone(),
4929 response_version(&frame),
4930 frame.header.corr,
4931 &target_module_id,
4932 principal.clone(),
4933 bound_scope,
4934 Some(project_root),
4935 relay_deadline,
4936 )
4937 .await
4938 {
4939 Ok(pending) => pending,
4940 Err(err) => {
4941 return Ok(vec![self.route_open_refusal_frame(
4942 ctx,
4943 &caller,
4944 &frame,
4945 &target_module_id,
4946 "relay_reservation_failed",
4947 forwarding_error_code(&err),
4948 err.to_string(),
4949 )?])
4950 }
4951 };
4952 timings.enter(RouteOpenPhase::Admission);
4953 let crate::forwarding::PendingRouteBindRelay {
4954 endpoint,
4955 module_sink,
4956 negotiated_ver,
4957 client_channel,
4958 client_epoch,
4959 module_channel,
4960 module_epoch,
4961 corr: relay_corr,
4962 receiver,
4963 } = pending;
4964 let mut reservation =
4965 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4966
4967 let relay_registration = self
4972 .registry
4973 .get_module_by_connection(endpoint.connection_id)
4974 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4975 let relay_capabilities = relay_registration
4976 .as_ref()
4977 .and_then(|registration| registration.manifest.capabilities.as_ref());
4978 if let Some(selector) = &scope {
4979 if let Err(refusal) = self
4982 .scopes
4983 .read()
4984 .unwrap_or_else(|poisoned| poisoned.into_inner())
4985 .check_bind_proof(&principal, selector)
4986 {
4987 reservation.release_and_disarm();
4988 return Ok(vec![self.route_open_refusal_frame(
4989 ctx,
4990 &caller,
4991 &frame,
4992 &target_module_id,
4993 refusal.code,
4994 refusal.code,
4995 refusal.message,
4996 )?]);
4997 }
4998 }
4999 if let Some(stamp) = scope_stamp
5000 .as_ref()
5001 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5002 {
5003 if let Err(refusal) = crate::scopes::check_target_flow_support(
5004 stamp,
5005 &target_module_id,
5006 relay_capabilities,
5007 )
5008 .and_then(|()| {
5009 crate::scopes::check_target_agent_run_support(
5010 stamp,
5011 &target_module_id,
5012 relay_capabilities,
5013 )
5014 }) {
5015 reservation.release_and_disarm();
5016 return Ok(vec![self.route_open_refusal_frame(
5017 ctx,
5018 &caller,
5019 &frame,
5020 &target_module_id,
5021 refusal.code,
5022 refusal.code,
5023 refusal.message,
5024 )?]);
5025 }
5026 }
5027
5028 if !relay_capabilities.is_some_and(|caps| {
5031 caps.provides
5032 .iter()
5033 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5034 }) {
5035 if let Some(stamp) = scope_stamp.as_mut() {
5036 stamp.workdir = None;
5037 stamp.project_id = None;
5038 }
5039 }
5040 if !relay_capabilities.is_some_and(|caps| {
5041 caps.provides
5042 .iter()
5043 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5044 }) {
5045 if let Some(stamp) = scope_stamp.as_mut() {
5046 stamp.attributes.on_behalf_of_agent = None;
5047 }
5048 }
5049 let call_mode = if relay_capabilities
5050 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5051 {
5052 Some(derive_call_mode(scope_stamp.as_ref()))
5053 } else {
5054 if let Some(stamp) = scope_stamp.as_mut() {
5055 stamp.expires_at_ms = None;
5056 }
5057 None
5058 };
5059
5060 debug!(
5061 connection_id = ctx.connection_id.get(),
5062 client_channel,
5063 client_epoch,
5064 module_channel,
5065 module_epoch,
5066 "reserved route handle pair"
5067 );
5068 let route_principal_label = principal_label(&principal);
5071 let relay = ModuleControlRequest::RouteBind {
5072 route_channel: module_channel,
5073 epoch: module_epoch,
5074 target,
5075 identity,
5076 principal: Some(principal),
5077 consumer_capabilities,
5078 role_versions,
5079 admission_facts,
5080 scope: scope_stamp,
5081 call_mode,
5082 };
5083 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
5084 RouterError::backend(
5085 0,
5086 frame.header.corr,
5087 format!("failed to encode route.bind request: {err}"),
5088 )
5089 })?;
5090 let relay_frame = Frame::build_with_version(
5091 negotiated_ver,
5092 FrameType::Request,
5093 control_flags(),
5094 0,
5095 0,
5096 relay_corr,
5097 relay_body,
5098 )
5099 .map_err(RouterError::FrameBuild)?;
5100
5101 timings.enter(RouteOpenPhase::BindSend);
5102 let sent = module_sink.send(relay_frame).await;
5103 timings.enter(RouteOpenPhase::Admission);
5104 if let Err(err) = sent {
5105 reservation.release_and_disarm();
5106 return Ok(vec![self.route_open_refusal_frame(
5107 ctx,
5108 &caller,
5109 &frame,
5110 &target_module_id,
5111 "relay_send_failed",
5112 "target_unavailable",
5113 err.to_string(),
5114 )?]);
5115 }
5116
5117 #[cfg(test)]
5118 if let Some(hold) = &self.route_bind_relay_sent_hold {
5119 hold.wait().await;
5123 hold.wait().await;
5124 }
5125
5126 if self
5131 .forwarding
5132 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5133 .map_err(RouterError::Forwarding)?
5134 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5135 {
5136 self.send_abandoned_route_bind_goodbye(
5137 &module_sink,
5138 negotiated_ver,
5139 module_channel,
5140 module_epoch,
5141 );
5142 }
5143
5144 timings.enter(RouteOpenPhase::BindAckWait);
5145 let outcome = timeout_at(relay_deadline, receiver).await;
5146 timings.enter(RouteOpenPhase::Admission);
5147 match outcome {
5148 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5149 reservation.disarm();
5150 if breaker.record_accepted() {
5151 info!(
5152 module_id = %target_module_id,
5153 "route.bind breaker closed: the probe was accepted"
5154 );
5155 }
5156 self.observe_route_open_accept(
5157 ctx,
5158 &target_module_id,
5159 &route_principal_label,
5160 &caller.scope,
5161 );
5162 Ok(Vec::new())
5163 }
5164 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5165 reservation.release_and_disarm();
5166 breaker.record_inconclusive();
5170 let scope_code = match body.code.as_str() {
5174 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
5175 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
5176 _ => None,
5177 };
5178 if let Some(code) = scope_code {
5179 self.observe_route_open_refusal(
5180 ctx,
5181 &caller,
5182 &target_module_id,
5183 "scope_changed_before_commit",
5184 code,
5185 );
5186 return Ok(vec![control_error_body_frame(&frame, body)?]);
5187 }
5188 self.counters
5189 .increment_route_open_refused("module_rejected");
5190 info!(
5191 target: "control",
5192 code = "module_rejected",
5193 module_code = ?body.code,
5194 module_id = ?target_module_id,
5195 connection_id = ctx.connection_id.get(),
5196 principal = %caller.principal,
5197 scope = %caller.scope,
5198 harness = %caller.harness,
5199 "route.open refused"
5200 );
5201 Ok(vec![control_error_body_frame(&frame, body)?])
5202 }
5203 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5204 reservation.release_and_disarm();
5205 breaker.record_inconclusive();
5206 tracing::warn!(
5213 module_id = %target_module_id,
5214 "route.bind relay abandoned: {message}"
5215 );
5216 Ok(vec![self.route_open_refusal_frame(
5217 ctx,
5218 &caller,
5219 &frame,
5220 &target_module_id,
5221 "relay_abandoned",
5222 "target_unavailable",
5223 message,
5224 )?])
5225 }
5226 Ok(Err(_)) => {
5227 reservation.release_and_disarm();
5228 breaker.record_inconclusive();
5229 Ok(vec![self.route_open_refusal_frame(
5230 ctx,
5231 &caller,
5232 &frame,
5233 &target_module_id,
5234 "relay_waiter_canceled",
5235 "target_unavailable",
5236 "route.bind relay waiter was canceled before the module responded",
5237 )?])
5238 }
5239 Err(_) => {
5240 reservation.release_and_disarm();
5241 if let Some(opened) = breaker.record_timeout(
5245 self.route_bind_breaker_threshold,
5246 self.route_bind_breaker_cooldown,
5247 ) {
5248 warn!(
5249 module_id = %target_module_id,
5250 consecutive_timeouts = opened.consecutive_timeouts,
5251 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5252 reopened_after_probe = opened.reopened_after_probe,
5253 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5254 );
5255 }
5256 tracing::warn!(
5264 module_id = %target_module_id,
5265 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5266 "route.bind relay timed out: module did not ack within budget"
5267 );
5268 Ok(vec![self.route_open_refusal_frame(
5269 ctx,
5270 &caller,
5271 &frame,
5272 &target_module_id,
5273 "relay_timed_out",
5274 "module_timeout",
5275 format!(
5276 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5277 route_bind_relay_timeout
5278 ),
5279 )?])
5280 }
5281 }
5282 }
5283
5284 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5285 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5286 snapshot: self.supervisor.spawn_snapshot(),
5287 };
5288 Ok(vec![control_response_body_frame(
5289 &frame,
5290 &response,
5291 "ClientControlResponse::SupervisorSpawnSnapshot",
5292 )?])
5293 }
5294
5295 fn handle_supervisor_spawn_subscribe(
5296 &self,
5297 ctx: &RouteCtx,
5298 frame: Frame,
5299 since: Option<SpawnCursor>,
5300 ) -> Result<Vec<Frame>, RouterError> {
5301 match self.supervisor.subscribe_spawns(
5302 ctx.connection_id,
5303 frame.header.corr,
5304 response_version(&frame),
5305 since,
5306 ctx.egress.clone(),
5307 ) {
5308 Ok(()) => Ok(Vec::new()),
5309 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5310 Ok(vec![control_error_body_frame(
5311 &frame,
5312 ErrorBody {
5313 code: "spawn_cursor_incarnation_mismatch".to_string(),
5314 message: "spawn cursor belongs to a different daemon incarnation"
5315 .to_string(),
5316 detail: Some(serde_json::json!({
5317 "current_daemon_incarnation": current
5318 })),
5319 },
5320 )?])
5321 }
5322 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5323 &frame,
5324 ErrorBody {
5325 code: "spawn_cursor_too_old".to_string(),
5326 message: "spawn cursor predates the retained event ring".to_string(),
5327 detail: Some(serde_json::json!({
5328 "oldest_retained_cursor": oldest
5329 })),
5330 },
5331 )?]),
5332 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5333 0,
5334 frame.header.corr,
5335 format!("failed to open supervisor spawn subscription: {error}"),
5336 )),
5337 }
5338 }
5339
5340 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5341 let generation = self
5342 .registry
5343 .generation()
5344 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5345 let mut modules = Vec::new();
5346 for module in self.supervisor.list() {
5347 let status = module.status_for_control("list").map_err(|err| {
5348 RouterError::backend(
5349 0,
5350 frame.header.corr,
5351 format!("failed to read supervisor status: {err}"),
5352 )
5353 })?;
5354 let (configured, _) = module.configuration().map_err(|err| {
5355 RouterError::backend(
5356 0,
5357 frame.header.corr,
5358 format!("failed to read module configuration: {err}"),
5359 )
5360 })?;
5361 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
5362 RouterError::backend(
5363 0,
5364 frame.header.corr,
5365 format!("failed to read retry time: {err}"),
5366 )
5367 })?;
5368 let image = module.running_image_agreement().await;
5370 let resources = Some(module.child_resource_usage());
5373 let mut verdict =
5374 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5375 verdict.spec = Some(launch_spec_agreement(
5376 &configured,
5377 status.process_alive,
5378 status.launch_spec_digest.as_ref(),
5379 ));
5380 let pending_reload = Some(verdict);
5381 let configured_enabled = self
5382 .capability_evaluator
5383 .configured_enabled(&status.module_id);
5384 modules.push(
5387 SupervisorEntry::new(
5388 status.module_id,
5389 status.state.to_string(),
5390 status.enabled,
5391 status.live,
5392 status.health.status,
5393 )
5394 .with_configured_enabled(configured_enabled)
5395 .with_next_retry_at_ms(next_retry_at_ms)
5396 .with_launch_nonce_env(Some(!cfg!(unix)))
5397 .with_launch_nonce_source(status.launch_nonce_source)
5398 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5399 .with_protocol(status.protocol)
5400 .with_pending_reload(pending_reload)
5401 .with_last_probe_ms(status.health.last_probe_ms)
5402 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5403 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5404 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5405 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5406 .with_restart_count(Some(status.restart_count))
5407 .with_max_restarts(Some(status.max_restarts))
5408 .with_lifetime_restarts(Some(status.lifetime_restarts))
5409 .with_spawn_generation(Some(status.spawn_generation))
5410 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5411 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5412 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5413 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5414 .with_resources(resources),
5415 );
5416 }
5417 let response = ClientControlResponse::SupervisorList {
5418 generation,
5419 modules,
5420 };
5421 Ok(vec![control_response_body_frame(
5422 &frame,
5423 &response,
5424 "ClientControlResponse::SupervisorList",
5425 )?])
5426 }
5427
5428 fn handle_supervisor_stderr_tail(
5429 &self,
5430 frame: Frame,
5431 module_id: String,
5432 max_lines: Option<u32>,
5433 max_bytes: Option<u32>,
5434 ) -> Result<Vec<Frame>, RouterError> {
5435 let Some(module) = self.supervisor.get(&module_id) else {
5436 return Ok(vec![control_error_frame(
5437 &frame,
5438 "unknown_module",
5439 format!("module_id '{module_id}' is not supervised"),
5440 )?]);
5441 };
5442
5443 let snapshot = module.stderr_tail(
5444 max_lines.map(|value| value as usize),
5445 max_bytes.map(|value| value as usize),
5446 );
5447
5448 let response = ClientControlResponse::SupervisorStderrTail {
5449 module_id,
5450 tail: StderrTail {
5451 capture: match snapshot.capture {
5452 CaptureState::Captured => StderrCaptureState::Captured,
5453 CaptureState::Incomplete { reason } => {
5454 StderrCaptureState::Incomplete { reason }
5455 }
5456 CaptureState::NotCaptured { reason } => {
5457 StderrCaptureState::NotCaptured { reason }
5458 }
5459 },
5460 entries: snapshot
5461 .entries
5462 .into_iter()
5463 .map(|entry| match entry {
5464 TailEntry::Line {
5465 text,
5466 truncated,
5467 at_ms,
5468 } => StderrTailEntry::Line {
5469 text,
5470 truncated,
5471 at_ms,
5472 },
5473 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5474 })
5475 .collect(),
5476 dropped_lines: snapshot.dropped_lines,
5477 },
5478 };
5479 Ok(vec![control_response_body_frame(
5480 &frame,
5481 &response,
5482 "ClientControlResponse::SupervisorStderrTail",
5483 )?])
5484 }
5485
5486 async fn handle_supervisor_terminals(
5487 &self,
5488 frame: Frame,
5489 module_id: String,
5490 ) -> Result<Vec<Frame>, RouterError> {
5491 let Some(module) = self.supervisor.get(&module_id) else {
5492 return Ok(vec![control_error_frame(
5493 &frame,
5494 "unknown_module",
5495 format!("module_id '{module_id}' is not supervised"),
5496 )?]);
5497 };
5498
5499 let terminals = module
5502 .read_durable_terminal_history()
5503 .await
5504 .map_err(|error| {
5505 RouterError::backend(
5506 0,
5507 frame.header.corr,
5508 format!("failed to read terminal history: {error}"),
5509 )
5510 })?;
5511 let response = ClientControlResponse::SupervisorTerminals {
5512 module_id,
5513 terminals,
5514 };
5515 Ok(vec![control_response_body_frame(
5516 &frame,
5517 &response,
5518 "ClientControlResponse::SupervisorTerminals",
5519 )?])
5520 }
5521
5522 fn handle_supervisor_routes(
5523 &self,
5524 frame: Frame,
5525 module_id: Option<String>,
5526 ) -> Result<Vec<Frame>, RouterError> {
5527 let modules = self
5528 .forwarding
5529 .route_census(module_id.as_deref())
5530 .map_err(RouterError::Forwarding)?
5531 .into_iter()
5532 .map(|(module_id, routes)| SupervisorRouteModule {
5533 module_id,
5534 routes: routes
5535 .into_iter()
5536 .map(|route| SupervisorRoute {
5537 consumer: match route.principal {
5538 Principal::Reserved { module_id } => {
5539 SupervisorRouteConsumer::Reserved { module_id }
5540 }
5541 Principal::Direct | Principal::Unverified => {
5542 SupervisorRouteConsumer::Direct {
5543 connection_id: route.goodbye_target.connection_id.get(),
5544 }
5545 }
5546 },
5547 age_ms: Instant::now()
5548 .saturating_duration_since(route.bound_at)
5549 .as_millis()
5550 .try_into()
5551 .unwrap_or(u64::MAX),
5552 draining: route.draining,
5553 drain_reason: route.drain_reason,
5554 })
5555 .collect(),
5556 })
5557 .collect();
5558 let response = ClientControlResponse::SupervisorRoutes { modules };
5559 Ok(vec![control_response_body_frame(
5560 &frame,
5561 &response,
5562 "ClientControlResponse::SupervisorRoutes",
5563 )?])
5564 }
5565
5566 async fn handle_supervisor_provenance(
5567 &self,
5568 frame: Frame,
5569 module_id: Option<String>,
5570 ) -> Result<Vec<Frame>, RouterError> {
5571 let mut selected = if let Some(module_id) = module_id {
5572 let Some(module) = self.supervisor.get(&module_id) else {
5573 return Ok(vec![control_error_frame(
5574 &frame,
5575 "unknown_module",
5576 format!("module_id '{module_id}' is not supervised"),
5577 )?]);
5578 };
5579 vec![module]
5580 } else {
5581 self.supervisor.list()
5582 };
5583
5584 let mut modules = Vec::with_capacity(selected.len());
5585 for module in selected.drain(..) {
5586 let (status, observed_image) = module
5587 .status_and_running_image_agreement()
5588 .await
5589 .map_err(|err| {
5590 RouterError::backend(
5591 0,
5592 frame.header.corr,
5593 format!("failed to read supervisor status: {err}"),
5594 )
5595 })?;
5596 let module_declared = self
5597 .registry
5598 .get_module(&status.module_id)
5599 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5600 .and_then(|registration| registration.manifest.provenance)
5601 .map(|build| ModuleDeclaredProvenance::Reported { build })
5602 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5603 #[cfg(test)]
5604 let running_image = match &self.provenance_probe_override {
5605 Some(result) => result.clone(),
5606 None => observed_image,
5607 };
5608 #[cfg(not(test))]
5609 let running_image = observed_image;
5610 modules.push(SupervisorModuleProvenance {
5611 module_id: status.module_id,
5612 module_declared,
5613 daemon_observed: SupervisorObservedProcess {
5614 launch_nonce_source: status.launch_nonce_source,
5615 pid: status.pid,
5616 spawned_at_ms: status.spawned_at_ms,
5617 spawned_from: status.spawned_from,
5618 running_image,
5619 },
5620 });
5621 }
5622 let daemon = SupervisorDaemonProvenance {
5623 daemon_build: self.daemon_provenance.build.clone(),
5624 daemon_observed: DaemonObservedProcess {
5625 pid: self.daemon_provenance.pid,
5626 started_at_ms: self
5627 .daemon_provenance
5628 .start_clock
5629 .map(|clock| clock.started_at_ms())
5630 .or(self.daemon_provenance.started_at_ms),
5631 running_image: self
5632 .daemon_provenance
5633 .probe
5634 .observe(
5635 self.daemon_provenance.pid,
5636 self.daemon_provenance.executable_path.as_deref(),
5637 self.daemon_provenance.executable_identity.clone(),
5638 self.daemon_provenance.process_start_time,
5639 )
5640 .await,
5641 },
5642 };
5643 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5644 Ok(vec![control_response_body_frame(
5645 &frame,
5646 &response,
5647 "ClientControlResponse::SupervisorProvenance",
5648 )?])
5649 }
5650
5651 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5652 self.refresh_capability_requirements();
5653 let generation = self
5654 .registry
5655 .generation()
5656 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5657 let modules = self
5658 .supervisor
5659 .list()
5660 .into_iter()
5661 .map(|module| {
5662 let status = module.status_for_control("health").map_err(|err| {
5663 RouterError::backend(
5664 0,
5665 frame.header.corr,
5666 format!("failed to read supervisor health: {err}"),
5667 )
5668 })?;
5669 let module_id = status.module_id;
5670 let capability_detail = self
5671 .capability_evaluator
5672 .required_problem_detail(&module_id);
5673 Ok(SupervisorHealthEntry {
5674 module_id,
5675 status: status.health.status,
5676 detail: append_capability_problem_detail(
5677 status.health.detail,
5678 capability_detail,
5679 ),
5680 metrics: status.health.metrics,
5681 consecutive_failures: status.health.consecutive_failures,
5682 late_answer_count: status.health.late_answer_count,
5683 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5684 last_action: status.health.last_action,
5685 last_action_ms: status.health.last_action_ms,
5686 last_probe_ms: status.health.last_probe_ms,
5687 })
5688 })
5689 .collect::<Result<Vec<_>, RouterError>>()?;
5690 let response = ClientControlResponse::SupervisorHealth {
5691 generation,
5692 modules,
5693 };
5694 Ok(vec![control_response_body_frame(
5695 &frame,
5696 &response,
5697 "ClientControlResponse::SupervisorHealth",
5698 )?])
5699 }
5700
5701 async fn handle_supervisor_restart(
5702 &self,
5703 frame: Frame,
5704 module_id: String,
5705 drain_timeout_ms: Option<u64>,
5706 requester: Principal,
5707 ) -> Result<Vec<Frame>, RouterError> {
5708 let operation_lock = self.supervisor.operation_lock();
5709 let _operation_guard = operation_lock.lock().await;
5710 let Some(module) = self.supervisor.get(&module_id) else {
5711 return Ok(vec![control_error_frame(
5712 &frame,
5713 "unknown_module",
5714 format!("module_id '{module_id}' is not supervised"),
5715 )?]);
5716 };
5717
5718 self.route_outages.mark_operator_action(&module_id);
5719 if let Err(err) = module
5720 .restart_requested(drain_timeout_ms, Some(requester))
5721 .await
5722 {
5723 self.route_outages
5724 .operator_action_ended_unrefused(&module_id);
5725 let (code, message) = match err {
5726 crate::supervise::SuperviseError::Disabled { .. } => {
5727 ("module_disabled", err.to_string())
5728 }
5729 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5730 ("swap_in_progress", err.to_string())
5731 }
5732 _ => (
5733 "target_unavailable",
5734 format!("failed to restart module_id '{module_id}': {err}"),
5735 ),
5736 };
5737 return Ok(vec![control_error_frame(&frame, code, message)?]);
5738 }
5739
5740 let response = ClientControlResponse::SupervisorAck {
5741 module_id,
5742 applied: true,
5743 };
5744 Ok(vec![control_response_body_frame(
5745 &frame,
5746 &response,
5747 "ClientControlResponse::SupervisorAck",
5748 )?])
5749 }
5750
5751 async fn handle_supervisor_swap(
5755 &self,
5756 frame: Frame,
5757 module_id: String,
5758 ready_timeout_ms: Option<u64>,
5759 requester: Principal,
5760 ) -> Result<Vec<Frame>, RouterError> {
5761 let module = {
5768 let operation_lock = self.supervisor.operation_lock();
5769 let _operation_guard = operation_lock.lock().await;
5770 self.supervisor.get(&module_id)
5771 };
5772 let Some(module) = module else {
5773 return Ok(vec![control_error_frame(
5774 &frame,
5775 "unknown_module",
5776 format!("module_id '{module_id}' is not supervised"),
5777 )?]);
5778 };
5779
5780 self.route_outages.mark_operator_action(&module_id);
5781 if let Err(err) = module
5782 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5783 .await
5784 {
5785 self.route_outages
5786 .operator_action_ended_unrefused(&module_id);
5787 use crate::supervise::SuperviseError;
5788 let message = err.to_string();
5789 let error = match err {
5790 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5791 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5792 code: "swap_refused".to_string(),
5793 message,
5794 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5795 },
5796 SuperviseError::SwapFailed {
5797 arm,
5798 candidate_exit,
5799 ..
5800 } => ErrorBody {
5801 code: "swap_failed".to_string(),
5802 message,
5803 detail: Some(serde_json::json!({
5804 "arm": arm.as_str(),
5805 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5806 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5807 })),
5808 },
5809 _ => ErrorBody::new(
5810 "target_unavailable",
5811 format!("failed to swap module_id '{module_id}': {message}"),
5812 ),
5813 };
5814 return Ok(vec![control_error_body_frame(&frame, error)?]);
5815 }
5816 self.route_outages
5820 .operator_action_ended_unrefused(&module_id);
5821
5822 let response = ClientControlResponse::SupervisorAck {
5823 module_id,
5824 applied: true,
5825 };
5826 Ok(vec![control_response_body_frame(
5827 &frame,
5828 &response,
5829 "ClientControlResponse::SupervisorAck",
5830 )?])
5831 }
5832
5833 async fn handle_supervisor_reload(
5834 &self,
5835 frame: Frame,
5836 module_id: String,
5837 requester: Principal,
5838 ) -> Result<Vec<Frame>, RouterError> {
5839 let operation_lock = self.supervisor.operation_lock();
5840 let _operation_guard = operation_lock.lock().await;
5841 let Some(module) = self.supervisor.get(&module_id) else {
5842 return Ok(vec![control_error_frame(
5843 &frame,
5844 "unknown_module",
5845 format!("module_id '{module_id}' is not supervised"),
5846 )?]);
5847 };
5848
5849 self.route_outages.mark_operator_action(&module_id);
5850 if let Err(err) = module.reload_requested(Some(requester)).await {
5851 self.route_outages
5852 .operator_action_ended_unrefused(&module_id);
5853 let (code, message) = match err {
5854 crate::supervise::SuperviseError::Disabled { .. } => {
5855 ("module_disabled", err.to_string())
5856 }
5857 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5858 ("swap_in_progress", err.to_string())
5859 }
5860 _ => (
5861 "reload_failed",
5862 format!("failed to reload module_id '{module_id}': {err}"),
5863 ),
5864 };
5865 return Ok(vec![control_error_frame(&frame, code, message)?]);
5866 }
5867
5868 let response = ClientControlResponse::SupervisorAck {
5869 module_id,
5870 applied: true,
5871 };
5872 Ok(vec![control_response_body_frame(
5873 &frame,
5874 &response,
5875 "ClientControlResponse::SupervisorAck",
5876 )?])
5877 }
5878
5879 async fn handle_supervisor_rescan(
5883 &self,
5884 frame: Frame,
5885 mode: RescanMode,
5886 requester: Principal,
5887 ) -> Result<Vec<Frame>, RouterError> {
5888 let preview = mode.previews();
5889 let Some(context) = self.rescan.clone() else {
5890 return Ok(vec![control_error_frame(
5891 &frame,
5892 "rescan_unavailable",
5893 "the daemon was not started with a reloadable config path".to_string(),
5894 )?]);
5895 };
5896
5897 let operation_lock = self.supervisor.operation_lock();
5898 let _operation_guard = operation_lock.lock().await;
5899 let loaded = match self.load_rescan_config(&mode, &context) {
5903 Ok(config) => config,
5904 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5905 };
5906 let Some((config, activity_config)) = loaded else {
5914 return Ok(vec![control_error_frame(
5915 &frame,
5916 "invalid_daemon_config",
5917 format!(
5918 "daemon config not found at {}; refusing to rescan (an absent config would \
5919 retire every supervised module)",
5920 context.config_path.display()
5921 ),
5922 )?]);
5923 };
5924 let (
5925 configured_port,
5926 storage_config,
5927 admission_facts_carrier_module_id,
5928 admission_facts_targets,
5929 scope_authority_owners,
5930 privileged_operator_prompt,
5931 modules,
5932 reserved_capabilities,
5933 ) = (
5934 config.port,
5935 config.storage,
5936 config.admission_facts_carrier_module_id,
5937 config.admission_facts_targets,
5938 config.scope_authority_owners,
5939 config.privileged_operator_prompt,
5940 config.modules,
5941 config.reserved_capabilities,
5942 );
5943
5944 let mut restart_required = Vec::new();
5953 for section in RestartRequiredSection::ALL {
5954 let changed = match section {
5955 RestartRequiredSection::Port => configured_port != context.configured_port,
5956 RestartRequiredSection::Storage => storage_config != context.storage_config,
5957 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5958 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5959 }
5960 RestartRequiredSection::AdmissionFactsTargets => {
5961 admission_facts_targets != context.admission_facts_targets
5962 }
5963 RestartRequiredSection::ScopeAuthorityOwners => {
5964 scope_authority_owners != context.scope_authority_owners
5965 }
5966 RestartRequiredSection::PrivilegedOperatorPrompt => {
5967 privileged_operator_prompt != context.privileged_operator_prompt
5968 }
5969 };
5970 if changed {
5971 restart_required.push(section.label().to_string());
5972 }
5973 }
5974 if !restart_required.is_empty() {
5975 warn!(
5976 config_path = %context.config_path.display(),
5977 sections = %restart_required.join(", "),
5978 "daemon config changed outside the modules section; restart the daemon to apply those changes"
5979 );
5980 }
5981
5982 for configured in &modules {
5983 if let Err(err) = validate_spec(&configured.module_spec()) {
5984 return Ok(vec![control_error_frame(
5985 &frame,
5986 "invalid_daemon_config",
5987 format!("supervisor rescan rejected daemon config: {err}"),
5988 )?]);
5989 }
5990 }
5991
5992 let configured_capabilities = modules
5993 .iter()
5994 .map(|module| (module.module_id.clone(), module.enabled))
5995 .collect::<Vec<_>>();
5996 let preview_capability_warnings = if preview {
5997 let (_, registrations) = self.runtime_capability_snapshot()?;
5998 let current_modules = self
5999 .supervisor
6000 .list()
6001 .into_iter()
6002 .map(|module| module.module_id().to_string())
6003 .collect::<BTreeSet<_>>();
6004 let resulting_modules = configured_capabilities.clone();
6005 let removed = current_modules
6006 .into_iter()
6007 .filter(|module_id| {
6008 !resulting_modules
6009 .iter()
6010 .any(|(configured_id, _)| configured_id == module_id)
6011 })
6012 .collect::<Vec<_>>();
6013 let disabled = match &mode {
6016 RescanMode::Candidate(_) => resulting_modules
6017 .iter()
6018 .filter(|(_, enabled)| !enabled)
6019 .map(|(module_id, _)| module_id.clone())
6020 .collect::<Vec<_>>(),
6021 _ => Vec::new(),
6022 };
6023 self.capability_evaluator.preview_warnings(
6024 resulting_modules,
6025 &removed,
6026 &disabled,
6027 ®istrations,
6028 )
6029 } else {
6030 Vec::new()
6031 };
6032 let reconciled = match &mode {
6033 RescanMode::Candidate(candidate) => self
6034 .preview_candidate(&context, modules, candidate)
6035 .map(|result| (result, Vec::new())),
6036 _ => {
6037 self.reconcile_supervised_modules(
6038 &context.supervisor,
6039 modules,
6040 preview,
6041 requester.clone(),
6042 )
6043 .await
6044 }
6045 };
6046 let (result, changed) = match reconciled {
6047 Ok(result) => result,
6048 Err(message) => {
6049 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6050 }
6051 };
6052 if !preview {
6053 if let Some(ledger) = &self.activity {
6054 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6055 ledger.config_changed(&result.added, &result.removed, &changed, requester);
6056 }
6057 if let Err(code) = self.configure_activity(activity_config).await {
6058 warn!(
6059 code,
6060 "activity retention could not be applied during rescan"
6061 );
6062 }
6063 self.capability_evaluator
6064 .configure(configured_capabilities, reserved_capabilities);
6065 self.capability_evaluator.wake_deadline_loop();
6066 self.refresh_capability_requirements();
6067 }
6068 let mut result = result;
6069 result.restart_required = restart_required;
6070 result.capability_warnings = preview_capability_warnings;
6071 let response = ClientControlResponse::SupervisorRescan { result };
6072 Ok(vec![control_response_body_frame(
6073 &frame,
6074 &response,
6075 "ClientControlResponse::SupervisorRescan",
6076 )?])
6077 }
6078
6079 async fn handle_supervisor_release_reserved(
6080 &self,
6081 frame: Frame,
6082 module_id: String,
6083 ) -> Result<Vec<Frame>, RouterError> {
6084 let Some(context) = self.rescan.clone() else {
6085 return Ok(vec![control_error_frame(
6086 &frame,
6087 "release_unavailable",
6088 "reserved-id release requires a daemon started with a reloadable config path",
6089 )?]);
6090 };
6091 let operation_lock = self.supervisor.operation_lock();
6092 let _operation_guard = operation_lock.lock().await;
6093 let loaded = match crate::daemon_config::load(&context.config_path) {
6094 Ok(Some(config)) => config,
6095 Ok(None) => {
6096 return Ok(vec![control_error_frame(
6097 &frame,
6098 "invalid_daemon_config",
6099 format!(
6100 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6101 context.config_path.display()
6102 ),
6103 )?])
6104 }
6105 Err(err) => {
6106 return Ok(vec![control_error_frame(
6107 &frame,
6108 "invalid_daemon_config",
6109 format!("unable to verify reserved-id release against daemon config: {err}"),
6110 )?])
6111 }
6112 };
6113 if loaded
6114 .modules
6115 .iter()
6116 .any(|configured| configured.module_id == module_id)
6117 {
6118 return Ok(vec![control_error_frame(
6119 &frame,
6120 "reserved_module_configured",
6121 format!(
6122 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6123 ),
6124 )?]);
6125 }
6126 if !self.supervisor.release_retained_reserved_gate(&module_id) {
6127 return Ok(vec![control_error_frame(
6128 &frame,
6129 "reserved_gate_not_retained",
6130 format!(
6131 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6132 ),
6133 )?]);
6134 }
6135
6136 let response = ClientControlResponse::SupervisorAck {
6137 module_id,
6138 applied: true,
6139 };
6140 Ok(vec![control_response_body_frame(
6141 &frame,
6142 &response,
6143 "ClientControlResponse::SupervisorAck",
6144 )?])
6145 }
6146
6147 fn plan_rescan(
6151 &self,
6152 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6153 ) -> Result<RescanPlan, String> {
6154 let mut current = BTreeMap::new();
6155 for module in self.supervisor.list() {
6156 let (spec, health) = module.configuration().map_err(|err| {
6157 format!(
6158 "failed to read configuration for module_id '{}': {err}",
6159 module.module_id()
6160 )
6161 })?;
6162 let enabled = module
6163 .status()
6164 .map_err(|err| {
6165 format!(
6166 "failed to read status for module_id '{}': {err}",
6167 module.module_id()
6168 )
6169 })?
6170 .enabled;
6171 current.insert(
6172 module.module_id().to_string(),
6173 (module, spec, health, enabled),
6174 );
6175 }
6176 let configured = configured_modules
6177 .into_iter()
6178 .map(|module| (module.module_id.clone(), module))
6179 .collect::<BTreeMap<_, _>>();
6180
6181 let added = configured
6182 .keys()
6183 .filter(|module_id| !current.contains_key(*module_id))
6184 .cloned()
6185 .collect::<Vec<_>>();
6186 let removed = current
6187 .keys()
6188 .filter(|module_id| !configured.contains_key(*module_id))
6189 .cloned()
6190 .collect::<Vec<_>>();
6191 let mut changed_pending_reload = Vec::new();
6192 let mut configuration_changes = BTreeSet::new();
6193 let mut enabled_changes = BTreeSet::new();
6194 let mut unchanged = 0_u32;
6195
6196 for (module_id, configured_module) in &configured {
6197 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6198 else {
6199 continue;
6200 };
6201 let launch_changed = *current_spec != configured_module.module_spec();
6205 let configuration_changed =
6206 launch_changed || *current_health != configured_module.health;
6207 let enabled_changed = *current_enabled != configured_module.enabled;
6208 if configuration_changed {
6209 configuration_changes.insert(module_id.clone());
6210 }
6211 if launch_changed {
6212 changed_pending_reload.push(module_id.clone());
6213 }
6214 if enabled_changed {
6215 enabled_changes.insert(module_id.clone());
6216 }
6217 if !configuration_changed && !enabled_changed {
6218 unchanged = unchanged.saturating_add(1);
6219 }
6220 }
6221 Ok(RescanPlan {
6222 current,
6223 configured,
6224 added,
6225 removed,
6226 changed_pending_reload,
6227 configuration_changes,
6228 enabled_changes,
6229 unchanged,
6230 })
6231 }
6232
6233 async fn reconcile_supervised_modules(
6242 &self,
6243 supervisor: &Supervisor,
6244 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6245 preview: bool,
6246 requester: Principal,
6247 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6248 let plan = self.plan_rescan(configured_modules)?;
6249
6250 if preview {
6256 return Ok((plan.preview_result(), Vec::new()));
6257 }
6258 let changed = plan
6259 .configuration_changes
6260 .union(&plan.enabled_changes)
6261 .cloned()
6262 .collect::<Vec<_>>();
6263 let RescanPlan {
6264 current,
6265 configured,
6266 added,
6267 removed,
6268 changed_pending_reload,
6269 configuration_changes,
6270 enabled_changes,
6271 unchanged,
6272 } = plan;
6273 if !(added.is_empty()
6278 && removed.is_empty()
6279 && configuration_changes.is_empty()
6280 && enabled_changes.is_empty())
6281 {
6282 if let Some(context) = &self.rescan {
6283 context
6284 .generation
6285 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6286 }
6287 }
6288
6289 for module_id in &removed {
6290 let module = ¤t
6291 .get(module_id)
6292 .expect("removed module came from current supervisor state")
6293 .0;
6294 module
6295 .retire_requested(Some(requester.clone()))
6296 .await
6297 .map_err(|err| {
6298 format!("failed to retire module_id '{module_id}' during rescan: {err}")
6299 })?;
6300 self.supervisor.record_rescan_removal(module_id);
6334 self.supervisor.retire(module_id);
6335 self.route_outages.forget(module_id);
6336 }
6337
6338 for module_id in configured.keys() {
6339 let Some((module, _, _, _)) = current.get(module_id) else {
6340 continue;
6341 };
6342 let configured_module = configured
6343 .get(module_id)
6344 .expect("configured module id came from configured map");
6345 if configuration_changes.contains(module_id) {
6346 module
6347 .update_configuration(
6348 configured_module.module_spec(),
6349 configured_module.health.clone(),
6350 configured_module.drain_timeout_ms,
6351 )
6352 .await
6353 .map_err(|err| {
6354 format!(
6355 "failed to update module_id '{module_id}' configuration during rescan: {err}"
6356 )
6357 })?;
6358 }
6359 if enabled_changes.contains(module_id) {
6360 self.route_outages.mark_operator_action(module_id);
6363 module
6364 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6365 .await
6366 .map_err(|err| {
6367 self.route_outages.operator_action_ended_unrefused(module_id);
6368 format!(
6369 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6370 configured_module.enabled
6371 )
6372 })?;
6373 }
6374 }
6375
6376 for module_id in &added {
6377 let configured_module = configured
6378 .get(module_id)
6379 .expect("added module id came from configured map");
6380 supervisor
6381 .supervise_configured_requested(
6382 configured_module.module_spec(),
6383 configured_module.enabled,
6384 configured_module.health.clone(),
6385 configured_module.drain_timeout_ms,
6386 configured_module.restart,
6387 Some(requester.clone()),
6388 )
6389 .map_err(|err| {
6390 format!("failed to add module_id '{module_id}' during rescan: {err}")
6391 })?;
6392 }
6393
6394 Ok((
6395 SupervisorRescanResult {
6396 added,
6397 removed,
6398 changed_pending_reload,
6399 enabled_changes: enabled_changes.iter().cloned().collect(),
6400 unchanged,
6401 preview: false,
6402 restart_required: Vec::new(),
6405 capability_warnings: Vec::new(),
6406 module_actions: Vec::new(),
6407 earlier_pending: Vec::new(),
6408 expect: None,
6409 },
6410 changed,
6411 ))
6412 }
6413
6414 async fn handle_supervisor_set_enabled(
6415 &self,
6416 frame: Frame,
6417 module_id: String,
6418 enabled: bool,
6419 requester: Principal,
6420 ) -> Result<Vec<Frame>, RouterError> {
6421 let operation_lock = self.supervisor.operation_lock();
6422 let _operation_guard = operation_lock.lock().await;
6423 let Some(module) = self.supervisor.get(&module_id) else {
6424 return Ok(vec![control_error_frame(
6425 &frame,
6426 "unknown_module",
6427 format!("module_id '{module_id}' is not supervised"),
6428 )?]);
6429 };
6430
6431 self.route_outages.mark_operator_action(&module_id);
6434 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6435 Ok(applied) => applied,
6436 Err(err) => {
6437 self.route_outages
6438 .operator_action_ended_unrefused(&module_id);
6439 return Ok(vec![control_error_frame(
6440 &frame,
6441 "target_unavailable",
6442 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6443 )?]);
6444 }
6445 };
6446 if !applied {
6447 self.route_outages
6450 .operator_action_ended_unrefused(&module_id);
6451 }
6452
6453 self.capability_evaluator.wake_deadline_loop();
6454 self.refresh_capability_requirements();
6455 let response = ClientControlResponse::SupervisorAck { module_id, applied };
6456 Ok(vec![control_response_body_frame(
6457 &frame,
6458 &response,
6459 "ClientControlResponse::SupervisorAck",
6460 )?])
6461 }
6462
6463 async fn handle_supervisor_health_probe(
6464 &self,
6465 frame: Frame,
6466 module_id: String,
6467 ) -> Result<Vec<Frame>, RouterError> {
6468 self.refresh_capability_requirements();
6469 let Some(registration) = self
6470 .registry
6471 .get_module(&module_id)
6472 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6473 else {
6474 return Ok(vec![control_error_frame(
6475 &frame,
6476 "unknown_module",
6477 format!("module_id '{module_id}' is not registered"),
6478 )?]);
6479 };
6480
6481 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6494 {
6495 return Ok(vec![control_error_frame(
6496 &frame,
6497 "health_not_advertised",
6498 format!("module_id '{module_id}' did not advertise health.check"),
6499 )?]);
6500 }
6501
6502 let deadline = Instant::now() + self.health_probe_timeout;
6503 let pending = match self.forwarding.begin_module_control_rpc_for(
6504 &module_id,
6505 MODULE_CONTROL_OP_HEALTH_CHECK,
6506 deadline,
6507 ) {
6508 Ok(pending) => pending,
6509 Err(err) => {
6510 return Ok(vec![control_error_frame(
6511 &frame,
6512 forwarding_error_code(&err),
6513 err.to_string(),
6514 )?])
6515 }
6516 };
6517
6518 let PendingModuleControlRpc {
6519 endpoint,
6520 module_sink,
6521 negotiated_ver,
6522 corr: probe_corr,
6523 receiver,
6524 } = pending;
6525 let mut guard =
6526 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6527 let probe_body =
6528 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6529 RouterError::backend(
6530 0,
6531 frame.header.corr,
6532 format!("failed to encode health.check request: {err}"),
6533 )
6534 })?;
6535 let probe_frame = Frame::build_with_version(
6536 negotiated_ver,
6537 FrameType::Request,
6538 control_flags(),
6539 0,
6540 0,
6541 probe_corr,
6542 probe_body,
6543 )
6544 .map_err(RouterError::FrameBuild)?;
6545
6546 match timeout_at(deadline, module_sink.send(probe_frame)).await {
6549 Ok(Ok(())) => {}
6550 Ok(Err(err)) => {
6551 return Ok(vec![control_error_frame(
6552 &frame,
6553 "target_unavailable",
6554 err.to_string(),
6555 )?]);
6556 }
6557 Err(_) => {
6558 return Ok(vec![control_error_frame(
6559 &frame,
6560 "module_timeout",
6561 format!(
6562 "module_id '{module_id}' did not answer health.check within {:?}",
6563 self.health_probe_timeout
6564 ),
6565 )?]);
6566 }
6567 }
6568
6569 match timeout_at(deadline, receiver).await {
6570 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6571 guard.disarm();
6572 let Some(report) = response.health_report() else {
6573 return Ok(vec![control_error_frame(
6574 &frame,
6575 "invalid_control_body",
6576 "health.check RPC returned a non-health response",
6577 )?]);
6578 };
6579 let HealthReport {
6584 status,
6585 detail,
6586 metrics,
6587 } = report;
6588 let capability_detail = self
6589 .capability_evaluator
6590 .required_problem_detail(&module_id);
6591 let response = ClientControlResponse::SupervisorHealthProbe {
6592 module_id,
6593 status,
6594 detail: append_capability_problem_detail(detail, capability_detail),
6595 metrics,
6596 };
6597 Ok(vec![control_response_body_frame(
6598 &frame,
6599 &response,
6600 "ClientControlResponse::SupervisorHealthProbe",
6601 )?])
6602 }
6603 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6604 guard.disarm();
6605 Ok(vec![control_error_body_frame(&frame, body)?])
6606 }
6607 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6608 guard.disarm();
6609 Ok(vec![control_error_frame(
6610 &frame,
6611 "target_unavailable",
6612 message,
6613 )?])
6614 }
6615 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6616 guard.disarm();
6617 Ok(vec![control_error_frame(
6618 &frame,
6619 "invalid_control_body",
6620 message,
6621 )?])
6622 }
6623 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6624 guard.disarm();
6625 Ok(vec![control_error_frame(
6626 &frame,
6627 "invalid_control_body",
6628 format!("expected module-control op '{expected}', got '{actual}'"),
6629 )?])
6630 }
6631 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6632 guard.disarm();
6633 Ok(vec![control_error_frame(
6634 &frame,
6635 "module_timeout",
6636 format!(
6637 "module_id '{module_id}' answered health.check after {:?}",
6638 self.health_probe_timeout
6639 ),
6640 )?])
6641 }
6642 Ok(Err(_)) => Ok(vec![control_error_frame(
6643 &frame,
6644 "target_unavailable",
6645 "health.check waiter was canceled before the module responded",
6646 )?]),
6647 Err(_) => Ok(vec![control_error_frame(
6648 &frame,
6649 "module_timeout",
6650 format!(
6651 "module_id '{module_id}' did not answer health.check within {:?}",
6652 self.health_probe_timeout
6653 ),
6654 )?]),
6655 }
6656 }
6657
6658 fn supervisor_status(
6659 &self,
6660 module_id: &str,
6661 corr: u64,
6662 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6663 self.supervisor
6664 .get(module_id)
6665 .map(|module| {
6666 let warming = module.is_warming_for_control("status").map_err(|err| {
6667 RouterError::backend(
6668 0,
6669 corr,
6670 format!(
6671 "failed to read supervisor warming state for module_id '{module_id}': {err}"
6672 ),
6673 )
6674 })?;
6675 module.status_for_control("status").map_err(|err| {
6676 RouterError::backend(
6677 0,
6678 corr,
6679 format!(
6680 "failed to read supervisor status for module_id '{module_id}': {err}"
6681 ),
6682 )
6683 }).map(|status| (status, warming))
6684 })
6685 .transpose()
6686 }
6687
6688 fn guard_module_control_op(
6689 &self,
6690 frame: &Frame,
6691 module_id: &str,
6692 op: &str,
6693 ) -> Result<Option<Frame>, RouterError> {
6694 if self.module_grants_op(module_id, op, frame.header.corr)? {
6695 return Ok(None);
6696 }
6697
6698 Ok(Some(control_error_frame(
6699 frame,
6700 "op_not_allowed",
6701 format!("module_id '{module_id}' did not grant control op '{op}'"),
6702 )?))
6703 }
6704
6705 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6706 let Some(registration) = self
6707 .registry
6708 .get_module(module_id)
6709 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6710 else {
6711 return Ok(false);
6712 };
6713 Ok(module_registration_grants_op(®istration.control_ops, op))
6714 }
6715
6716 fn handle_status_update(
6717 &self,
6718 endpoint: ModuleEndpointId,
6719 frame: Frame,
6720 ) -> Result<Vec<Frame>, RouterError> {
6721 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6722 Ok(update) => update,
6723 Err(err) => {
6724 if is_known_module_push_op(&frame.body) {
6729 return Ok(vec![control_error_frame(
6730 &frame,
6731 "invalid_control_body",
6732 format!("malformed module control push body: {err}"),
6733 )?]);
6734 }
6735 return Ok(Vec::new());
6736 }
6737 };
6738
6739 match update {
6740 ModuleControlPush::RouteStatus {
6741 route_channel,
6742 route_epoch,
6743 status,
6744 } => {
6745 self.forwarding
6746 .cache_status(endpoint, route_channel, route_epoch, status)
6747 .map_err(RouterError::Forwarding)?;
6748 }
6749 }
6750 Ok(Vec::new())
6751 }
6752
6753 fn handle_route_poll(
6754 &self,
6755 ctx: &RouteCtx,
6756 frame: Frame,
6757 route_channel: u16,
6758 route_epoch: u32,
6759 kind: PollKind,
6760 ) -> Result<Vec<Frame>, RouterError> {
6761 let snapshot = self
6762 .forwarding
6763 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6764 .map_err(RouterError::Forwarding)?;
6765 let response = match (kind, snapshot) {
6766 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6767 ClientControlResponse::RoutePoll {
6768 route_channel,
6769 route_epoch,
6770 status,
6771 live: None,
6772 }
6773 }
6774 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6775 route_channel,
6776 route_epoch,
6777 status: None,
6778 live: None,
6779 },
6780 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6781 let live = self
6807 .process_liveness
6808 .as_ref()
6809 .and_then(|source| source.process_live(&module_id))
6810 .unwrap_or(true);
6811 ClientControlResponse::RoutePoll {
6812 route_channel,
6813 route_epoch,
6814 status: None,
6815 live: Some(live),
6816 }
6817 }
6818 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6819 route_channel,
6820 route_epoch,
6821 status: None,
6822 live: Some(false),
6823 },
6824 };
6825
6826 Ok(vec![control_response_body_frame(
6827 &frame,
6828 &response,
6829 "ClientControlResponse::RoutePoll",
6830 )?])
6831 }
6832
6833 pub(crate) fn observe_module_control_completion(
6834 &self,
6835 completion: ModuleControlRpcCompletion,
6836 ) -> bool {
6837 match completion {
6838 ModuleControlRpcCompletion::Unknown => false,
6839 ModuleControlRpcCompletion::Settled => true,
6840 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6841 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6842 info!(
6843 module_id = %module_id,
6844 latency_ms,
6845 "late health.check answer proves the module is alive"
6846 );
6847 match self
6848 .supervisor
6849 .record_late_health_answer(&module_id, latency_ms)
6850 {
6851 Ok(true) => {}
6852 Ok(false) => debug!(
6853 module_id = %module_id,
6854 latency_ms,
6855 "late health.check answer has no active supervisor snapshot"
6856 ),
6857 Err(err) => warn!(
6858 module_id = %module_id,
6859 latency_ms,
6860 error = %err,
6861 "failed to record late health.check answer"
6862 ),
6863 }
6864 true
6865 }
6866 }
6867 }
6868
6869 fn refuse_to_end_module_connection_for_a_client(
6890 &self,
6891 module_connection_id: ConnectionId,
6892 corr: u64,
6893 err: ForwardingError,
6894 ) -> Result<(), RouterError> {
6895 if let ForwardingError::ConnectionClosing { connection_id } = err {
6896 if connection_id != module_connection_id {
6897 warn!(
6898 module_connection_id = module_connection_id.get(),
6899 client_connection_id = connection_id.get(),
6900 corr,
6901 "dropping a route.bind response for a closing client; the module connection keeps serving"
6902 );
6903 return Ok(());
6904 }
6905 }
6906 Err(RouterError::Forwarding(err))
6907 }
6908
6909 fn handle_module_relay_response(
6910 &self,
6911 connection_id: ConnectionId,
6912 frame: Frame,
6913 ) -> Result<Vec<Frame>, RouterError> {
6914 let mut secondary_error = None;
6915 let outcome = match frame.header.ty {
6916 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6917 Ok(probe) if probe.op == "route.bind" => {
6918 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6919 Ok(ModuleControlResponse::RouteBindAck {}) => {
6920 RouteBindRelayOutcome::Accepted
6921 }
6922 Ok(other) => {
6923 let message =
6924 format!("route.bind response carried unexpected body: {other:?}");
6925 secondary_error = Some(control_error_frame(
6926 &frame,
6927 "invalid_control_body",
6928 message.clone(),
6929 )?);
6930 RouteBindRelayOutcome::ModuleGone(message)
6931 }
6932 Err(err) => {
6933 let message = format!("malformed route.bind response body: {err}");
6934 secondary_error = Some(control_error_frame(
6935 &frame,
6936 "invalid_control_body",
6937 message.clone(),
6938 )?);
6939 RouteBindRelayOutcome::ModuleGone(message)
6940 }
6941 }
6942 }
6943 Ok(probe) => {
6944 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6945 {
6946 Ok(response) => ModuleControlRpcOutcome::Response(response),
6947 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6948 "malformed {} response body: {err}",
6949 probe.op
6950 )),
6951 };
6952 let completion = self
6953 .forwarding
6954 .complete_module_control_rpc(
6955 connection_id,
6956 frame.header.corr,
6957 Some(&probe.op),
6958 outcome,
6959 )
6960 .map_err(RouterError::Forwarding)?;
6961 if !self.observe_module_control_completion(completion) {
6962 debug!(
6963 connection_id = connection_id.get(),
6964 corr = frame.header.corr,
6965 op = %probe.op,
6966 "dropping late or unknown module-control RPC response"
6967 );
6968 }
6969 return Ok(Vec::new());
6970 }
6971 Err(err) => {
6972 if let Some(expected_op) = self
6973 .forwarding
6974 .pending_module_control_op(connection_id, frame.header.corr)
6975 .map_err(RouterError::Forwarding)?
6976 {
6977 let completion = self
6978 .forwarding
6979 .complete_module_control_rpc(
6980 connection_id,
6981 frame.header.corr,
6982 None,
6983 ModuleControlRpcOutcome::MalformedResponse(format!(
6984 "malformed {expected_op} response body: {err}"
6985 )),
6986 )
6987 .map_err(RouterError::Forwarding)?;
6988 if !self.observe_module_control_completion(completion) {
6989 debug!(
6990 connection_id = connection_id.get(),
6991 corr = frame.header.corr,
6992 "dropping late malformed module-control RPC response"
6993 );
6994 }
6995 return Ok(Vec::new());
6996 }
6997 let message = format!("malformed route.bind response body: {err}");
6998 secondary_error = Some(control_error_frame(
6999 &frame,
7000 "invalid_control_body",
7001 message.clone(),
7002 )?);
7003 RouteBindRelayOutcome::ModuleGone(message)
7004 }
7005 },
7006 FrameType::Error => {
7007 if self
7008 .forwarding
7009 .pending_module_control_op(connection_id, frame.header.corr)
7010 .map_err(RouterError::Forwarding)?
7011 .is_some()
7012 {
7013 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7014 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7015 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7016 "malformed module-control ERROR body: {err}"
7017 )),
7018 };
7019 let completion = self
7020 .forwarding
7021 .complete_module_control_rpc(
7022 connection_id,
7023 frame.header.corr,
7024 None,
7025 outcome,
7026 )
7027 .map_err(RouterError::Forwarding)?;
7028 if !self.observe_module_control_completion(completion) {
7029 debug!(
7030 connection_id = connection_id.get(),
7031 corr = frame.header.corr,
7032 "dropping late or unknown module-control RPC error"
7033 );
7034 }
7035 return Ok(Vec::new());
7036 }
7037 match serde_json::from_slice::<ErrorBody>(&frame.body) {
7038 Ok(body) => RouteBindRelayOutcome::Rejected(body),
7039 Err(err) => {
7040 let message = format!("malformed route.bind ERROR body: {err}");
7041 secondary_error = Some(control_error_frame(
7042 &frame,
7043 "invalid_control_body",
7044 message.clone(),
7045 )?);
7046 RouteBindRelayOutcome::ModuleGone(message)
7047 }
7048 }
7049 }
7050 ty => {
7051 return Ok(vec![control_error_frame(
7052 &frame,
7053 "unsupported_control_frame",
7054 format!("unsupported module channel-0 frame {ty:?}"),
7055 )?])
7056 }
7057 };
7058
7059 let settled =
7060 self.forwarding
7061 .complete_pending_relay(connection_id, frame.header.corr, outcome);
7062 let completion = match settled {
7063 Ok(completion) => completion,
7064 Err(err) => {
7065 self.refuse_to_end_module_connection_for_a_client(
7066 connection_id,
7067 frame.header.corr,
7068 err,
7069 )?;
7070 return Ok(secondary_error.into_iter().collect());
7071 }
7072 };
7073 if let Some(target) = completion.abandoned.as_ref() {
7074 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7075 }
7076 if !completion.settled {
7077 debug!(
7078 connection_id = connection_id.get(),
7079 corr = frame.header.corr,
7080 frame_type = ?frame.header.ty,
7081 "dropping late or unknown route.bind relay response"
7082 );
7083 }
7084 Ok(secondary_error.into_iter().collect())
7085 }
7086
7087 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7088 debug!(connection_id = connection_id.get(), "handling GOODBYE");
7089 self.cleanup_connection_with_end_reason(
7093 connection_id,
7094 RegistrationEndReason::ExplicitGoodbye,
7095 )
7096 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7097 Ok(Vec::new())
7098 }
7099}
7100
7101impl Default for ControlHandler {
7102 fn default() -> Self {
7103 Self::new(Arc::new(Registry::default()))
7104 }
7105}
7106
7107impl crate::supervise::SwapPromotionObserver for ControlHandler {
7108 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7109 self.apply_registration_capabilities(registration);
7110 }
7111}
7112
7113fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7114 CapabilityRequirementStatus {
7115 consumer: status.consumer,
7116 capability: status.capability,
7117 need: match status.need {
7118 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7119 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7120 },
7121 verdict: status.verdict.as_str().to_string(),
7122 episode_seq: status.episode_seq,
7123 config_satisfiable: status.config_satisfiable,
7124 runtime_available: status.runtime_available,
7125 detail: status.detail,
7126 }
7127}
7128
7129fn append_capability_problem_detail(
7130 detail: Option<String>,
7131 capability_detail: Option<String>,
7132) -> Option<String> {
7133 match (detail, capability_detail) {
7134 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7135 (Some(detail), None) => Some(detail),
7136 (None, Some(capability_detail)) => Some(capability_detail),
7137 (None, None) => None,
7138 }
7139}
7140
7141fn subc_ops() -> Vec<String> {
7142 SUBC_CONTROL_OPS
7143 .iter()
7144 .map(|op| (*op).to_string())
7145 .collect()
7146}
7147
7148fn module_subc_ops() -> Vec<String> {
7149 SUBC_CONTROL_OPS
7150 .iter()
7151 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7152 .map(|op| (*op).to_string())
7153 .collect()
7154}
7155
7156#[cfg(test)]
7157fn module_baseline_control_ops() -> Vec<String> {
7158 MODULE_BASELINE_CONTROL_OPS
7159 .iter()
7160 .map(|op| (*op).to_string())
7161 .collect()
7162}
7163
7164fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7165 let mut seen = HashSet::new();
7166 let mut effective = Vec::new();
7167 for op in MODULE_BASELINE_CONTROL_OPS {
7168 if seen.insert((*op).to_string()) {
7169 effective.push((*op).to_string());
7170 }
7171 }
7172 for op in declared.unwrap_or_default() {
7173 if seen.insert(op.clone()) {
7174 effective.push(op);
7175 }
7176 }
7177 effective
7178}
7179
7180fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7181 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7182}
7183
7184fn target_module_id(target: &RouteTarget) -> &str {
7185 match target {
7186 RouteTarget::ToolProvider { module_id }
7187 | RouteTarget::ManagementSurface { module_id }
7188 | RouteTarget::InternalService { module_id, .. } => module_id,
7189 }
7190}
7191
7192fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7193 roles.iter().any(|role| match (target, role) {
7194 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7195 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7196 (
7197 RouteTarget::InternalService { service_id, .. },
7198 ProviderRole::InternalService {
7199 service_id: provided,
7200 ..
7201 },
7202 ) => service_id == provided,
7203 _ => false,
7204 })
7205}
7206
7207fn is_routable_role(role: &ProviderRole) -> bool {
7208 matches!(
7209 role,
7210 ProviderRole::ToolProvider { .. }
7211 | ProviderRole::ManagementSurface { .. }
7212 | ProviderRole::InternalService { .. }
7213 )
7214}
7215
7216#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7217enum ControlRequestBodyError {
7218 UnknownOp,
7219 InvalidBody,
7220}
7221
7222#[derive(Debug, Deserialize)]
7223struct ControlOpProbe {
7224 op: String,
7225}
7226
7227const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7230
7231fn is_known_module_push_op(body: &[u8]) -> bool {
7232 serde_json::from_slice::<ControlOpProbe>(body)
7233 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7234 .unwrap_or(false)
7235}
7236
7237fn is_known_module_request_op(body: &[u8]) -> bool {
7238 serde_json::from_slice::<ControlOpProbe>(body)
7239 .map(|probe| is_module_to_subc_op(&probe.op))
7240 .unwrap_or(false)
7241}
7242
7243fn is_module_to_subc_op(op: &str) -> bool {
7244 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7245}
7246
7247fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7248 debug!(
7249 op = %op,
7250 connection_id = connection_id.get(),
7251 corr,
7252 "control dispatch"
7253 );
7254}
7255
7256fn log_slow_control_dispatch(
7257 dispatch_started_at: Option<StdInstant>,
7258 op: &'static str,
7259 connection_id: ConnectionId,
7260 corr: u64,
7261 route_open_timings: Option<&RouteOpenTimings>,
7262) {
7263 let Some(dispatch_started_at) = dispatch_started_at else {
7264 return;
7265 };
7266 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7268 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7269 if let Some(timings) = route_open_timings {
7270 warn!(
7271 op = %op,
7272 connection_id = connection_id.get(),
7273 corr,
7274 elapsed_ms = elapsed.as_millis() as u64,
7275 module_id = ?timings.module_id,
7276 principal = %timings.principal,
7277 admission_ms = timings.admission.as_millis() as u64,
7278 project_root_ms = timings.project_root.as_millis() as u64,
7279 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7280 bind_send_ms = timings.bind_send.as_millis() as u64,
7281 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7282 "slow control dispatch"
7283 );
7284 return;
7285 }
7286 warn!(
7287 op = %op,
7288 connection_id = connection_id.get(),
7289 corr,
7290 elapsed_ms = elapsed.as_millis() as u64,
7291 "slow control dispatch"
7292 );
7293 }
7294}
7295
7296#[cfg(test)]
7298type TestCallerLookup =
7299 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7300
7301fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7303 match request {
7304 ClientControlRequest::SupervisorRestart { module_id, .. } => {
7305 Some(PrivilegedOperation::Restart { module_id })
7306 }
7307 ClientControlRequest::SupervisorSwap { module_id, .. } => {
7308 Some(PrivilegedOperation::Swap { module_id })
7309 }
7310 ClientControlRequest::SupervisorReload { module_id } => {
7311 Some(PrivilegedOperation::Reload { module_id })
7312 }
7313 ClientControlRequest::SupervisorRescan { preview } => {
7314 if *preview {
7315 None
7316 } else {
7317 Some(PrivilegedOperation::Rescan)
7318 }
7319 }
7320 ClientControlRequest::SupervisorRescanChecked {
7325 preview,
7326 candidate,
7327 expect,
7328 } => (!*preview && candidate.is_none() && expect.is_some())
7329 .then_some(PrivilegedOperation::Rescan),
7330 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7331 Some(PrivilegedOperation::SetEnabled {
7332 module_id,
7333 enabled: *enabled,
7334 })
7335 }
7336 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7337 Some(PrivilegedOperation::ReleaseReserved { module_id })
7338 }
7339 ClientControlRequest::ServerDescribe {}
7340 | ClientControlRequest::CatalogList { .. }
7341 | ClientControlRequest::RouteOpen { .. }
7342 | ClientControlRequest::RoutePoll { .. }
7343 | ClientControlRequest::SupervisorList {}
7344 | ClientControlRequest::SupervisorHealth {}
7345 | ClientControlRequest::SupervisorHealthProbe { .. }
7346 | ClientControlRequest::SupervisorRoutes { .. }
7347 | ClientControlRequest::SupervisorProvenance { .. }
7348 | ClientControlRequest::SupervisorStderrTail { .. }
7349 | ClientControlRequest::SupervisorTerminals { .. }
7350 | ClientControlRequest::SupervisorSpawnSnapshot {}
7351 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7352 }
7353}
7354
7355fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7356 match request {
7357 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7358 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7359 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7360 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7361 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7362 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7363 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7364 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7365 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7366 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7367 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7368 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7369 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7370 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7371 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7372 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7373 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7374 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7375 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7376 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7377 }
7378}
7379
7380struct RouteOpenCaller {
7388 principal: String,
7392 scope: String,
7395 harness: String,
7398}
7399
7400impl RouteOpenCaller {
7401 fn new(
7402 principal: Option<&Principal>,
7403 scope: Option<&ScopeSelector>,
7404 identity: &BindIdentity,
7405 ) -> Self {
7406 Self {
7407 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
7408 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
7409 harness: bounded_log_text(&identity.harness),
7410 }
7411 }
7412
7413 fn unknown() -> Self {
7416 Self {
7417 principal: "unknown".to_string(),
7418 scope: "unknown".to_string(),
7419 harness: "unknown".to_string(),
7420 }
7421 }
7422}
7423
7424const MAX_LOGGED_TEXT_CHARS: usize = 64;
7427
7428fn bounded_log_text(value: &str) -> String {
7432 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
7433 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
7434 bounded.push('…');
7435 }
7436 format!("{bounded:?}")
7437}
7438
7439fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7445 use subc_protocol::scope::ScopeKind;
7446 let kind = match stamp.kind {
7447 ScopeKind::Head => "head",
7448 ScopeKind::Worker => "worker",
7449 ScopeKind::Ephemeral => "ephemeral",
7450 };
7451 format!(
7452 "{kind}/{}/{}@{}",
7453 principal_label(&stamp.owner),
7454 bounded_log_text(&stamp.scope_ref),
7455 stamp.scope_epoch
7456 )
7457}
7458
7459fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
7466 let epoch = selector
7467 .scope_epoch
7468 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
7469 format!(
7470 "requested/{}/{}@{epoch}",
7471 bounded_log_text(&principal_label(&selector.owner)),
7472 bounded_log_text(&selector.scope_ref),
7473 )
7474}
7475
7476fn principal_label(principal: &Principal) -> String {
7477 match principal {
7478 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7479 Principal::Direct => "direct".to_string(),
7480 other => format!("{other:?}"),
7481 }
7482}
7483
7484fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7485 match request {
7486 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7487 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7488 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7489 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7490 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7491 }
7492}
7493
7494fn parse_client_control_request(
7495 body: &[u8],
7496) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7497 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7498 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7499 Ok(probe)
7500 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7501 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7502 {
7503 ControlRequestBodyError::InvalidBody
7504 }
7505 Ok(_) => ControlRequestBodyError::UnknownOp,
7506 Err(_) => ControlRequestBodyError::InvalidBody,
7507 };
7508 (err, classification)
7509 })
7510}
7511
7512fn parse_module_control_request_from_module(
7513 body: &[u8],
7514) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7515 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7516 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7517 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7518 Ok(_) => ControlRequestBodyError::UnknownOp,
7519 Err(_) => ControlRequestBodyError::InvalidBody,
7520 };
7521 (err, classification)
7522 })
7523}
7524
7525#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7526enum ProviderRoleKind {
7527 ToolProvider,
7528 PipelineStage,
7529 ManagementSurface,
7530 InternalService,
7531}
7532
7533fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7534 match role {
7535 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7536 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7537 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7538 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7539 }
7540}
7541
7542fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7543 roles.iter().map(provider_role_kind).collect()
7544}
7545
7546const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7549
7550#[derive(Debug, Default, PartialEq, Eq)]
7552struct ScopeOutcomeCounts {
7553 created: usize,
7554 replaced: usize,
7555 updated: usize,
7556 unchanged: usize,
7557 refused: usize,
7558}
7559
7560impl ScopeOutcomeCounts {
7561 fn of(results: &[ScopeRecordResult]) -> Self {
7562 let mut counts = Self::default();
7563 for result in results {
7564 let slot = match result.outcome {
7565 ScopeRecordOutcome::Created => &mut counts.created,
7566 ScopeRecordOutcome::Replaced => &mut counts.replaced,
7567 ScopeRecordOutcome::Updated => &mut counts.updated,
7568 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7569 ScopeRecordOutcome::Refused => &mut counts.refused,
7570 };
7571 *slot += 1;
7572 }
7573 counts
7574 }
7575}
7576
7577#[cfg(test)]
7578mod scope_outcome_count_tests {
7579 use super::*;
7580
7581 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7582 ScopeRecordResult {
7583 scope_ref: "r".to_string(),
7584 scope_epoch: 1,
7585 outcome,
7586 code: None,
7587 message: None,
7588 version: None,
7589 parent_state: None,
7590 }
7591 }
7592
7593 #[test]
7596 fn every_outcome_is_counted_in_its_own_field() {
7597 let results = [
7598 result(ScopeRecordOutcome::Created),
7599 result(ScopeRecordOutcome::Created),
7600 result(ScopeRecordOutcome::Replaced),
7601 result(ScopeRecordOutcome::Updated),
7602 result(ScopeRecordOutcome::Unchanged),
7603 result(ScopeRecordOutcome::Refused),
7604 result(ScopeRecordOutcome::Refused),
7605 result(ScopeRecordOutcome::Refused),
7606 ];
7607 assert_eq!(
7608 ScopeOutcomeCounts::of(&results),
7609 ScopeOutcomeCounts {
7610 created: 2,
7611 replaced: 1,
7612 updated: 1,
7613 unchanged: 1,
7614 refused: 3,
7615 }
7616 );
7617 }
7618}
7619
7620fn capability_census_trigger(
7624 old: Option<&CapabilityDeclarations>,
7625 new: Option<&CapabilityDeclarations>,
7626) -> bool {
7627 let old_provides = old
7628 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7629 .unwrap_or_default();
7630 let old_denies = old
7631 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7632 .unwrap_or_default();
7633 let new = new.cloned().unwrap_or(CapabilityDeclarations {
7634 provides: Vec::new(),
7635 requires: Vec::new(),
7636 must_never_reach: Vec::new(),
7637 });
7638
7639 new.provides
7640 .iter()
7641 .any(|capability| !old_provides.contains(capability))
7642 || new
7643 .must_never_reach
7644 .iter()
7645 .any(|capability| !old_denies.contains(capability))
7646}
7647
7648fn denied_capability<'a>(
7651 opening_manifest: &'a ModuleManifest,
7652 target_manifest: &ModuleManifest,
7653) -> Option<&'a str> {
7654 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7655 let target_capabilities = target_manifest.capabilities.as_ref()?;
7656 opening_capabilities
7657 .must_never_reach
7658 .iter()
7659 .find(|denied| {
7660 target_capabilities
7661 .provides
7662 .iter()
7663 .any(|provided| provided == *denied)
7664 })
7665 .map(String::as_str)
7666}
7667
7668fn catalog_update_frozen_field_message(
7669 registered: &ModuleManifest,
7670 provides: &[ProviderRole],
7671) -> Option<String> {
7672 let old_has_provides = !registered.provides.is_empty();
7673 let new_has_provides = !provides.is_empty();
7674 if old_has_provides != new_has_provides {
7675 return Some(format!(
7676 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7677 registered.module_id
7678 ));
7679 }
7680
7681 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
7682 return Some(format!(
7683 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7684 registered.module_id
7685 ));
7686 }
7687
7688 let registered_concurrency = manifest_concurrency(registered);
7689 let mut candidate = registered.clone();
7690 candidate.provides = provides.to_vec();
7691 let candidate_concurrency = manifest_concurrency(&candidate);
7692 if candidate_concurrency != registered_concurrency {
7693 return Some(format!(
7694 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7695 registered.module_id, registered_concurrency, candidate_concurrency
7696 ));
7697 }
7698
7699 None
7702}
7703
7704fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7705 manifest.provides.iter().any(is_routable_role)
7706}
7707
7708fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7714 manifest
7715 .provides
7716 .iter()
7717 .find_map(|provider| match provider {
7718 ProviderRole::ToolProvider { concurrency, .. }
7719 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7720 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7721 })
7722 .unwrap_or(Concurrency::ModuleManaged)
7723}
7724
7725fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7732 let has_management_surface = manifest
7733 .provides
7734 .iter()
7735 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7736 if !has_management_surface {
7737 return false;
7738 }
7739 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7740 return false;
7741 };
7742 let Some(provides) = raw
7743 .get("manifest")
7744 .and_then(|manifest| manifest.get("provides"))
7745 .and_then(serde_json::Value::as_array)
7746 else {
7747 return false;
7748 };
7749 provides.iter().any(|role| {
7754 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7755 && role.get("concurrency").is_none()
7756 })
7757}
7758
7759fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7760 if peer_version != PROTOCOL_VERSION {
7761 return Err(format!(
7762 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7763 ));
7764 }
7765 Ok(PROTOCOL_VERSION)
7766}
7767
7768fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7769 Frame::build_with_version(
7770 response_version(frame),
7771 FrameType::Pong,
7772 frame.header.flags,
7773 0,
7774 0,
7775 frame.header.corr,
7776 Vec::new(),
7777 )
7778 .map_err(RouterError::FrameBuild)
7779}
7780
7781pub(crate) fn control_error_frame(
7782 frame: &Frame,
7783 code: &'static str,
7784 message: impl Into<String>,
7785) -> Result<Frame, RouterError> {
7786 control_error_body_frame(
7787 frame,
7788 ErrorBody {
7789 code: code.to_string(),
7790 message: message.into(),
7791 detail: None,
7792 },
7793 )
7794}
7795
7796fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7797 let body = serde_json::to_vec(&error).map_err(|err| {
7798 RouterError::backend(
7799 0,
7800 frame.header.corr,
7801 format!("failed to encode control ERROR: {err}"),
7802 )
7803 })?;
7804
7805 Frame::build_with_version(
7806 response_version(frame),
7807 FrameType::Error,
7808 control_flags(),
7809 0,
7810 0,
7811 frame.header.corr,
7812 body,
7813 )
7814 .map_err(RouterError::FrameBuild)
7815}
7816
7817fn control_response_body_frame<T: Serialize>(
7818 frame: &Frame,
7819 reply: &T,
7820 label: &'static str,
7821) -> Result<Frame, RouterError> {
7822 let body = serde_json::to_vec(reply).map_err(|err| {
7823 RouterError::backend(
7824 0,
7825 frame.header.corr,
7826 format!("failed to encode {label}: {err}"),
7827 )
7828 })?;
7829
7830 Frame::build_with_version(
7831 response_version(frame),
7832 FrameType::Response,
7833 control_flags(),
7834 0,
7835 0,
7836 frame.header.corr,
7837 body,
7838 )
7839 .map_err(RouterError::FrameBuild)
7840}
7841
7842fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7872 match err {
7873 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7874 ForwardingError::NoModuleConnection => "target_unavailable",
7875 ForwardingError::ModuleReloading { .. } => "module_reloading",
7876 ForwardingError::ClientRouteChannelExhausted { .. }
7877 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7878 ForwardingError::StaleModuleEndpoint
7879 | ForwardingError::UnknownReservation { .. }
7880 | ForwardingError::ConnectionClosing { .. }
7881 | ForwardingError::ClientEgressClosed { .. }
7882 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7883 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7886 ForwardingError::RelayCorrelationExhausted
7887 | ForwardingError::RouteOpenBuild(_)
7888 | ForwardingError::Poisoned => "forwarding_error",
7889 }
7890}
7891
7892fn response_version(frame: &Frame) -> u8 {
7893 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7894 frame.header.ver
7895 } else {
7896 PROTOCOL_VERSION
7897 }
7898}
7899
7900fn control_flags() -> Flags {
7901 Flags::new(false, Priority::Passive, false)
7902}
7903
7904fn send_goodbye_target_best_effort(
7909 counters: &DaemonCounters,
7910 target: &GoodbyeTarget,
7911 context: &'static str,
7912) {
7913 let Ok(frame) = Frame::build_with_version(
7914 target.negotiated_ver,
7915 FrameType::Goodbye,
7916 control_flags(),
7917 target.channel,
7918 target.epoch,
7919 0,
7920 Vec::new(),
7921 ) else {
7922 return;
7923 };
7924 crate::forwarding::send_module_route_goodbye(
7925 counters,
7926 &target.sink,
7927 frame,
7928 target.module_id.as_deref(),
7929 context,
7930 );
7931}
7932
7933pub(crate) fn send_route_control_pushes(
7934 forwarding: &ForwardingTable,
7935 routes: Vec<EndpointRoute>,
7936 push: ClientControlPush,
7937) {
7938 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7939 for route in routes {
7940 let target = route.goodbye_target;
7941 if let Some((existing, channels)) = targets
7942 .iter_mut()
7943 .find(|(existing, _)| existing.connection_id == target.connection_id)
7944 {
7945 debug_assert_eq!(
7946 existing.negotiated_ver, target.negotiated_ver,
7947 "one connection cannot negotiate multiple frame versions"
7948 );
7949 if !channels.contains(&target.channel) {
7950 channels.push(target.channel);
7951 }
7952 continue;
7953 }
7954 let channel = target.channel;
7955 targets.push((target, vec![channel]));
7956 }
7957 for (target, mut channels) in targets {
7958 channels.sort_unstable();
7959 let mut push = push.clone();
7960 match &mut push {
7961 ClientControlPush::RouteClosing {
7962 channels: covered, ..
7963 }
7964 | ClientControlPush::RouteClosed {
7965 channels: covered, ..
7966 } => *covered = channels,
7967 }
7968 let body = match serde_json::to_vec(&push) {
7969 Ok(body) => body,
7970 Err(err) => {
7971 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7972 continue;
7973 }
7974 };
7975 let frame = match Frame::build_with_version(
7976 target.negotiated_ver,
7977 FrameType::Push,
7978 control_flags(),
7979 0,
7980 0,
7981 0,
7982 body.clone(),
7983 ) {
7984 Ok(frame) => frame,
7985 Err(err) => {
7986 warn!(
7987 route_channel = target.channel,
7988 error = %err,
7989 "failed to build route lifecycle control PUSH frame"
7990 );
7991 continue;
7992 }
7993 };
7994 if let Err(err) = target.sink.try_send(frame) {
7995 if target.close_on_delivery_failure() {
7996 warn!(
7997 target_connection_id = target.connection_id.get(),
7998 route_channel = target.channel,
7999 error = %err,
8000 "route lifecycle control PUSH was not delivered to client; closing target connection"
8001 );
8002 let _ = forwarding.escalate_client_delivery_failure(
8003 target.connection_id,
8004 target.channel,
8005 target.epoch,
8006 CloseReason::new(
8007 "route_lifecycle_push_delivery_failed",
8008 format!(
8009 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8010 target.channel
8011 ),
8012 ),
8013 crate::forwarding::UndeliveredFrame {
8014 module_id: target.module_id.as_deref(),
8015 sink: &target.sink,
8016 },
8017 );
8018 }
8019 }
8020 }
8021}
8022
8023#[cfg(test)]
8024mod tests {
8025 mod activity_read {
8026 include!("activity_read_tests.rs");
8027 }
8028
8029 mod lifecycle_activity {
8030 include!("control_lifecycle_tests.rs");
8031 }
8032 #[cfg(unix)]
8033 #[tokio::test]
8034 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8035 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8036 let path = dir.join("subc.jsonc");
8037 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8038 "program":"/bin/sleep","args":["60"],"protocol":"none",
8039 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8040 }}}).to_string()).unwrap();
8041 let mut configured = crate::daemon_config::load(&path)
8042 .unwrap()
8043 .unwrap()
8044 .modules
8045 .pop()
8046 .unwrap();
8047 let registry = std::sync::Arc::new(crate::Registry::default());
8048 let handle = crate::SupervisorHandle::new();
8049 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8050 .with_handle(handle.clone());
8051 let module = supervisor
8052 .supervise_configured_with_health(
8053 configured.module_spec(),
8054 true,
8055 configured.health.clone(),
8056 None,
8057 configured.restart,
8058 )
8059 .unwrap();
8060 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8061 let before = module.status().unwrap().pid;
8062 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8063 configured.health.cadence = std::time::Duration::from_secs(3600);
8064 let health_only = handler
8065 .reconcile_supervised_modules(
8066 &supervisor,
8067 vec![configured.clone()],
8068 false,
8069 Principal::Direct,
8070 )
8071 .await
8072 .unwrap();
8073 assert!(
8074 health_only.0.changed_pending_reload.is_empty(),
8075 "health policy is already applied live"
8076 );
8077 assert_eq!(module.status().unwrap().pid, before);
8078 assert_eq!(
8079 module.configuration().unwrap().1.http,
8080 configured.health.http
8081 );
8082 configured.args = vec!["61".into()];
8083 let launch = handler
8084 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8085 .await
8086 .unwrap();
8087 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8088 assert_eq!(
8089 module.status().unwrap().pid,
8090 before,
8091 "a launch edit is stored until reload"
8092 );
8093 module.drain().await.unwrap();
8094 }
8095 use cortexkit_test_support::ScratchDir;
8096 use std::{
8097 collections::BTreeMap,
8098 fmt,
8099 path::PathBuf,
8100 sync::{Arc, Mutex},
8101 time::Duration,
8102 };
8103
8104 use serde_json::{json, Value};
8105 use subc_protocol::{
8106 manifest::{
8107 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8108 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8109 },
8110 session::HealthStatus,
8111 FrameType,
8112 };
8113
8114 use super::*;
8115 use crate::{
8116 forwarding::{DataRoute, DataRouteState},
8117 registry::ChannelState,
8118 router::FrameSink,
8119 stderr_tail::DEFAULT_MAX_LINE_BYTES,
8120 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8121 RouteCtx, Router,
8122 };
8123 use tokio::{
8124 sync::mpsc,
8125 time::{sleep, Instant},
8126 };
8127 use tracing::{
8128 field::{Field, Visit},
8129 Event, Subscriber,
8130 };
8131 use tracing_subscriber::{layer::Context, prelude::*, Layer};
8132
8133 fn fake_aft_stub_path() -> PathBuf {
8152 let mut path = std::env::current_exe().expect("current_exe available in tests");
8153 path.pop(); path.pop(); path.push(if cfg!(windows) {
8156 "fake-aft-stub.exe"
8157 } else {
8158 "fake-aft-stub"
8159 });
8160 assert!(
8161 path.exists(),
8162 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8163 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8164 path.display()
8165 );
8166 path
8167 }
8168
8169 fn client_retries(code: &str) -> bool {
8174 subc_protocol::error_codes::is_retryable_route_open(code)
8175 }
8176
8177 #[test]
8188 fn retryability_of_forwarding_codes_matches_the_failure() {
8189 let transient = [
8192 ForwardingError::NoModuleConnection,
8193 ForwardingError::ModuleReloading {
8194 module_id: "m".into(),
8195 },
8196 ForwardingError::StaleModuleEndpoint,
8197 ForwardingError::UnknownReservation {
8198 client_channel: 1,
8199 module_channel: 1,
8200 },
8201 ForwardingError::ConnectionClosing {
8202 connection_id: ConnectionId::new(1),
8203 },
8204 ForwardingError::ClientEgressClosed {
8205 connection_id: ConnectionId::new(1),
8206 },
8207 ForwardingError::ModuleEgressUnavailable {
8208 connection_id: ConnectionId::new(1),
8209 },
8210 ];
8211 for err in transient {
8212 let code = forwarding_error_code(&err);
8213 assert!(
8214 client_retries(code),
8215 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8216 );
8217 }
8218
8219 let permanent = [
8224 ForwardingError::ConnectionRoleConflict {
8225 connection_id: ConnectionId::new(1),
8226 },
8227 ForwardingError::ClientRouteChannelExhausted {
8228 connection_id: ConnectionId::new(1),
8229 },
8230 ForwardingError::ModuleRouteChannelExhausted {
8231 endpoint: ModuleEndpointId {
8232 connection_id: ConnectionId::new(1),
8233 generation: 1,
8234 },
8235 },
8236 ForwardingError::RelayCorrelationExhausted,
8237 ForwardingError::RouteOpenBuild("x".into()),
8238 ForwardingError::Poisoned,
8239 ];
8240 for err in permanent {
8241 let code = forwarding_error_code(&err);
8242 assert!(
8243 !client_retries(code),
8244 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8245 );
8246 }
8247 }
8248
8249 #[tokio::test]
8277 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8278 let handler = ControlHandler::default();
8279 let frame =
8280 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8281
8282 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8286 assert_eq!(
8287 stamped,
8288 Principal::Direct,
8289 "a caller that proved nothing must not be stamped as a supervised module"
8290 );
8291
8292 let forged = handler
8297 .route_open_principal(
8298 &frame,
8299 Some(ConsumerIdentity {
8300 module_id: "aft".to_string(),
8301 launch_nonce: "not-a-real-nonce".to_string(),
8302 }),
8303 )
8304 .unwrap();
8305 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8306 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8307 }
8308
8309 #[test]
8326 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8327 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8328 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8329 let ClientControlRequest::RouteOpen {
8330 consumer_identity, ..
8331 } = parsed
8332 else {
8333 panic!("route.open body must parse as RouteOpen");
8334 };
8335 assert_eq!(
8336 consumer_identity,
8337 Some(ConsumerIdentity {
8338 module_id: "aft".to_string(),
8339 launch_nonce: "n".to_string(),
8340 }),
8341 "the wire field name must reach the value route_open_principal reads"
8342 );
8343 }
8344
8345 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8346 ModuleManifest::builder(module_id, "0.1.0")
8347 .protocol_ver(protocol_ver)
8348 .provides(vec![ProviderRole::ToolProvider {
8349 tools: vec![Tool {
8350 name: "read".to_string(),
8351 description: None,
8352 execution_mode: ExecutionMode::Pure,
8353 schema: json!({"type": "object"}),
8354 }],
8355 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8356 concurrency: Concurrency::ModuleManaged,
8357 emits_push: true,
8358 sub_supervises: true,
8359 }])
8360 .build()
8361 }
8362
8363 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8364 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8365 }
8366
8367 fn hello_frame_with_control_ops(
8368 module_id: &str,
8369 protocol_ver: u8,
8370 corr: u64,
8371 control_ops: Option<Vec<String>>,
8372 ) -> Frame {
8373 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8374 }
8375
8376 fn hello_frame_with_nonce(
8377 module_id: &str,
8378 protocol_ver: u8,
8379 corr: u64,
8380 launch_nonce: Option<&str>,
8381 ) -> Frame {
8382 hello_frame_full(
8383 module_id,
8384 protocol_ver,
8385 corr,
8386 None,
8387 launch_nonce.map(ToOwned::to_owned),
8388 )
8389 }
8390
8391 fn hello_frame_full(
8392 module_id: &str,
8393 protocol_ver: u8,
8394 corr: u64,
8395 control_ops: Option<Vec<String>>,
8396 launch_nonce: Option<String>,
8397 ) -> Frame {
8398 let body = serde_json::to_vec(&ModuleHelloBody {
8399 manifest: manifest(module_id, protocol_ver),
8400 protocol_ver,
8401 control_ops,
8402 launch_nonce,
8403 })
8404 .unwrap();
8405 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8406 }
8407
8408 fn hello_frame_with_rich_ui(
8409 module_id: &str,
8410 protocol_ver: u8,
8411 corr: u64,
8412 launch_nonce: Option<&str>,
8413 rich_ui: Value,
8414 ) -> Frame {
8415 let mut body = serde_json::to_value(ModuleHelloBody {
8416 manifest: manifest(module_id, protocol_ver),
8417 protocol_ver,
8418 control_ops: None,
8419 launch_nonce: launch_nonce.map(str::to_string),
8420 })
8421 .unwrap();
8422 body["manifest"]["rich_ui"] = rich_ui;
8423 Frame::build(
8424 FrameType::Hello,
8425 control_flags(),
8426 0,
8427 0,
8428 corr,
8429 serde_json::to_vec(&body).unwrap(),
8430 )
8431 .unwrap()
8432 }
8433
8434 fn non_routable_hello_frame_with_control_ops(
8435 module_id: &str,
8436 corr: u64,
8437 control_ops: Option<Vec<String>>,
8438 ) -> Frame {
8439 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8440 manifest.provides.clear();
8441 let body = serde_json::to_vec(&ModuleHelloBody {
8442 manifest,
8443 protocol_ver: PROTOCOL_VERSION,
8444 control_ops,
8445 launch_nonce: None,
8446 })
8447 .unwrap();
8448 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8449 }
8450
8451 fn capability_grammar_hello_frame(
8452 capabilities: Value,
8453 runtime_computed: Option<Value>,
8454 corr: u64,
8455 ) -> Frame {
8456 let mut body = serde_json::to_value(ModuleHelloBody {
8457 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8458 protocol_ver: PROTOCOL_VERSION,
8459 control_ops: None,
8460 launch_nonce: None,
8461 })
8462 .expect("HELLO body serializes");
8463 body["manifest"]["capabilities"] = capabilities;
8464 if let Some(runtime_computed) = runtime_computed {
8465 body["runtime_computed"] = runtime_computed;
8466 }
8467 Frame::build(
8468 FrameType::Hello,
8469 control_flags(),
8470 0,
8471 0,
8472 corr,
8473 serde_json::to_vec(&body).expect("HELLO body reserializes"),
8474 )
8475 .expect("HELLO frame builds")
8476 }
8477
8478 fn channel_request(channel: u16, corr: u64) -> Frame {
8479 Frame::build(
8480 FrameType::Request,
8481 Flags::new(true, Priority::Interactive, false),
8482 channel,
8483 0,
8484 corr,
8485 b"opaque".to_vec(),
8486 )
8487 .unwrap()
8488 }
8489
8490 fn route_ctx(
8491 connection_id: ConnectionId,
8492 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8493 let (tx, rx) = mpsc::channel(8);
8494 (
8495 RouteCtx {
8496 connection_id,
8497 egress: FrameSink::new(tx),
8498 },
8499 rx,
8500 )
8501 }
8502
8503 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8504 serde_json::from_slice(&frame.body).unwrap()
8505 }
8506
8507 async fn hello_via_sink(
8512 handler: &ControlHandler,
8513 ctx: &RouteCtx,
8514 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8515 hello: Frame,
8516 ) -> Frame {
8517 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8518 assert!(
8519 replies.is_empty(),
8520 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8521 );
8522 let ack = rx
8523 .try_recv()
8524 .expect("HELLO_ACK is queued on the module sink")
8525 .frame;
8526 assert_eq!(ack.header.ty, FrameType::HelloAck);
8527 ack
8528 }
8529
8530 fn parse_error(frame: &Frame) -> Value {
8531 serde_json::from_slice(&frame.body).unwrap()
8532 }
8533
8534 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8535 serde_json::from_slice(&frame.body).unwrap()
8536 }
8537
8538 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8539 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8540 route_channel,
8541 route_epoch: 0,
8542 kind,
8543 })
8544 .unwrap();
8545 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8546 }
8547
8548 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8549 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8550 module_id: module_id.to_string(),
8551 })
8552 .unwrap();
8553 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8554 }
8555
8556 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8557 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8558 }
8559
8560 fn route_open_frame_with_consumer_capabilities(
8561 corr: u64,
8562 module_id: &str,
8563 project_root: ScratchDir,
8564 consumer_capabilities: Option<Vec<String>>,
8565 ) -> Frame {
8566 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8567 target: RouteTarget::ToolProvider {
8568 module_id: module_id.to_string(),
8569 },
8570 identity: BindIdentity::new(
8571 project_root.path().to_path_buf(),
8572 "unit".to_string(),
8573 "session".to_string(),
8574 ),
8575 consumer_identity: None,
8576 consumer_capabilities,
8577 role_versions: None,
8578 admission_facts: None,
8579 scope: None,
8580 })
8581 .unwrap();
8582 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8583 }
8584
8585 fn route_open_frame_with_role_versions(
8586 corr: u64,
8587 module_id: &str,
8588 project_root: ScratchDir,
8589 role_versions: Option<BTreeMap<String, String>>,
8590 ) -> Frame {
8591 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8592 target: RouteTarget::ToolProvider {
8593 module_id: module_id.to_string(),
8594 },
8595 identity: BindIdentity::new(
8596 project_root.path().to_path_buf(),
8597 "unit".to_string(),
8598 format!("session-{corr}"),
8599 ),
8600 consumer_identity: None,
8601 consumer_capabilities: None,
8602 role_versions,
8603 admission_facts: None,
8604 scope: None,
8605 })
8606 .unwrap();
8607 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8608 }
8609
8610 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8611 entries
8612 .iter()
8613 .map(|(role, version)| (role.to_string(), version.to_string()))
8614 .collect()
8615 }
8616
8617 fn route_open_frame_with_admission_facts(
8618 corr: u64,
8619 module_id: &str,
8620 project_root: ScratchDir,
8621 consumer_identity: Option<subc_control::ConsumerIdentity>,
8622 facts: Option<Value>,
8623 ) -> Frame {
8624 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8625 target: RouteTarget::ToolProvider {
8626 module_id: module_id.to_string(),
8627 },
8628 identity: BindIdentity::new(
8629 project_root.path().to_path_buf(),
8630 "unit".to_string(),
8631 format!("session-{corr}"),
8632 ),
8633 consumer_identity,
8634 consumer_capabilities: None,
8635 role_versions: None,
8636 admission_facts: facts,
8637 scope: None,
8638 })
8639 .unwrap();
8640 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8641 }
8642
8643 #[derive(Clone, Default)]
8644 struct EventCapture {
8645 events: Arc<Mutex<Vec<CapturedEvent>>>,
8646 }
8647
8648 #[derive(Clone, Debug)]
8649 struct CapturedEvent {
8650 target: String,
8651 level: tracing::Level,
8652 fields: BTreeMap<String, String>,
8653 }
8654
8655 impl EventCapture {
8656 fn events(&self) -> Vec<CapturedEvent> {
8657 self.events.lock().unwrap().clone()
8658 }
8659 }
8660
8661 impl<S> Layer<S> for EventCapture
8662 where
8663 S: Subscriber,
8664 {
8665 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8666 let mut visitor = EventFieldVisitor::default();
8667 event.record(&mut visitor);
8668 self.events.lock().unwrap().push(CapturedEvent {
8669 target: event.metadata().target().to_string(),
8670 level: *event.metadata().level(),
8671 fields: visitor.fields,
8672 });
8673 }
8674 }
8675
8676 #[derive(Default)]
8677 struct EventFieldVisitor {
8678 fields: BTreeMap<String, String>,
8679 }
8680
8681 impl Visit for EventFieldVisitor {
8682 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8683 self.fields
8684 .insert(field.name().to_string(), format!("{value:?}"));
8685 }
8686 }
8687
8688 fn health_response(corr: u64, status: HealthStatus) -> Frame {
8689 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8690 status,
8691 detail: Some("warming".to_string()),
8692 metrics: Some(json!({"queue_depth": 3})),
8693 })
8694 .unwrap();
8695 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8696 }
8697
8698 fn route_bind_ack(corr: u64) -> Frame {
8699 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8700 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8701 }
8702
8703 fn unique_project_root(label: &str) -> ScratchDir {
8704 ScratchDir::new(label)
8705 }
8706
8707 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8708 match parse_route_poll(frame) {
8709 ClientControlResponse::RoutePoll {
8710 status: None,
8711 live: Some(live),
8712 ..
8713 } => assert_eq!(live, expected_live),
8714 other => panic!("unexpected route.poll response: {other:?}"),
8715 }
8716 }
8717
8718 fn bind_liveness_route(
8719 registry: &Registry,
8720 forwarding: &ForwardingTable,
8721 module_id: &str,
8722 ) -> (RouteCtx, u16, u32) {
8723 let module_connection = ConnectionId::new(101);
8724 let client_connection = ConnectionId::new(202);
8725 let registration = registry
8726 .register_with_control_ops(
8727 manifest(module_id, PROTOCOL_VERSION),
8728 PROTOCOL_VERSION,
8729 module_connection,
8730 module_baseline_control_ops(),
8731 )
8732 .unwrap();
8733 let (module_tx, _module_rx) = mpsc::channel(8);
8734 let endpoint = forwarding
8735 .register_module_connection(
8736 module_connection,
8737 module_id.to_string(),
8738 PROTOCOL_VERSION,
8739 manifest_concurrency(®istration.manifest),
8740 FrameSink::new(module_tx),
8741 )
8742 .unwrap();
8743 let (client_ctx, _client_rx) = route_ctx(client_connection);
8744 let pending = forwarding
8745 .begin_route_bind_relay_for_test(
8746 client_connection,
8747 client_ctx.egress.clone(),
8748 1,
8749 module_id,
8750 )
8751 .unwrap();
8752 assert_eq!(pending.endpoint, endpoint);
8753 let route_channel = pending.client_channel;
8754 let route_epoch = pending.client_epoch;
8755 forwarding
8756 .complete_pending_relay(
8757 module_connection,
8758 pending.corr,
8759 RouteBindRelayOutcome::Accepted,
8760 )
8761 .unwrap();
8762 (client_ctx, route_channel, route_epoch)
8763 }
8764
8765 struct FakeProcessLiveness {
8766 live: Option<bool>,
8767 }
8768
8769 impl ModuleProcessLiveness for FakeProcessLiveness {
8770 fn process_live(&self, _module_id: &str) -> Option<bool> {
8771 self.live
8772 }
8773 }
8774
8775 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8776 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8777 {
8778 let registry = Arc::new(Registry::default());
8779 let supervisor_handle = SupervisorHandle::new();
8780 let supervisor = Supervisor::new_for_test(
8781 Arc::clone(®istry),
8782 RestartPolicy::new(1, Duration::from_millis(10)),
8783 )
8784 .with_handle(supervisor_handle.clone());
8785 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8786 let module = supervisor
8787 .spawn(ModuleSpec {
8788 module_id: "stderr-tail-wire".to_string(),
8789 program: fake_aft_stub_path(),
8790 args: Vec::new(),
8791 env: vec![
8792 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8793 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8794 ],
8795 reserved: false,
8796 reserved_prefixes: Vec::new(),
8797 protocol: ModuleProtocol::Subc,
8798 overlap: Default::default(),
8799 })
8800 .unwrap();
8801
8802 let deadline = Instant::now() + Duration::from_secs(5);
8803 loop {
8804 let tail = module.stderr_tail(None, None);
8805 if tail
8806 .entries
8807 .iter()
8808 .any(|entry| matches!(entry, TailEntry::ProcessStart))
8809 && tail.entries.iter().any(|entry| {
8810 matches!(
8811 entry,
8812 TailEntry::Line {
8813 truncated: true,
8814 ..
8815 }
8816 )
8817 })
8818 {
8819 break;
8820 }
8821 assert!(
8822 Instant::now() < deadline,
8823 "module did not produce a truncated line and restart boundary: {tail:?}"
8824 );
8825 sleep(Duration::from_millis(10)).await;
8826 }
8827
8828 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8829 let request = ClientControlRequest::SupervisorStderrTail {
8830 module_id: "stderr-tail-wire".to_string(),
8831 max_lines: None,
8832 max_bytes: None,
8833 };
8834 let frame = Frame::build(
8835 FrameType::Request,
8836 control_flags(),
8837 0,
8838 0,
8839 1,
8840 serde_json::to_vec(&request).unwrap(),
8841 )
8842 .unwrap();
8843 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8844 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8845 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8846 serde_json::from_slice(&responses[0].body).unwrap()
8847 else {
8848 panic!("expected supervisor.stderr_tail response");
8849 };
8850
8851 assert!(
8852 tail.entries
8853 .iter()
8854 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8855 "the control response lost the restart boundary"
8856 );
8857 let Some(StderrTailEntry::Line {
8858 text,
8859 truncated,
8860 at_ms,
8861 }) = tail.entries.iter().find(|entry| {
8862 matches!(
8863 entry,
8864 StderrTailEntry::Line {
8865 truncated: true,
8866 ..
8867 }
8868 )
8869 })
8870 else {
8871 panic!("the control response lost the truncated line");
8872 };
8873 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8874 assert!(*truncated);
8875 assert!(
8876 at_ms.is_some(),
8877 "the control response lost the line's capture time"
8878 );
8879 }
8880
8881 #[tokio::test(flavor = "current_thread")]
8886 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8887 let dir = ScratchDir::new("terminals-off-worker");
8888 let journal_path = dir.join("terminals.jsonl");
8889 let registry = Arc::new(Registry::default());
8890 let supervisor_handle = SupervisorHandle::new();
8891 let supervisor =
8892 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8893 .with_handle(supervisor_handle.clone())
8894 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8895 let module = supervisor
8896 .spawn(ModuleSpec {
8897 module_id: "terminal-off-worker".to_string(),
8898 program: fake_aft_stub_path(),
8899 args: Vec::new(),
8900 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8901 reserved: false,
8902 reserved_prefixes: Vec::new(),
8903 protocol: ModuleProtocol::Subc,
8904 overlap: Default::default(),
8905 })
8906 .unwrap();
8907 let deadline = Instant::now() + Duration::from_secs(5);
8908 while module.terminal_history().entries.len() != 2 {
8909 assert!(Instant::now() < deadline, "module did not record two exits");
8910 sleep(Duration::from_millis(10)).await;
8911 }
8912
8913 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8914 let handler =
8915 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
8916 let frame = Frame::build(
8917 FrameType::Request,
8918 control_flags(),
8919 0,
8920 0,
8921 1,
8922 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8923 module_id: "terminal-off-worker".to_string(),
8924 })
8925 .unwrap(),
8926 )
8927 .unwrap();
8928 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8929 let spawned_at = std::time::Instant::now();
8930 let read = tokio::spawn({
8931 let handler = Arc::clone(&handler);
8932 async move { handler.handle_control_frame(&ctx, frame).await }
8933 });
8934 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8937 .await
8938 .unwrap()
8939 .expect("the history read reached its pause");
8940 let elapsed = spawned_at.elapsed();
8941 assert!(
8942 elapsed < Duration::from_secs(2) && !read.is_finished(),
8943 "this task could not run while the history read was paused \
8944 (resumed after {elapsed:?}, read finished: {})",
8945 read.is_finished()
8946 );
8947
8948 drop(release);
8949 let responses = read.await.unwrap().unwrap();
8950 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8951 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8952 panic!("expected supervisor.terminals response");
8953 };
8954 assert_eq!(terminals.entries.len(), 2);
8955 assert_eq!(terminals.journal_skipped_lines, 0);
8956 assert_eq!(terminals.journal_read_errors, 0);
8957 }
8958
8959 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8960 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8961 let registry = Arc::new(Registry::default());
8962 let supervisor_handle = SupervisorHandle::new();
8963 let supervisor =
8964 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8965 .with_handle(supervisor_handle.clone());
8966 let module = supervisor
8967 .spawn(ModuleSpec {
8968 module_id: "terminal-golden".to_string(),
8969 program: fake_aft_stub_path(),
8970 args: Vec::new(),
8971 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8972 reserved: false,
8973 reserved_prefixes: Vec::new(),
8974 protocol: ModuleProtocol::Subc,
8975 overlap: Default::default(),
8976 })
8977 .unwrap();
8978
8979 let deadline = Instant::now() + Duration::from_secs(5);
8980 while module.terminal_history().entries.len() != 2 {
8981 assert!(
8982 Instant::now() < deadline,
8983 "module did not retain two terminal exits: {:?}",
8984 module.terminal_history()
8985 );
8986 sleep(Duration::from_millis(10)).await;
8987 }
8988
8989 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8990 let request = ClientControlRequest::SupervisorTerminals {
8991 module_id: "terminal-golden".to_string(),
8992 };
8993 let frame = Frame::build(
8994 FrameType::Request,
8995 control_flags(),
8996 0,
8997 0,
8998 1,
8999 serde_json::to_vec(&request).unwrap(),
9000 )
9001 .unwrap();
9002 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9003 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9004 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9005 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9006 panic!("expected supervisor.terminals response");
9007 };
9008 assert_eq!(terminals.entries.len(), 2);
9009 assert_eq!(terminals.dropped, 0);
9010
9011 let mut rendered = serde_json::to_value(response).unwrap();
9012 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9015 for (index, entry) in rendered["entries"]
9016 .as_array_mut()
9017 .expect("terminal response entries array")
9018 .iter_mut()
9019 .enumerate()
9020 {
9021 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9022 }
9023
9024 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9025 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9026 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9027 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9028 std::fs::write(&golden_path, &serialized).unwrap();
9029 }
9030 let expected: Value =
9031 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9032 assert_eq!(rendered, expected);
9033 }
9034
9035 #[test]
9036 fn hello_registers_manifest_and_returns_ack() {
9037 let registry = Arc::new(Registry::default());
9038 let handler = ControlHandler::new(Arc::clone(®istry));
9039 let conn = ConnectionId::new(1);
9040
9041 let responses = handler
9042 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9043 .unwrap();
9044
9045 assert_eq!(responses.len(), 1);
9046 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9047 assert_eq!(responses[0].header.channel, 0);
9048 assert_eq!(responses[0].header.corr, 7);
9049 let ack = parse_ack(&responses[0]);
9050 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9051 assert!(ack
9052 .subc_capabilities
9053 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9054 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9055 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9056 assert!(ack
9057 .subc_ops
9058 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9059 assert!(ack
9060 .subc_ops
9061 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9062
9063 let registration = registry.get_module("aft").unwrap().unwrap();
9064 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9065 assert_eq!(registration.state, ChannelState::Active);
9066 assert_eq!(registration.connection_id, conn);
9067 assert_eq!(registration.control_ops, module_baseline_control_ops());
9068 }
9069
9070 #[test]
9071 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9072 let invalid_identifiers = [
9073 ("case_change", "credentials-Provider/v1"),
9074 ("leading_zero", "credentials-provider/v01"),
9075 ("trailing_hyphen", "credentials-provider-/v1"),
9076 ("consecutive_hyphens", "credentials--provider/v1"),
9077 ("uppercase", "Credentials-provider/v1"),
9078 ("missing_v", "credentials-provider/1"),
9079 ("whitespace", "credentials provider/v1"),
9080 ("zero_version", "credentials-provider/v0"),
9081 ("out_of_range_version", "credentials-provider/v4294967296"),
9082 (
9083 "overlength_name",
9084 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9085 ),
9086 ];
9087 let mut cases = invalid_identifiers
9088 .into_iter()
9089 .map(|(name, identifier)| {
9090 (
9091 format!("identifier_{name}"),
9092 "capabilities.provides[0]".to_string(),
9093 identifier.to_string(),
9094 json!({ "provides": [identifier] }),
9095 None,
9096 )
9097 })
9098 .collect::<Vec<_>>();
9099 cases.extend([
9100 (
9101 "unknown_need".to_string(),
9102 "capabilities.requires[0].need".to_string(),
9103 "deferred".to_string(),
9104 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9105 None,
9106 ),
9107 (
9108 "duplicate_provides".to_string(),
9109 "capabilities.provides[1]".to_string(),
9110 "credentials-provider/v1".to_string(),
9111 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9112 None,
9113 ),
9114 (
9115 "duplicate_must_never_reach".to_string(),
9116 "capabilities.must_never_reach[1]".to_string(),
9117 "credentials-provider/v1".to_string(),
9118 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9119 None,
9120 ),
9121 (
9122 "duplicate_requires_same_need".to_string(),
9123 "capabilities.requires[1]".to_string(),
9124 "credentials-provider/v1".to_string(),
9125 json!({ "requires": [
9126 { "capability": "credentials-provider/v1", "need": "required" },
9127 { "capability": "credentials-provider/v1", "need": "required" }
9128 ] }),
9129 None,
9130 ),
9131 (
9132 "duplicate_requires_conflicting_need".to_string(),
9133 "capabilities.requires[1]".to_string(),
9134 "credentials-provider/v1".to_string(),
9135 json!({ "requires": [
9136 { "capability": "credentials-provider/v1", "need": "required" },
9137 { "capability": "credentials-provider/v1", "need": "optional" }
9138 ] }),
9139 None,
9140 ),
9141 (
9142 "capabilities_root_pointer".to_string(),
9143 "runtime_computed[0]".to_string(),
9144 "/capabilities".to_string(),
9145 json!({}),
9146 Some(json!(["/capabilities"])),
9147 ),
9148 (
9149 "capabilities_descendant_pointer".to_string(),
9150 "runtime_computed[0]".to_string(),
9151 "/capabilities/provides".to_string(),
9152 json!({}),
9153 Some(json!(["/capabilities/provides"])),
9154 ),
9155 (
9156 "malformed_pointer_without_leading_slash".to_string(),
9157 "runtime_computed[0]".to_string(),
9158 "capabilities".to_string(),
9159 json!({}),
9160 Some(json!(["capabilities"])),
9161 ),
9162 (
9163 "malformed_pointer_escape".to_string(),
9164 "runtime_computed[0]".to_string(),
9165 "/roles/~2/tools".to_string(),
9166 json!({}),
9167 Some(json!(["/roles/~2/tools"])),
9168 ),
9169 (
9170 "unknown_capabilities_field".to_string(),
9171 "capabilities.future".to_string(),
9172 "<array>".to_string(),
9173 json!({ "future": [] }),
9174 None,
9175 ),
9176 ]);
9177
9178 for (index, (name, field, value, capabilities, runtime_computed)) in
9179 cases.into_iter().enumerate()
9180 {
9181 let registry = Arc::new(Registry::default());
9182 let handler = ControlHandler::new(Arc::clone(®istry));
9183 let response = handler
9184 .handle_control(
9185 ConnectionId::new((index + 1) as u64),
9186 capability_grammar_hello_frame(
9187 capabilities,
9188 runtime_computed,
9189 index as u64 + 1,
9190 ),
9191 )
9192 .expect("invalid HELLO returns a refusal");
9193
9194 assert_eq!(response.len(), 1, "{name} must emit one refusal");
9195 let error = parse_error(&response[0]);
9196 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9197 let message = error["message"]
9198 .as_str()
9199 .expect("error message is a string");
9200 assert!(
9201 message.contains(&field),
9202 "{name}: field missing from {message}"
9203 );
9204 assert!(
9205 message.contains(&value),
9206 "{name}: value missing from {message}"
9207 );
9208 assert_eq!(
9209 registry
9210 .active_registration_count()
9211 .expect("registry reads"),
9212 0,
9213 "{name}: refused HELLO must not create a catalog entry"
9214 );
9215 }
9216 }
9217
9218 #[test]
9219 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9220 let registry = Arc::new(Registry::default());
9221 let handler = ControlHandler::new(Arc::clone(®istry));
9222 let capabilities = json!({
9223 "provides": ["credentials-provider/v1"],
9224 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9225 "must_never_reach": ["federation-transport/v1"]
9226 });
9227 let response = handler
9228 .handle_control(
9229 ConnectionId::new(99),
9230 capability_grammar_hello_frame(
9231 capabilities.clone(),
9232 Some(json!(["/roles/0/tools"])),
9233 99,
9234 ),
9235 )
9236 .expect("valid HELLO registers");
9237 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9238
9239 let request = Frame::build(
9240 FrameType::Request,
9241 control_flags(),
9242 0,
9243 0,
9244 100,
9245 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9246 .expect("catalog request serializes"),
9247 )
9248 .expect("catalog request frame builds");
9249 let response = handler
9250 .handle_catalog_list(request, None)
9251 .expect("catalog list succeeds");
9252 let ClientControlResponse::CatalogList { modules, .. } =
9253 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9254 else {
9255 panic!("catalog request must return catalog.list");
9256 };
9257 assert_eq!(modules.len(), 1);
9258 assert_eq!(
9259 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9260 capabilities
9261 );
9262 }
9263
9264 #[test]
9265 fn catalog_list_mirrors_management_operation_description() {
9266 let registry = Arc::new(Registry::default());
9267 let handler = ControlHandler::new(Arc::clone(®istry));
9268 let description = "List managed records and return their identifiers and metadata.";
9269 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9270 manifest.provides = vec![ProviderRole::ManagementSurface {
9271 operations: vec![ManagementOperation {
9272 name: "records.list".to_string(),
9273 kind: ManagementOperationKind::Query,
9274 description: Some(description.to_string()),
9275 }],
9276 config_schema: json!({"type": "object"}),
9277 observability: vec![ObservabilitySurface {
9278 name: "records.stats".to_string(),
9279 kind: ObservabilityKind::Snapshot,
9280 }],
9281 identity_scope: vec![IdentityScope::Project],
9282 concurrency: Concurrency::ModuleManaged,
9283 }];
9284 registry
9285 .register_with_control_ops(
9286 manifest,
9287 PROTOCOL_VERSION,
9288 ConnectionId::new(99),
9289 Vec::new(),
9290 )
9291 .expect("described management manifest registers");
9292
9293 let request = Frame::build(
9294 FrameType::Request,
9295 control_flags(),
9296 0,
9297 0,
9298 100,
9299 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9300 .expect("catalog request serializes"),
9301 )
9302 .expect("catalog request frame builds");
9303 let response = handler
9304 .handle_catalog_list(request, None)
9305 .expect("catalog list succeeds");
9306 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9307 assert_eq!(
9308 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9309 "catalog.list must preserve the declared operation description verbatim"
9310 );
9311 }
9312
9313 #[test]
9314 fn catalog_list_mirrors_registered_operation_delivery() {
9315 let registry = Arc::new(Registry::default());
9316 let handler = ControlHandler::new(Arc::clone(®istry));
9317 let delivery =
9318 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9319 let mut body = serde_json::to_value(ModuleHelloBody {
9320 manifest: manifest("watch-management", PROTOCOL_VERSION),
9321 protocol_ver: PROTOCOL_VERSION,
9322 control_ops: None,
9323 launch_nonce: None,
9324 })
9325 .unwrap();
9326 body["manifest"]["provides"] = json!([{
9327 "role": "management_surface",
9328 "operations": [
9329 {"name": "status", "kind": "query"},
9330 {"name": "future", "kind": "query"},
9331 {"name": "reset", "kind": "mutate"}
9332 ],
9333 "config_schema": {}, "observability": [], "identity_scope": []
9334 }]);
9335 body["manifest"]["operation_delivery"] = delivery.clone();
9336 let hello = Frame::build(
9337 FrameType::Hello,
9338 control_flags(),
9339 0,
9340 0,
9341 99,
9342 serde_json::to_vec(&body).unwrap(),
9343 )
9344 .unwrap();
9345 let response = handler
9346 .handle_control(ConnectionId::new(99), hello)
9347 .unwrap();
9348 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9349
9350 let (_, registrations) = registry.list_modules().unwrap();
9351 let registered = ®istrations[0].manifest;
9352 assert!(registered.is_watch_operation("status"));
9353 assert!(!registered.is_watch_operation("future"));
9354 assert!(!registered.is_watch_operation("reset"));
9355 assert!(!registered.is_watch_operation("missing"));
9356
9357 let request = Frame::build(
9358 FrameType::Request,
9359 control_flags(),
9360 0,
9361 0,
9362 100,
9363 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9364 )
9365 .unwrap();
9366 let response = handler.handle_catalog_list(request, None).unwrap();
9367 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9368 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9369 let ClientControlResponse::CatalogList { modules, .. } =
9370 serde_json::from_slice(&response[0].body).unwrap()
9371 else {
9372 panic!("expected catalog.list");
9373 };
9374 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9375 }
9376
9377 #[test]
9378 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
9379 let registry = Arc::new(Registry::default());
9380 let supervisor = SupervisorHandle::new();
9381 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9382 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9383 let descriptor = json!({
9384 "schema": "ckios.richui.manifest/1",
9385 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9386 "bytes": 4096,
9387 "chunks": 4
9388 });
9389 let connection = ConnectionId::new(201);
9390 let hello = handler
9391 .handle_control(
9392 connection,
9393 hello_frame_with_rich_ui(
9394 "vault",
9395 PROTOCOL_VERSION,
9396 201,
9397 Some("the-real-nonce"),
9398 descriptor.clone(),
9399 ),
9400 )
9401 .unwrap();
9402 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9403
9404 let request =
9405 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
9406 let provides = registry
9407 .get_module_by_connection(connection)
9408 .unwrap()
9409 .unwrap()
9410 .manifest
9411 .provides;
9412 handler
9413 .handle_catalog_update(connection, request, provides, None, Some(false))
9414 .unwrap();
9415
9416 let request = Frame::build(
9417 FrameType::Request,
9418 control_flags(),
9419 0,
9420 0,
9421 203,
9422 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9423 )
9424 .unwrap();
9425 let response = handler.handle_catalog_list(request, None).unwrap();
9426 let ClientControlResponse::CatalogList { modules, .. } =
9427 serde_json::from_slice(&response[0].body).unwrap()
9428 else {
9429 panic!("expected catalog.list");
9430 };
9431 assert_eq!(
9432 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
9433 descriptor
9434 );
9435 assert!(modules[0].nonce_verified);
9436 }
9437
9438 fn hello_with_role_declarations(declarations: Value) -> Frame {
9439 let mut body = serde_json::to_value(ModuleHelloBody {
9440 manifest: manifest("role-test", PROTOCOL_VERSION),
9441 protocol_ver: PROTOCOL_VERSION,
9442 control_ops: None,
9443 launch_nonce: None,
9444 })
9445 .unwrap();
9446 body["manifest"]["role_declarations"] = declarations;
9447 Frame::build(
9448 FrameType::Hello,
9449 control_flags(),
9450 0,
9451 0,
9452 301,
9453 serde_json::to_vec(&body).unwrap(),
9454 )
9455 .unwrap()
9456 }
9457
9458 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
9459 let request = Frame::build(
9460 FrameType::Request,
9461 control_flags(),
9462 0,
9463 0,
9464 302,
9465 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9466 )
9467 .unwrap();
9468 let response = handler.handle_catalog_list(request, None).unwrap();
9469 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
9470 let ClientControlResponse::CatalogList { mut modules, .. } =
9471 serde_json::from_slice(&response[0].body).unwrap()
9472 else {
9473 panic!("expected catalog.list");
9474 };
9475 (raw["modules"][0].clone(), modules.remove(0))
9476 }
9477
9478 #[test]
9479 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
9480 let registry = Arc::new(Registry::default());
9481 let handler = ControlHandler::new(Arc::clone(®istry));
9482 let declarations = json!({
9483 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
9484 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
9485 });
9486 let connection = ConnectionId::new(301);
9487 let response = handler
9488 .handle_control(
9489 connection,
9490 hello_with_role_declarations(declarations.clone()),
9491 )
9492 .unwrap();
9493 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9494 let registered = registry
9495 .get_module_by_connection(connection)
9496 .unwrap()
9497 .unwrap()
9498 .manifest;
9499 assert_eq!(
9503 serde_json::to_value(®istered.role_declarations).unwrap(),
9504 declarations
9505 );
9506 let (raw, entry) = role_declarations_catalog(&handler);
9507 assert_eq!(raw["role_declarations"], declarations);
9508 assert_eq!(entry.role_declarations, registered.role_declarations);
9509
9510 let request =
9511 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
9512 handler
9513 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
9514 .unwrap();
9515 let (raw, entry) = role_declarations_catalog(&handler);
9516 assert_eq!(raw["role_declarations"], declarations);
9517 assert_eq!(entry.role_declarations, registered.role_declarations);
9518 }
9519
9520 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
9521 let registry = Arc::new(Registry::default());
9522 let handler = ControlHandler::new(Arc::clone(®istry));
9523 let capture = EventCapture::default();
9524 let _subscriber =
9525 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9526 let response = handler
9527 .handle_control(
9528 ConnectionId::new(301),
9529 hello_with_role_declarations(declarations),
9530 )
9531 .unwrap();
9532 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9533 assert!(registry
9534 .get_module_by_connection(ConnectionId::new(301))
9535 .unwrap()
9536 .unwrap()
9537 .manifest
9538 .role_declarations
9539 .is_empty());
9540 let (raw, entry) = role_declarations_catalog(&handler);
9541 assert!(raw.get("role_declarations").is_none());
9542 assert!(entry.role_declarations.is_empty());
9543 let warnings: Vec<_> = capture
9544 .events()
9545 .into_iter()
9546 .filter(|event| {
9547 event.fields.get("message").map(String::as_str)
9548 == Some("dropping whole role_declarations map from HELLO")
9549 })
9550 .collect();
9551 assert_eq!(warnings.len(), 1);
9552 assert_eq!(warnings[0].level, tracing::Level::WARN);
9553 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
9554 assert_eq!(warnings[0].fields["reason"], reason);
9555 }
9556
9557 #[test]
9558 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
9559 for value in [
9560 json!(null),
9561 json!(false),
9562 json!(0),
9563 json!("opaque"),
9564 json!([]),
9565 ] {
9566 assert_hello_drops_role_declarations_with_warning(
9567 json!({"routing/v1": {}, "broken/v1": value}),
9568 "role_declarations values must be objects",
9569 );
9570 }
9571 assert_hello_drops_role_declarations_with_warning(
9572 json!([]),
9573 "role_declarations must be an object",
9574 );
9575 }
9576
9577 #[test]
9578 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
9579 assert_hello_drops_role_declarations_with_warning(
9580 json!({"routing/v1": {}, "bad key": {}}),
9581 "role_declarations key must use capability-name grammar",
9582 );
9583 }
9584
9585 #[test]
9586 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
9587 assert_hello_drops_role_declarations_with_warning(
9588 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
9589 "role_declarations exceeds 16384 compact JSON bytes",
9590 );
9591 }
9592
9593 #[test]
9594 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
9595 let registry = Arc::new(Registry::default());
9596 let handler = ControlHandler::new(registry.clone());
9597 let descriptor = json!({
9598 "schema": "ckios.richui.manifest/1",
9599 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9600 "bytes": 128,
9601 "chunks": 2
9602 });
9603 let malformed = [
9604 ("schema", json!("")),
9605 ("schema", json!("a".repeat(65))),
9606 ("schema", json!("Bad schema")),
9607 (
9608 "manifest_digest",
9609 json!(format!("sha256:{}", "A".repeat(64))),
9610 ),
9611 ("manifest_digest", json!("sha256:abc")),
9612 ("bytes", json!(0)),
9613 ("bytes", json!(16_777_217)),
9614 ("bytes", json!("128")),
9615 ("chunks", json!(0)),
9616 ("chunks", json!(1025)),
9617 ("chunks", json!("2")),
9618 ];
9619 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
9620 let mut invalid = descriptor.clone();
9621 invalid[field] = invalid_value;
9622 let response = handler
9623 .handle_control(
9624 ConnectionId::new(210 + index as u64),
9625 hello_frame_with_rich_ui(
9626 &format!("bad-rich-ui-{index}"),
9627 PROTOCOL_VERSION,
9628 210 + index as u64,
9629 None,
9630 invalid,
9631 ),
9632 )
9633 .unwrap();
9634 assert_eq!(response[0].header.ty, FrameType::Error);
9635 let error = parse_error(&response[0]);
9636 assert_eq!(error["code"], "invalid_manifest");
9637 assert!(error["message"]
9638 .as_str()
9639 .unwrap()
9640 .contains(&format!("rich_ui.{field}")));
9641 assert!(registry
9642 .get_module(&format!("bad-rich-ui-{index}"))
9643 .unwrap()
9644 .is_none());
9645 }
9646 }
9647
9648 #[test]
9649 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
9650 let registry = Arc::new(Registry::default());
9651 let handler = ControlHandler::new(Arc::clone(®istry));
9652 let connection = ConnectionId::new(219);
9653 registry
9654 .register_with_control_ops(
9655 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
9656 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
9657 "bad schema",
9658 format!("sha256:{}", "a".repeat(64)),
9659 128,
9660 2,
9661 )))
9662 .build(),
9663 PROTOCOL_VERSION,
9664 connection,
9665 Vec::new(),
9666 )
9667 .unwrap();
9668 let request =
9669 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
9670 let response = handler
9671 .handle_catalog_update(connection, request, Vec::new(), None, None)
9672 .unwrap();
9673 assert_eq!(response[0].header.ty, FrameType::Error);
9674 let error = parse_error(&response[0]);
9675 assert_eq!(error["code"], "invalid_manifest");
9676 assert!(error["message"]
9677 .as_str()
9678 .unwrap()
9679 .contains("rich_ui.schema"));
9680 }
9681
9682 #[test]
9683 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
9684 let registry = Arc::new(Registry::default());
9685 let supervisor = SupervisorHandle::new();
9686 supervisor.open_swap("provider", "candidate-nonce".to_string());
9687 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9688 let connection = ConnectionId::new(218);
9689 let hello = handler
9690 .handle_control(
9691 connection,
9692 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
9693 )
9694 .unwrap();
9695 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9696 assert!(registry.list_modules().unwrap().1.is_empty());
9697
9698 registry.promote_candidate("provider").unwrap().unwrap();
9699 let request = Frame::build(
9700 FrameType::Request,
9701 control_flags(),
9702 0,
9703 0,
9704 219,
9705 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9706 )
9707 .unwrap();
9708 let response = handler.handle_catalog_list(request, None).unwrap();
9709 let ClientControlResponse::CatalogList { modules, .. } =
9710 serde_json::from_slice(&response[0].body).unwrap()
9711 else {
9712 panic!("expected catalog.list");
9713 };
9714 assert_eq!(modules.len(), 1);
9715 assert!(modules[0].nonce_verified);
9716 }
9717
9718 #[test]
9719 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
9720 let registry = Arc::new(Registry::default());
9721 let handler = ControlHandler::new(Arc::clone(®istry));
9722 let hello = Frame::build(
9723 FrameType::Hello,
9724 control_flags(),
9725 0,
9726 0,
9727 220,
9728 serde_json::to_vec(&ModuleHelloBody {
9729 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
9730 protocol_ver: PROTOCOL_VERSION,
9731 control_ops: None,
9732 launch_nonce: Some("unverified".to_string()),
9733 })
9734 .unwrap(),
9735 )
9736 .unwrap();
9737 assert_eq!(
9738 handler
9739 .handle_control(ConnectionId::new(220), hello)
9740 .unwrap()[0]
9741 .header
9742 .ty,
9743 FrameType::HelloAck
9744 );
9745 let request = Frame::build(
9746 FrameType::Request,
9747 control_flags(),
9748 0,
9749 0,
9750 221,
9751 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9752 )
9753 .unwrap();
9754 let response = handler.handle_catalog_list(request, None).unwrap();
9755 let ClientControlResponse::CatalogList { modules, .. } =
9756 serde_json::from_slice(&response[0].body).unwrap()
9757 else {
9758 panic!("expected catalog.list");
9759 };
9760 assert_eq!(
9761 serde_json::to_vec(&modules[0]).unwrap(),
9762 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
9763 );
9764 assert!(!modules[0].nonce_verified);
9765 }
9766
9767 #[test]
9768 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
9769 let registry = Arc::new(Registry::default());
9770 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9771 [("vault".to_string(), true), ("squatter".to_string(), true)],
9772 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9773 );
9774 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
9775 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9776 provides: vec!["credentials-provider/v1".to_string()],
9777 requires: Vec::new(),
9778 must_never_reach: Vec::new(),
9779 });
9780 let frame = Frame::build(
9781 FrameType::Hello,
9782 control_flags(),
9783 0,
9784 0,
9785 77,
9786 serde_json::to_vec(&ModuleHelloBody {
9787 manifest: squatter,
9788 protocol_ver: PROTOCOL_VERSION,
9789 control_ops: None,
9790 launch_nonce: None,
9791 })
9792 .expect("HELLO serializes"),
9793 )
9794 .expect("HELLO frame builds");
9795 let response = handler
9796 .handle_control(ConnectionId::new(77), frame)
9797 .expect("reserved claim receives a typed refusal");
9798 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9799 assert_eq!(
9800 registry
9801 .active_registration_count()
9802 .expect("registry reads"),
9803 0,
9804 "a reserved capability refusal must not leave a catalog entry"
9805 );
9806 }
9807
9808 #[test]
9809 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
9810 for settlement in ["timeout", "inconclusive", "drop"] {
9811 let breakers = RouteBindBreakers::default();
9812 let RouteBindAdmission::Admitted {
9813 guard: mut old,
9814 probe: false,
9815 } = breakers.admit("prov")
9816 else {
9817 panic!("ordinary relay admitted")
9818 };
9819 let RouteBindAdmission::Admitted {
9820 guard: mut opener, ..
9821 } = breakers.admit("prov")
9822 else {
9823 panic!("second relay admitted")
9824 };
9825 assert!(
9826 !opener
9827 .record_timeout(1, Duration::ZERO)
9828 .unwrap()
9829 .reopened_after_probe
9830 );
9831 let RouteBindAdmission::Admitted {
9832 guard: mut probe,
9833 probe: true,
9834 } = breakers.admit("prov")
9835 else {
9836 panic!("one cooldown probe admitted")
9837 };
9838 match settlement {
9839 "timeout" => assert!(
9840 !old.record_timeout(1, Duration::ZERO)
9841 .unwrap()
9842 .reopened_after_probe
9843 ),
9844 "inconclusive" => old.record_inconclusive(),
9845 "drop" => drop(old),
9846 _ => unreachable!(),
9847 }
9848 assert!(
9849 matches!(
9850 breakers.admit("prov"),
9851 RouteBindAdmission::Refused {
9852 probe_in_flight: true,
9853 ..
9854 }
9855 ),
9856 "{settlement} of a pre-open relay cannot release the real probe"
9857 );
9858 assert!(
9859 probe
9860 .record_timeout(1, Duration::ZERO)
9861 .unwrap()
9862 .reopened_after_probe
9863 );
9864 assert!(matches!(
9865 breakers.admit("prov"),
9866 RouteBindAdmission::Admitted { probe: true, .. }
9867 ));
9868 }
9869 let breakers = RouteBindBreakers::default();
9870 let admit = || match breakers.admit("prov") {
9871 RouteBindAdmission::Admitted { guard, .. } => guard,
9872 _ => panic!("relay admitted"),
9873 };
9874 admit().record_timeout(1, Duration::ZERO);
9875 let mut old_probe = admit();
9876 breakers.reset_for_new_module_connection("prov");
9877 admit().record_timeout(1, Duration::ZERO);
9878 let _new_probe = admit();
9879 old_probe.record_inconclusive();
9880 assert!(matches!(
9881 breakers.admit("prov"),
9882 RouteBindAdmission::Refused {
9883 probe_in_flight: true,
9884 ..
9885 }
9886 ));
9887 }
9888
9889 #[tokio::test]
9890 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
9891 for candidate in [false, true] {
9892 let registry = Arc::new(Registry::default());
9893 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9894 [("vault".to_string(), true), ("squatter".to_string(), true)],
9895 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9896 );
9897 let conn = ConnectionId::new(77);
9898 let (ctx, mut rx) = route_ctx(conn);
9899 let initial = capability_manifest("squatter", &[], &[]);
9900 if candidate {
9901 registry
9902 .register_candidate_with_control_ops(
9903 initial.clone(),
9904 PROTOCOL_VERSION,
9905 conn,
9906 module_baseline_control_ops(),
9907 )
9908 .unwrap();
9909 handler
9910 .forwarding
9911 .register_candidate_module_connection(
9912 conn,
9913 "squatter".to_string(),
9914 PROTOCOL_VERSION,
9915 manifest_concurrency(&initial),
9916 ctx.egress.clone(),
9917 )
9918 .unwrap();
9919 } else {
9920 hello_via_sink(
9921 &handler,
9922 &ctx,
9923 &mut rx,
9924 hello_frame_with_manifest(initial.clone(), 1),
9925 )
9926 .await;
9927 }
9928 let response = handler
9929 .handle_control_frame(
9930 &ctx,
9931 catalog_update_with_capabilities_frame(
9932 2,
9933 capability_manifest("squatter", &["credentials-provider/v1"], &[])
9934 .capabilities
9935 .unwrap(),
9936 ),
9937 )
9938 .await
9939 .unwrap();
9940 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9941 assert_eq!(
9942 registry
9943 .get_module_by_connection(conn)
9944 .unwrap()
9945 .unwrap()
9946 .manifest,
9947 initial
9948 );
9949 }
9950 }
9951
9952 #[test]
9953 fn server_describe_surfaces_required_capability_verdict_fields() {
9954 let registry = Arc::new(Registry::default());
9955 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9956 [
9957 ("consumer".to_string(), true),
9958 ("provider".to_string(), false),
9959 ],
9960 BTreeMap::new(),
9961 );
9962 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9963 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9964 provides: Vec::new(),
9965 requires: vec![subc_protocol::manifest::CapabilityRequirement {
9966 capability: "credentials-provider/v1".to_string(),
9967 need: subc_protocol::manifest::CapabilityNeed::Required,
9968 }],
9969 must_never_reach: Vec::new(),
9970 });
9971 let hello = Frame::build(
9972 FrameType::Hello,
9973 control_flags(),
9974 0,
9975 0,
9976 78,
9977 serde_json::to_vec(&ModuleHelloBody {
9978 manifest: consumer,
9979 protocol_ver: PROTOCOL_VERSION,
9980 control_ops: None,
9981 launch_nonce: None,
9982 })
9983 .expect("HELLO serializes"),
9984 )
9985 .expect("HELLO frame builds");
9986 handler
9987 .handle_control(ConnectionId::new(78), hello)
9988 .expect("consumer registers");
9989 let describe = Frame::build(
9990 FrameType::Request,
9991 control_flags(),
9992 0,
9993 0,
9994 79,
9995 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
9996 .expect("request serializes"),
9997 )
9998 .expect("describe frame builds");
9999 let response = handler
10000 .handle_server_describe(describe)
10001 .expect("server.describe succeeds");
10002 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
10003 let requirement = &rendered["capability_requirements"][0];
10004 assert_eq!(requirement["consumer"], "consumer");
10005 assert_eq!(requirement["verdict"], "never_provided");
10006 assert_eq!(requirement["episode_seq"], 1);
10007 assert_eq!(requirement["config_satisfiable"], false);
10008 assert_eq!(requirement["runtime_available"], false);
10009 assert!(requirement["detail"]
10010 .as_str()
10011 .expect("detail string")
10012 .contains("credentials-provider/v1"));
10013 }
10014
10015 fn deregister_required_provider(
10021 daemon_shutdown_begun: bool,
10022 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
10023 const CAPABILITY: &str = "project-identity/v1";
10024 let registry = Arc::new(Registry::default());
10025 let daemon_shutdown = DaemonShutdownFlag::default();
10026 let handler = ControlHandler::new(Arc::clone(®istry))
10027 .with_capability_config(
10028 [
10029 ("consumer".to_string(), true),
10030 ("provider".to_string(), true),
10031 ],
10032 BTreeMap::new(),
10033 )
10034 .with_daemon_shutdown_flag(daemon_shutdown.clone());
10035 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10036 consumer.capabilities = Some(CapabilityDeclarations {
10037 provides: Vec::new(),
10038 requires: vec![subc_protocol::manifest::CapabilityRequirement {
10039 capability: CAPABILITY.to_string(),
10040 need: subc_protocol::manifest::CapabilityNeed::Required,
10041 }],
10042 must_never_reach: Vec::new(),
10043 });
10044 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
10045 let provider_conn = ConnectionId::new(182);
10046 for (conn, manifest, corr) in [
10047 (ConnectionId::new(181), consumer, 181),
10048 (provider_conn, provider, 182),
10049 ] {
10050 let hello = handler
10051 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
10052 .expect("module registers");
10053 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10054 }
10055 assert_eq!(
10056 handler.capability_evaluator.verdict("consumer", CAPABILITY),
10057 Some(CapabilityVerdict::Provided),
10058 "the registered provider must satisfy the requirement first"
10059 );
10060 if daemon_shutdown_begun {
10061 daemon_shutdown.set_for_test();
10062 }
10063
10064 let capture = EventCapture::default();
10065 let _subscriber =
10066 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10067 handler
10068 .cleanup_connection(provider_conn)
10069 .expect("provider deregisters");
10070 let events = capture
10071 .events()
10072 .into_iter()
10073 .filter(|event| {
10074 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
10075 })
10076 .collect();
10077 (
10078 events,
10079 handler.capability_evaluator.verdict("consumer", CAPABILITY),
10080 )
10081 }
10082
10083 #[test]
10084 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
10085 let (events, verdict) = deregister_required_provider(true);
10086 assert_eq!(
10087 verdict,
10088 Some(CapabilityVerdict::NeverProvided),
10089 "shutdown must not change the verdict route.open refusals read"
10090 );
10091 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10092 let event = &events[0];
10093 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
10094 assert_eq!(
10095 event.fields.get("cause").map(String::as_str),
10096 Some("\"daemon_shutdown\""),
10097 "{event:?}"
10098 );
10099 assert_eq!(
10100 event.fields.get("verdict").map(String::as_str),
10101 Some("\"never_provided\""),
10102 "{event:?}"
10103 );
10104 }
10105
10106 #[test]
10107 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
10108 let (events, verdict) = deregister_required_provider(false);
10109 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
10110 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10111 let event = &events[0];
10112 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
10113 assert_eq!(event.fields.get("cause"), None, "{event:?}");
10114 assert_eq!(
10115 event.fields.get("verdict").map(String::as_str),
10116 Some("\"never_provided\""),
10117 "{event:?}"
10118 );
10119 }
10120
10121 #[test]
10122 fn catalog_list_omits_capabilities_for_legacy_manifest() {
10123 let registry = Arc::new(Registry::default());
10124 let handler = ControlHandler::new(Arc::clone(®istry));
10125 let hello = handler
10126 .handle_control(
10127 ConnectionId::new(101),
10128 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
10129 )
10130 .expect("legacy HELLO registers");
10131 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10132
10133 let request = Frame::build(
10134 FrameType::Request,
10135 control_flags(),
10136 0,
10137 0,
10138 102,
10139 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10140 .expect("catalog request serializes"),
10141 )
10142 .expect("catalog request frame builds");
10143 let response = handler
10144 .handle_catalog_list(request, None)
10145 .expect("catalog list succeeds");
10146 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10147 assert!(
10148 body["modules"][0].get("capabilities").is_none(),
10149 "legacy manifest must retain an absent capabilities field on catalog.list"
10150 );
10151 }
10152
10153 #[test]
10154 fn hello_ack_omits_storage_when_no_storage_config() {
10155 let registry = Arc::new(Registry::default());
10156 let handler = ControlHandler::new(Arc::clone(®istry));
10157 let responses = handler
10158 .handle_control(
10159 ConnectionId::new(1),
10160 hello_frame("aft", PROTOCOL_VERSION, 7),
10161 )
10162 .unwrap();
10163 let ack = parse_ack(&responses[0]);
10164 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
10165 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
10166 }
10167
10168 #[tokio::test]
10169 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
10170 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
10171 let registry = Arc::new(Registry::default());
10172 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
10173 let responses = handler
10174 .handle_control(
10175 ConnectionId::new(1),
10176 hello_frame("aft", PROTOCOL_VERSION, 7),
10177 )
10178 .unwrap();
10179 let ack = parse_ack(&responses[0]);
10180 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
10181
10182 let described = handler
10183 .handle_control_frame(
10184 &route_ctx(ConnectionId::new(2)).0,
10185 Frame::build(
10186 FrameType::Request,
10187 control_flags(),
10188 0,
10189 0,
10190 9,
10191 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10192 )
10193 .unwrap(),
10194 )
10195 .await
10196 .unwrap();
10197 let ClientControlResponse::ServerDescribe { machine_id, .. } =
10198 serde_json::from_slice(&described[0].body).unwrap()
10199 else {
10200 panic!("server.describe answered with another shape");
10201 };
10202 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
10203 }
10204
10205 #[test]
10206 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
10207 let registry = Arc::new(Registry::default());
10210 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
10211 crate::daemon_config::StorageConfig::Sqlite {
10212 data_home: std::path::PathBuf::from("/data"),
10213 },
10214 ));
10215
10216 let responses = handler
10217 .handle_control(
10218 ConnectionId::new(1),
10219 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
10220 )
10221 .unwrap();
10222 let ack = parse_ack(&responses[0]);
10223 assert_eq!(
10224 ack.storage,
10225 Some(serde_json::json!({
10226 "module_id": "alfonso-routing",
10227 "storage_namespace": "default",
10228 "isolation": { "kind": "module" },
10229 "backend": {
10230 "backend": "sqlite",
10231 "path": "/data/cortexkit/alfonso-routing/store.db"
10232 }
10233 })),
10234 "the delivered descriptor is the module's own sqlite store path"
10235 );
10236 }
10237
10238 #[test]
10239 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
10240 let registry = Arc::new(Registry::default());
10241 let handler = ControlHandler::new(Arc::clone(®istry));
10242 let conn = ConnectionId::new(1);
10243 let responses = handler
10244 .handle_control(
10245 conn,
10246 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10247 )
10248 .unwrap();
10249 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
10250 let registration = registry.get_module("aft").unwrap().unwrap();
10251 assert_eq!(registration.control_ops, module_baseline_control_ops());
10252
10253 let frame =
10254 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
10255 assert!(handler
10256 .guard_module_control_op(&frame, "aft", "route.bind")
10257 .unwrap()
10258 .is_none());
10259 let error = handler
10260 .guard_module_control_op(&frame, "aft", "test.synthetic")
10261 .unwrap()
10262 .expect("synthetic ungranted op should be rejected");
10263 assert_eq!(error.header.ty, FrameType::Error);
10264 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
10265 }
10266
10267 #[test]
10268 fn hello_control_ops_some_adds_optional_grants() {
10269 let registry = Arc::new(Registry::default());
10270 let handler = ControlHandler::new(Arc::clone(®istry));
10271 handler
10272 .handle_control(
10273 ConnectionId::new(1),
10274 hello_frame_with_control_ops(
10275 "aft",
10276 PROTOCOL_VERSION,
10277 7,
10278 Some(vec![
10279 "future.synthetic".to_string(),
10280 "route.bind".to_string(),
10281 ]),
10282 ),
10283 )
10284 .unwrap();
10285 let registration = registry.get_module("aft").unwrap().unwrap();
10286 assert_eq!(
10287 registration.control_ops,
10288 vec![
10289 "route.bind".to_string(),
10290 "route.status".to_string(),
10291 "future.synthetic".to_string(),
10292 ]
10293 );
10294 let frame =
10295 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
10296 assert!(handler
10297 .guard_module_control_op(&frame, "aft", "future.synthetic")
10298 .unwrap()
10299 .is_none());
10300 }
10301
10302 #[tokio::test]
10303 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
10304 let registry = Arc::new(Registry::default());
10305 let forwarding = Arc::new(ForwardingTable::default());
10306 let handler =
10307 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10308 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
10309 hello_via_sink(
10310 &handler,
10311 &module_ctx,
10312 &mut module_rx,
10313 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10314 )
10315 .await;
10316
10317 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
10318 let responses = handler
10319 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
10320 .await
10321 .unwrap();
10322 assert_eq!(responses.len(), 1);
10323 assert_eq!(responses[0].header.ty, FrameType::Error);
10324 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
10325 assert!(module_rx.try_recv().is_err());
10326 }
10327
10328 #[tokio::test]
10329 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
10330 let registry = Arc::new(Registry::default());
10331 let forwarding = Arc::new(ForwardingTable::default());
10332 let handler =
10333 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10334 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
10335 hello_via_sink(
10336 &handler,
10337 &module_ctx,
10338 &mut module_rx,
10339 hello_frame_with_control_ops(
10340 "aft",
10341 PROTOCOL_VERSION,
10342 7,
10343 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10344 ),
10345 )
10346 .await;
10347
10348 let project_root = unique_project_root("demux");
10349 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
10350 let route_handler = handler.clone();
10351 let route_task = tokio::spawn(async move {
10352 route_handler
10353 .handle_control_frame(
10354 &route_client_ctx,
10355 route_open_frame(100, "aft", project_root),
10356 )
10357 .await
10358 .unwrap()
10359 });
10360 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10361 .await
10362 .unwrap()
10363 .unwrap();
10364 assert!(matches!(
10365 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
10366 ModuleControlRequest::RouteBind { .. }
10367 ));
10368
10369 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
10370 let health_handler = handler.clone();
10371 let health_task = tokio::spawn(async move {
10372 health_handler
10373 .handle_control_frame(
10374 &health_client_ctx,
10375 supervisor_health_probe_frame(101, "aft"),
10376 )
10377 .await
10378 .unwrap()
10379 });
10380 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10381 .await
10382 .unwrap()
10383 .unwrap();
10384 assert_eq!(
10385 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10386 ModuleControlRequest::HealthCheck {}
10387 );
10388
10389 handler
10390 .handle_control_frame(
10391 &module_ctx,
10392 health_response(health_frame.header.corr, HealthStatus::Degraded),
10393 )
10394 .await
10395 .unwrap();
10396 let health_response = health_task.await.unwrap();
10397 assert_eq!(health_response.len(), 1);
10398 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10399 ClientControlResponse::SupervisorHealthProbe {
10400 module_id,
10401 status,
10402 detail,
10403 metrics,
10404 } => {
10405 assert_eq!(module_id, "aft");
10406 assert_eq!(status, HealthStatus::Degraded);
10407 assert_eq!(detail.as_deref(), Some("warming"));
10408 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
10409 }
10410 other => panic!("unexpected health response: {other:?}"),
10411 }
10412
10413 handler
10414 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10415 .await
10416 .unwrap();
10417 let route_response = route_task.await.unwrap();
10418 assert!(route_response.is_empty());
10419 let published = route_client_rx.recv().await.unwrap();
10420 assert!(matches!(
10421 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10422 ClientControlResponse::RouteOpen { .. }
10423 ));
10424 }
10425
10426 async fn relay_route_open(
10431 handler: &ControlHandler,
10432 client_connection: ConnectionId,
10433 client_egress: &FrameSink,
10434 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10435 corr: u64,
10436 module_id: &str,
10437 project_root_label: &str,
10438 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
10439 let ctx = RouteCtx {
10440 connection_id: client_connection,
10441 egress: client_egress.clone(),
10442 };
10443 let handler = handler.clone();
10444 let project_root = unique_project_root(project_root_label);
10445 let module_id = module_id.to_string();
10446 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
10447 let task = tokio::spawn(async move {
10448 let _guard = tracing::dispatcher::set_default(&dispatch);
10449 handler
10450 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
10451 .await
10452 .unwrap()
10453 });
10454 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
10455 .await
10456 .expect("module receives the relayed route.bind")
10457 .expect("module egress is open");
10458 (task, bind.frame)
10459 }
10460
10461 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
10462 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
10463 ModuleControlRequest::RouteBind {
10464 route_channel,
10465 epoch,
10466 ..
10467 } => (route_channel, epoch),
10468 other => panic!("expected a route.bind request, got {other:?}"),
10469 }
10470 }
10471
10472 fn published_route(frame: &Frame) -> (u16, u32) {
10473 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
10474 ClientControlResponse::RouteOpen {
10475 route_channel,
10476 route_epoch,
10477 } => (route_channel, route_epoch),
10478 other => panic!("expected a route.open response, got {other:?}"),
10479 }
10480 }
10481
10482 #[tokio::test]
10502 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
10503 let registry = Arc::new(Registry::default());
10504 let forwarding = Arc::new(ForwardingTable::default());
10505 let handler =
10506 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10507
10508 let module_connection = ConnectionId::new(30);
10509 let (module_ctx, mut module_rx) = route_ctx(module_connection);
10510 hello_via_sink(
10511 &handler,
10512 &module_ctx,
10513 &mut module_rx,
10514 hello_frame("aft", PROTOCOL_VERSION, 7),
10515 )
10516 .await;
10517
10518 let dying_client = ConnectionId::new(31);
10519 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
10520
10521 let (first_task, first_bind) = relay_route_open(
10524 &handler,
10525 dying_client,
10526 &dying_ctx.egress,
10527 &mut module_rx,
10528 100,
10529 "aft",
10530 "closing-first",
10531 )
10532 .await;
10533 handler
10534 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
10535 .await
10536 .unwrap();
10537 assert!(first_task.await.unwrap().is_empty());
10538 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
10539
10540 let (second_task, second_bind) = relay_route_open(
10542 &handler,
10543 dying_client,
10544 &dying_ctx.egress,
10545 &mut module_rx,
10546 101,
10547 "aft",
10548 "closing-second",
10549 )
10550 .await;
10551 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
10552
10553 assert!(forwarding
10556 .escalate_client_delivery_failure(
10557 dying_client,
10558 first_channel,
10559 first_epoch,
10560 CloseReason::new(
10561 "module_to_client_delivery_failed",
10562 "client egress refused a module frame",
10563 ),
10564 crate::forwarding::UndeliveredFrame {
10565 module_id: None,
10566 sink: &dying_ctx.egress,
10567 },
10568 )
10569 .unwrap());
10570 assert!(!dying_ctx.egress.is_closed());
10571
10572 let ack = handler
10574 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
10575 .await;
10576 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10577 if module_loop_error.is_some() {
10578 handler.cleanup_connection(module_connection).unwrap();
10582 }
10583 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10587 .await
10588 .ok()
10589 .flatten();
10590
10591 assert!(
10593 registry
10594 .get_module_by_connection(module_connection)
10595 .unwrap()
10596 .is_some(),
10597 "one client's closing connection ended the shared module connection: \
10598 {module_loop_error:?}"
10599 );
10600 let cotenant = ConnectionId::new(32);
10602 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10603 let (cotenant_task, cotenant_bind) = relay_route_open(
10604 &handler,
10605 cotenant,
10606 &cotenant_ctx.egress,
10607 &mut module_rx,
10608 102,
10609 "aft",
10610 "closing-cotenant",
10611 )
10612 .await;
10613 handler
10614 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
10615 .await
10616 .unwrap();
10617 assert!(cotenant_task.await.unwrap().is_empty());
10618 let (cotenant_channel, cotenant_epoch) =
10619 published_route(&cotenant_rx.recv().await.unwrap());
10620 assert!(matches!(
10621 forwarding
10622 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10623 .unwrap(),
10624 DataRoute::Client(DataRouteState::Bound(_))
10625 ));
10626
10627 let goodbye = post_ack_module_frame
10630 .expect("module receives a GOODBYE for the abandoned route channel");
10631 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10632 assert_eq!(goodbye.header.channel, abandoned_channel);
10633 assert_eq!(goodbye.header.epoch, abandoned_epoch);
10634
10635 assert!(dying_rx.try_recv().is_err());
10639 let second_response = second_task.await.unwrap();
10640 assert_eq!(second_response.len(), 1);
10641 assert_eq!(
10642 parse_error(&second_response[0])["code"],
10643 "target_unavailable"
10644 );
10645 }
10646
10647 async fn catalog_snapshot(
10650 handler: &ControlHandler,
10651 client_ctx: &RouteCtx,
10652 corr: u64,
10653 ) -> (u64, Vec<(String, bool)>) {
10654 let body =
10655 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
10656 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
10657 let reply = handler
10658 .handle_control_frame(client_ctx, request)
10659 .await
10660 .unwrap()
10661 .pop()
10662 .unwrap();
10663 let ClientControlResponse::CatalogList {
10664 generation,
10665 modules,
10666 ..
10667 } = serde_json::from_slice(&reply.body).unwrap()
10668 else {
10669 panic!("not a catalog.list reply");
10670 };
10671 (
10672 generation,
10673 modules
10674 .into_iter()
10675 .map(|entry| (entry.module_id, entry.ready))
10676 .collect(),
10677 )
10678 }
10679
10680 fn hello_held_before_forwarding() -> (
10686 ControlHandler,
10687 Arc<std::sync::Barrier>,
10688 std::thread::JoinHandle<Vec<Frame>>,
10689 RouteCtx,
10690 mpsc::Receiver<crate::router::OutboundFrame>,
10691 ) {
10692 let hold = Arc::new(std::sync::Barrier::new(2));
10693 let handler = ControlHandler::with_forwarding(
10694 Arc::new(Registry::default()),
10695 Arc::new(ForwardingTable::default()),
10696 )
10697 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
10698 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10699 let hello = {
10700 let handler = handler.clone();
10701 let module_ctx = module_ctx.clone();
10702 std::thread::spawn(move || {
10703 handler
10704 .handle_hello(
10705 module_ctx.connection_id,
10706 Some(module_ctx.egress.clone()),
10707 hello_frame("aft", PROTOCOL_VERSION, 7),
10708 )
10709 .unwrap()
10710 })
10711 };
10712 (handler, hold, hello, module_ctx, module_rx)
10713 }
10714
10715 #[tokio::test]
10724 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
10725 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
10726 let client = ConnectionId::new(41);
10727 let (client_ctx, mut client_rx) = route_ctx(client);
10728
10729 hold.wait();
10730 assert!(handler.registry.get_module("aft").unwrap().is_some());
10732 assert!(!handler
10733 .forwarding
10734 .has_live_module_connection("aft")
10735 .unwrap());
10736 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
10737 assert!(
10738 listed.is_empty(),
10739 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
10740 );
10741 let refused = handler
10742 .handle_control_frame(
10743 &client_ctx,
10744 route_open_frame(301, "aft", unique_project_root("listed-window")),
10745 )
10746 .await
10747 .unwrap();
10748 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
10749 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
10750 let duplicate = handler
10751 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
10752 .await
10753 .unwrap();
10754 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10755
10756 hold.wait();
10757 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10758 let ack = module_rx.try_recv().unwrap().frame;
10759 assert_eq!(ack.header.ty, FrameType::HelloAck);
10760 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10761 assert_eq!(listed, vec![("aft".to_string(), true)]);
10762
10763 let (open_task, bind) = relay_route_open(
10764 &handler,
10765 client,
10766 &client_ctx.egress,
10767 &mut module_rx,
10768 303,
10769 "aft",
10770 "listed-after",
10771 )
10772 .await;
10773 assert!(handler
10774 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10775 .await
10776 .unwrap()
10777 .is_empty());
10778 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
10779 published_route(&client_rx.try_recv().unwrap().frame);
10780 }
10781
10782 #[tokio::test]
10789 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
10790 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
10791 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10792 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10793
10794 hold.wait();
10795 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
10796 assert!(listed.is_empty());
10797 assert_eq!(
10798 during, before,
10799 "the generation moved while the module was not listed"
10800 );
10801
10802 hold.wait();
10803 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10804 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10805 assert_eq!(listed, vec![("aft".to_string(), true)]);
10806 assert_eq!(
10807 after,
10808 before + 1,
10809 "the module appeared without exactly one generation bump"
10810 );
10811 }
10812
10813 #[tokio::test]
10818 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
10819 let registry = Arc::new(Registry::default());
10820 let handler = ControlHandler::with_forwarding(
10821 Arc::clone(®istry),
10822 Arc::new(ForwardingTable::default()),
10823 );
10824 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10825 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10826 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10827 drop(module_rx);
10828
10829 let replies = handler
10830 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
10831 .await
10832 .unwrap();
10833 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
10834 assert!(registry.get_module("aft").unwrap().is_none());
10835 assert!(registry
10836 .get_module_by_connection(module_ctx.connection_id)
10837 .unwrap()
10838 .is_none());
10839 assert_eq!(
10840 catalog_snapshot(&handler, &client_ctx, 301).await,
10841 (before, Vec::new())
10842 );
10843 }
10844
10845 async fn route_open_held_after_bind_send() -> (
10849 ControlHandler,
10850 Arc<tokio::sync::Barrier>,
10851 RouteCtx,
10852 mpsc::Receiver<crate::router::OutboundFrame>,
10853 ) {
10854 let hold = Arc::new(tokio::sync::Barrier::new(2));
10855 let handler = ControlHandler::with_forwarding(
10856 Arc::new(Registry::default()),
10857 Arc::new(ForwardingTable::default()),
10858 )
10859 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
10860 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10861 hello_via_sink(
10862 &handler,
10863 &module_ctx,
10864 &mut module_rx,
10865 hello_frame("aft", PROTOCOL_VERSION, 7),
10866 )
10867 .await;
10868 (handler, hold, module_ctx, module_rx)
10869 }
10870
10871 #[tokio::test]
10878 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
10879 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10880 let client = ConnectionId::new(41);
10881 let (client_ctx, mut client_rx) = route_ctx(client);
10882
10883 let (open_task, bind) = relay_route_open(
10884 &handler,
10885 client,
10886 &client_ctx.egress,
10887 &mut module_rx,
10888 200,
10889 "aft",
10890 "acked-before-mark",
10891 )
10892 .await;
10893 let (module_channel, module_epoch) = route_bind_channel(&bind);
10894 hold.wait().await;
10896
10897 assert!(handler
10899 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10900 .await
10901 .unwrap()
10902 .is_empty());
10903 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
10904
10905 hold.wait().await;
10906 assert!(
10907 open_task.await.unwrap().is_empty(),
10908 "route.open answers accepted"
10909 );
10910
10911 match module_rx.try_recv() {
10914 Err(mpsc::error::TryRecvError::Empty) => {}
10915 Ok(outbound) => panic!(
10916 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
10917 outbound.frame.header.ty,
10918 outbound.frame.header.channel,
10919 outbound.frame.header.epoch
10920 ),
10921 Err(err) => panic!("module egress closed: {err}"),
10922 }
10923
10924 let router = Router::with_control_handler(Arc::new(handler.clone()));
10927 let request = Frame::build(
10928 FrameType::Request,
10929 Flags::new(true, Priority::Interactive, false),
10930 client_channel,
10931 client_epoch,
10932 201,
10933 b"first request".to_vec(),
10934 )
10935 .unwrap();
10936 router
10937 .route_for_connection(&client_ctx, request)
10938 .await
10939 .unwrap();
10940 let delivered = module_rx.try_recv().unwrap().frame;
10941 assert_eq!(delivered.header.ty, FrameType::Request);
10942 assert_eq!(
10943 (delivered.header.channel, delivered.header.epoch),
10944 (module_channel, module_epoch)
10945 );
10946 assert_eq!(delivered.body, b"first request");
10947 }
10948
10949 #[tokio::test]
10955 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
10956 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10957 let client = ConnectionId::new(42);
10958 let (client_ctx, _client_rx) = route_ctx(client);
10959
10960 let (open_task, bind) = relay_route_open(
10961 &handler,
10962 client,
10963 &client_ctx.egress,
10964 &mut module_rx,
10965 300,
10966 "aft",
10967 "released-before-mark",
10968 )
10969 .await;
10970 let (module_channel, module_epoch) = route_bind_channel(&bind);
10971 hold.wait().await;
10972
10973 handler.cleanup_connection(client).unwrap();
10974 assert!(
10975 module_rx.try_recv().is_err(),
10976 "client teardown must not GOODBYE a bind route.open is still queueing"
10977 );
10978
10979 hold.wait().await;
10980 let response = open_task.await.unwrap();
10981 assert_eq!(response.len(), 1);
10982 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10983
10984 let goodbye = module_rx
10985 .try_recv()
10986 .expect("module is told to drop the binding the daemon abandoned")
10987 .frame;
10988 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10989 assert_eq!(
10990 (goodbye.header.channel, goodbye.header.epoch),
10991 (module_channel, module_epoch)
10992 );
10993 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10994 }
10995
10996 #[tokio::test]
11001 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
11002 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11003 let client = ConnectionId::new(43);
11004 let (client_ctx, client_rx) = route_ctx(client);
11005
11006 let (open_task, bind) = relay_route_open(
11007 &handler,
11008 client,
11009 &client_ctx.egress,
11010 &mut module_rx,
11011 400,
11012 "aft",
11013 "gone-client-before-mark",
11014 )
11015 .await;
11016 let (module_channel, module_epoch) = route_bind_channel(&bind);
11017 hold.wait().await;
11018
11019 drop(client_rx);
11020 assert!(client_ctx.egress.is_closed());
11021 handler
11022 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11023 .await
11024 .unwrap();
11025
11026 hold.wait().await;
11027 let response = open_task.await.unwrap();
11028 assert_eq!(response.len(), 1);
11029 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11030
11031 let goodbye = module_rx
11032 .try_recv()
11033 .expect("module is told to drop the binding it acked")
11034 .frame;
11035 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11036 assert_eq!(
11037 (goodbye.header.channel, goodbye.header.epoch),
11038 (module_channel, module_epoch)
11039 );
11040 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11041 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11042 }
11043
11044 #[test]
11051 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
11052 let handler = ControlHandler::default();
11053 let module_connection = ConnectionId::new(30);
11054 let client_connection = ConnectionId::new(31);
11055
11056 handler
11057 .refuse_to_end_module_connection_for_a_client(
11058 module_connection,
11059 77,
11060 ForwardingError::ConnectionClosing {
11061 connection_id: client_connection,
11062 },
11063 )
11064 .expect("a closing client must never end the module connection");
11065
11066 assert!(matches!(
11067 handler.refuse_to_end_module_connection_for_a_client(
11068 module_connection,
11069 78,
11070 ForwardingError::ConnectionClosing {
11071 connection_id: module_connection,
11072 },
11073 ),
11074 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
11075 connection_id
11076 })) if connection_id == module_connection
11077 ));
11078 assert!(matches!(
11079 handler.refuse_to_end_module_connection_for_a_client(
11080 module_connection,
11081 79,
11082 ForwardingError::Poisoned,
11083 ),
11084 Err(RouterError::Forwarding(ForwardingError::Poisoned))
11085 ));
11086 assert!(matches!(
11087 handler.refuse_to_end_module_connection_for_a_client(
11088 module_connection,
11089 80,
11090 ForwardingError::StaleModuleEndpoint,
11091 ),
11092 Err(RouterError::Forwarding(
11093 ForwardingError::StaleModuleEndpoint
11094 ))
11095 ));
11096 }
11097
11098 #[tokio::test]
11111 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
11112 let registry = Arc::new(Registry::default());
11113 let forwarding = Arc::new(ForwardingTable::default());
11114 let supervisor = SupervisorHandle::new();
11115 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11116 let handler =
11117 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11118 .with_supervisor(supervisor);
11119
11120 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11121 hello_via_sink(
11122 &handler,
11123 &target_ctx,
11124 &mut target_rx,
11125 hello_frame("target", PROTOCOL_VERSION, 1),
11126 )
11127 .await;
11128
11129 let wrong_nonce = handler
11133 .handle_control_frame(
11134 &route_ctx(ConnectionId::new(91)).0,
11135 route_open_frame_with_admission_facts(
11136 20,
11137 "target",
11138 unique_project_root("admission-facts"),
11139 Some(subc_control::ConsumerIdentity {
11140 module_id: "fed".to_string(),
11141 launch_nonce: "not-the-real-nonce".to_string(),
11142 }),
11143 None,
11144 ),
11145 )
11146 .await
11147 .unwrap();
11148 assert_eq!(
11149 parse_error(&wrong_nonce[0])["code"],
11150 "bad_consumer_identity",
11151 "a mismatched launch nonce must be refused, not stamped Reserved"
11152 );
11153
11154 let never_spawned = handler
11158 .handle_control_frame(
11159 &route_ctx(ConnectionId::new(92)).0,
11160 route_open_frame_with_admission_facts(
11161 21,
11162 "target",
11163 unique_project_root("admission-facts"),
11164 Some(subc_control::ConsumerIdentity {
11165 module_id: "never-spawned".to_string(),
11166 launch_nonce: "any-nonce".to_string(),
11167 }),
11168 None,
11169 ),
11170 )
11171 .await
11172 .unwrap();
11173 assert_eq!(
11174 parse_error(&never_spawned[0])["code"],
11175 "bad_consumer_identity",
11176 "an unspawned module_id must be refused rather than accepted for lack of a record"
11177 );
11178 }
11179
11180 #[tokio::test]
11203 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
11204 let registry = Arc::new(Registry::default());
11205 let forwarding = Arc::new(ForwardingTable::default());
11206 let supervisor = SupervisorHandle::new();
11207 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11208 let handler =
11209 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11210 .with_supervisor(supervisor);
11211
11212 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
11213 hello_via_sink(
11214 &handler,
11215 &target_ctx,
11216 &mut target_rx,
11217 hello_frame("target", PROTOCOL_VERSION, 1),
11218 )
11219 .await;
11220
11221 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
11222 let route_handler = handler.clone();
11223 let route_task = tokio::spawn(async move {
11224 route_handler
11225 .handle_control_frame(
11226 &client_ctx,
11227 route_open_frame_with_admission_facts(
11228 30,
11229 "target",
11230 unique_project_root("admission-facts"),
11231 Some(subc_control::ConsumerIdentity {
11232 module_id: "fed".to_string(),
11233 launch_nonce: "fed-nonce".to_string(),
11234 }),
11235 None,
11236 ),
11237 )
11238 .await
11239 .unwrap()
11240 });
11241
11242 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11249 .await
11250 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
11251 .expect("module control channel closed before route.bind");
11252 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11253 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11254 panic!("expected route.bind")
11255 };
11256 assert_eq!(
11257 principal,
11258 Some(Principal::Reserved {
11259 module_id: "fed".to_string()
11260 }),
11261 "a correctly attested consumer must be stamped Reserved for its own id"
11262 );
11263
11264 handler
11265 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11266 .await
11267 .unwrap();
11268 assert!(route_task.await.unwrap().is_empty());
11269 assert!(
11270 matches!(
11271 serde_json::from_slice::<ClientControlResponse>(
11272 &client_rx.recv().await.unwrap().body
11273 )
11274 .unwrap(),
11275 ClientControlResponse::RouteOpen { .. }
11276 ),
11277 "the route must actually open, not merely avoid an error"
11278 );
11279 }
11280
11281 #[tokio::test(start_paused = true)]
11282 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
11283 let registry = Arc::new(Registry::default());
11284 let forwarding = Arc::new(ForwardingTable::default());
11285 let supervisor = SupervisorHandle::new();
11286 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11287 let handler =
11288 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11289 .with_supervisor(supervisor);
11290
11291 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
11292 hello_via_sink(
11293 &handler,
11294 &target_ctx,
11295 &mut target_rx,
11296 hello_frame("target", PROTOCOL_VERSION, 1),
11297 )
11298 .await;
11299
11300 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
11301 let direct_handler = handler.clone();
11302 let direct_open = tokio::spawn(async move {
11303 direct_handler
11304 .handle_control_frame(
11305 &direct_ctx,
11306 route_open_frame(2, "target", unique_project_root("route-census-direct")),
11307 )
11308 .await
11309 .unwrap()
11310 });
11311 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11312 .await
11313 .expect("no direct route.bind within 5s")
11314 .expect("target control channel closed before direct route.bind");
11315 handler
11316 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
11317 .await
11318 .unwrap();
11319 assert!(direct_open.await.unwrap().is_empty());
11320 let _ = direct_rx.recv().await.unwrap();
11321
11322 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
11323 let reserved_handler = handler.clone();
11324 let reserved_open = tokio::spawn(async move {
11325 reserved_handler
11326 .handle_control_frame(
11327 &reserved_ctx,
11328 route_open_frame_with_admission_facts(
11329 3,
11330 "target",
11331 unique_project_root("admission-facts"),
11332 Some(ConsumerIdentity {
11333 module_id: "fed".to_string(),
11334 launch_nonce: "fed-nonce".to_string(),
11335 }),
11336 None,
11337 ),
11338 )
11339 .await
11340 .unwrap()
11341 });
11342 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11343 .await
11344 .expect("no reserved route.bind within 5s")
11345 .expect("target control channel closed before reserved route.bind");
11346 handler
11347 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
11348 .await
11349 .unwrap();
11350 assert!(reserved_open.await.unwrap().is_empty());
11351 let _ = reserved_rx.recv().await.unwrap();
11352
11353 forwarding
11354 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
11355 .unwrap();
11356 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
11357 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
11358 module_id: Some("target".to_string()),
11359 })
11360 .unwrap();
11361 let census_frame =
11362 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
11363 let response = handler
11364 .handle_control_frame(&census_ctx, census_frame)
11365 .await
11366 .unwrap()
11367 .pop()
11368 .unwrap();
11369 let actual: Value = serde_json::from_slice(&response.body).unwrap();
11370 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
11371 assert!(matches!(
11372 decoded,
11373 ClientControlResponse::SupervisorRoutes { .. }
11374 ));
11375 let routes = actual["modules"][0]["routes"].as_array().unwrap();
11376 assert_eq!(routes.len(), 2);
11377 assert!(routes.iter().all(|route| route["draining"] == true));
11378 assert!(
11381 routes.iter().all(|route| route["drain_reason"] == "reload"),
11382 "draining routes must name the drain's reason: {routes:?}"
11383 );
11384 assert!(routes.iter().any(|route| {
11385 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
11386 }));
11387 assert!(routes.iter().any(|route| {
11388 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
11389 }));
11390
11391 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11392 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
11393 if std::env::var_os("UPDATE_GOLDEN").is_some() {
11394 std::fs::write(
11395 &golden_path,
11396 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
11397 )
11398 .unwrap();
11399 }
11400 let expected: Value =
11401 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
11402 assert_eq!(actual, expected);
11403 }
11404
11405 async fn query_live_roots(
11406 handler: &ControlHandler,
11407 module_ctx: &RouteCtx,
11408 ) -> ModuleControlResponseToModule {
11409 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
11410 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
11411 let response = handler
11412 .handle_control_frame(module_ctx, frame)
11413 .await
11414 .unwrap()
11415 .pop()
11416 .unwrap();
11417 serde_json::from_slice(&response.body).unwrap()
11418 }
11419
11420 #[tokio::test(start_paused = true)]
11421 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
11422 let registry = Arc::new(Registry::default());
11423 let forwarding = Arc::new(ForwardingTable::default());
11424 let handler = ControlHandler::with_forwarding(registry, forwarding);
11425 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
11426 hello_via_sink(
11427 &handler,
11428 &target_ctx,
11429 &mut target_rx,
11430 hello_frame("target", PROTOCOL_VERSION, 1),
11431 )
11432 .await;
11433 let root = unique_project_root("live-roots-known");
11434 let path = ProjectRootId::from_path_allowing_missing(root.path())
11435 .unwrap()
11436 .as_path()
11437 .to_path_buf();
11438 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
11439 let open_handler = handler.clone();
11440 let opened = tokio::spawn(async move {
11441 open_handler
11442 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
11443 .await
11444 .unwrap()
11445 });
11446 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11447 .await
11448 .unwrap()
11449 .unwrap();
11450 handler
11451 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
11452 .await
11453 .unwrap();
11454 assert!(opened.await.unwrap().is_empty());
11455 let _ = client_rx.recv().await.unwrap();
11456
11457 let root = unique_project_root("live-roots-pending");
11458 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
11459 .unwrap()
11460 .as_path()
11461 .to_path_buf();
11462 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
11463 let open_handler = handler.clone();
11464 let pending = tokio::spawn(async move {
11465 open_handler
11466 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
11467 .await
11468 .unwrap()
11469 });
11470 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11471 .await
11472 .unwrap()
11473 .unwrap();
11474 let actual = query_live_roots(&handler, &target_ctx).await;
11475 let ModuleControlResponseToModule::LiveRoots {
11476 roots,
11477 unknown_root_bindings,
11478 total_bindings,
11479 } = actual
11480 else {
11481 panic!("expected live roots")
11482 };
11483 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
11484 assert_eq!(unknown_root_bindings, 0);
11485 assert_eq!(
11486 roots.len(),
11487 2,
11488 "root-known arm must retain each canonical root"
11489 );
11490 assert_eq!(
11491 total_bindings,
11492 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11493 );
11494 let counts = roots
11495 .iter()
11496 .map(|root| (root.project_root.clone(), root.bound, root.pending))
11497 .collect::<Vec<_>>();
11498 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
11499 expected.sort_by(|a, b| a.0.cmp(&b.0));
11500 assert_eq!(
11501 counts, expected,
11502 "roots must sort by path and count pending separately"
11503 );
11504 handler
11505 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
11506 .await
11507 .unwrap();
11508 assert!(pending.await.unwrap().is_empty());
11509 }
11510
11511 #[tokio::test(start_paused = true)]
11512 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
11513 let forwarding = Arc::new(ForwardingTable::default());
11514 let handler =
11515 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11516 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
11517 hello_via_sink(
11518 &handler,
11519 &target_ctx,
11520 &mut target_rx,
11521 hello_frame("target", PROTOCOL_VERSION, 1),
11522 )
11523 .await;
11524 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
11525 let pending = forwarding
11526 .begin_route_bind_relay_for_test(
11527 client_ctx.connection_id,
11528 client_ctx.egress.clone(),
11529 2,
11530 "target",
11531 )
11532 .unwrap();
11533 forwarding
11534 .complete_pending_relay(
11535 target_ctx.connection_id,
11536 pending.corr,
11537 RouteBindRelayOutcome::Accepted,
11538 )
11539 .unwrap();
11540 let actual = query_live_roots(&handler, &target_ctx).await;
11541 let ModuleControlResponseToModule::LiveRoots {
11542 roots,
11543 unknown_root_bindings,
11544 total_bindings,
11545 } = actual
11546 else {
11547 panic!("expected live roots")
11548 };
11549 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
11550 assert_eq!(
11551 unknown_root_bindings, 1,
11552 "unknown-root arm must not read as no bindings"
11553 );
11554 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
11555 assert_eq!(
11556 total_bindings,
11557 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11558 );
11559 }
11560
11561 #[tokio::test(start_paused = true)]
11569 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
11570 let forwarding = Arc::new(ForwardingTable::default());
11571 let handler =
11572 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11573 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
11574 let replies = handler
11575 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
11576 .await
11577 .unwrap();
11578 let queued_by_hello = module_rx.len();
11579
11580 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
11581 let open_handler = handler.clone();
11582 let open = tokio::spawn(async move {
11583 open_handler
11584 .handle_control_frame(
11585 &client_ctx,
11586 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
11587 )
11588 .await
11589 .unwrap()
11590 });
11591 let mut spins = 0;
11593 while module_rx.len() == queued_by_hello {
11594 spins += 1;
11595 assert!(spins < 10_000, "route.open never queued a route.bind");
11596 tokio::task::yield_now().await;
11597 }
11598
11599 for reply in replies {
11601 module_ctx.egress.send(reply).await.unwrap();
11602 }
11603
11604 let first = module_rx.recv().await.unwrap().frame;
11605 assert_eq!(
11606 first.header.ty,
11607 FrameType::HelloAck,
11608 "the first frame a registering module reads must be its HELLO_ACK"
11609 );
11610 assert_eq!(first.header.corr, 7);
11611 let second = module_rx.recv().await.unwrap().frame;
11612 assert_eq!(second.header.ty, FrameType::Request);
11613 assert!(
11614 matches!(
11615 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
11616 ModuleControlRequest::RouteBind { .. }
11617 ),
11618 "the route.bind follows the ack"
11619 );
11620 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
11621
11622 handler
11623 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
11624 .await
11625 .unwrap();
11626 assert!(open.await.unwrap().is_empty());
11627 let _ = client_rx.recv().await.unwrap();
11628 }
11629
11630 #[tokio::test(start_paused = true)]
11631 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
11632 let handler = ControlHandler::with_forwarding(
11633 Arc::new(Registry::default()),
11634 Arc::new(ForwardingTable::default()),
11635 );
11636 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
11637 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
11638 hello_via_sink(
11639 &handler,
11640 &first_ctx,
11641 &mut first_rx,
11642 hello_frame("first", PROTOCOL_VERSION, 1),
11643 )
11644 .await;
11645 hello_via_sink(
11646 &handler,
11647 &second_ctx,
11648 &mut second_rx,
11649 hello_frame("second", PROTOCOL_VERSION, 2),
11650 )
11651 .await;
11652 let root = unique_project_root("second-only");
11653 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
11654 let cloned = handler.clone();
11655 let open = tokio::spawn(async move {
11656 cloned
11657 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
11658 .await
11659 .unwrap()
11660 });
11661 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
11662 .await
11663 .unwrap()
11664 .unwrap();
11665 let first = query_live_roots(&handler, &first_ctx).await;
11666 let second = query_live_roots(&handler, &second_ctx).await;
11667 assert!(
11668 matches!(
11669 first,
11670 ModuleControlResponseToModule::LiveRoots {
11671 total_bindings: 0,
11672 ..
11673 }
11674 ),
11675 "cross-module scope must not expose another module's roots"
11676 );
11677 assert!(
11678 matches!(
11679 second,
11680 ModuleControlResponseToModule::LiveRoots {
11681 total_bindings: 1,
11682 ..
11683 }
11684 ),
11685 "second module must see its pending route"
11686 );
11687 handler
11688 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
11689 .await
11690 .unwrap();
11691 assert!(open.await.unwrap().is_empty());
11692 }
11693
11694 #[tokio::test(start_paused = true)]
11695 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
11696 let handler = ControlHandler::with_forwarding(
11697 Arc::new(Registry::default()),
11698 Arc::new(ForwardingTable::default()),
11699 );
11700 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
11701 hello_via_sink(
11702 &handler,
11703 &target_ctx,
11704 &mut target_rx,
11705 hello_frame("target", PROTOCOL_VERSION, 1),
11706 )
11707 .await;
11708 let actual = query_live_roots(&handler, &target_ctx).await;
11709 let ModuleControlResponseToModule::LiveRoots {
11710 roots,
11711 unknown_root_bindings,
11712 total_bindings,
11713 } = actual
11714 else {
11715 panic!("expected live roots")
11716 };
11717 assert!(roots.is_empty());
11718 assert_eq!(unknown_root_bindings, 0);
11719 assert_eq!(total_bindings, 0);
11720 assert_eq!(
11721 total_bindings,
11722 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11723 );
11724 }
11725
11726 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
11733 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11734 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
11735 let text = std::fs::read_to_string(&path)
11736 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
11737 let vectors: Vec<(String, Value)> = text
11738 .lines()
11739 .filter(|line| !line.trim().is_empty())
11740 .map(|line| {
11741 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
11742 let id = entry["corpus_id"]
11743 .as_str()
11744 .expect("every vector carries a corpus_id")
11745 .to_string();
11746 (id, entry["package"].clone())
11747 })
11748 .collect();
11749 assert_eq!(
11753 vectors.len(),
11754 3,
11755 "vendored fed corpus changed size; re-sync from subc-federation"
11756 );
11757
11758 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
11773 let richest = vectors
11774 .iter()
11775 .filter_map(|(_, package)| package.as_object())
11776 .map(|object| {
11777 object
11778 .keys()
11779 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
11780 .count()
11781 })
11782 .max()
11783 .unwrap_or(0);
11784 assert!(
11785 richest >= 2,
11786 "vendored corpus no longer carries a package with unmodelled fields, \
11787 so the relay test can no longer distinguish a verbatim relay from a lossy one"
11788 );
11789
11790 vectors
11791 }
11792
11793 #[tokio::test]
11801 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
11802 for (corpus_id, package) in fed_admission_facts_vectors() {
11803 let registry = Arc::new(Registry::default());
11804 let forwarding = Arc::new(ForwardingTable::default());
11805 let supervisor = SupervisorHandle::new();
11806 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11807 let handler =
11808 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11809 .with_supervisor(supervisor)
11810 .with_admission_facts_config(
11811 Some("fed".to_string()),
11812 Some(vec!["target".to_string()]),
11813 );
11814
11815 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11816 hello_via_sink(
11817 &handler,
11818 &target_ctx,
11819 &mut target_rx,
11820 hello_frame("target", PROTOCOL_VERSION, 1),
11821 )
11822 .await;
11823
11824 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
11825 let route_handler = handler.clone();
11826 let expected = package.clone();
11827 let route_task = tokio::spawn(async move {
11828 route_handler
11829 .handle_control_frame(
11830 &client_ctx,
11831 route_open_frame_with_admission_facts(
11832 20,
11833 "target",
11834 unique_project_root("admission-facts"),
11835 Some(subc_control::ConsumerIdentity {
11836 module_id: "fed".to_string(),
11837 launch_nonce: "fed-nonce".to_string(),
11838 }),
11839 Some(package),
11840 ),
11841 )
11842 .await
11843 .unwrap()
11844 });
11845
11846 let bind_frame = target_rx.recv().await.unwrap();
11847 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11848 let ModuleControlRequest::RouteBind {
11849 admission_facts, ..
11850 } = bind
11851 else {
11852 panic!("{corpus_id}: expected route.bind")
11853 };
11854 assert_eq!(
11855 admission_facts,
11856 Some(expected),
11857 "{corpus_id}: relay must not add, drop or reshape any field"
11858 );
11859
11860 handler
11861 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11862 .await
11863 .unwrap();
11864 route_task.await.unwrap();
11865 }
11866 }
11867
11868 #[tokio::test]
11869 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
11870 let registry = Arc::new(Registry::default());
11871 let forwarding = Arc::new(ForwardingTable::default());
11872 let supervisor = SupervisorHandle::new();
11873 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11874 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
11875 let handler =
11876 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11877 .with_supervisor(supervisor)
11878 .with_admission_facts_config(
11879 Some("fed".to_string()),
11880 Some(vec!["target".to_string()]),
11881 );
11882
11883 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
11884 hello_via_sink(
11885 &handler,
11886 &target_ctx,
11887 &mut target_rx,
11888 hello_frame("target", PROTOCOL_VERSION, 1),
11889 )
11890 .await;
11891 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
11892 hello_via_sink(
11893 &handler,
11894 &other_ctx,
11895 &mut other_rx,
11896 hello_frame("other", PROTOCOL_VERSION, 2),
11897 )
11898 .await;
11899
11900 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
11901 let expected_facts = facts.clone();
11902 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
11903 let route_handler = handler.clone();
11904 let route_task = tokio::spawn(async move {
11905 route_handler
11906 .handle_control_frame(
11907 &client_ctx,
11908 route_open_frame_with_admission_facts(
11909 10,
11910 "target",
11911 unique_project_root("admission-facts"),
11912 Some(subc_control::ConsumerIdentity {
11913 module_id: "fed".to_string(),
11914 launch_nonce: "fed-nonce".to_string(),
11915 }),
11916 Some(facts.clone()),
11917 ),
11918 )
11919 .await
11920 .unwrap()
11921 });
11922 let bind_frame = target_rx.recv().await.unwrap();
11923 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11924 let ModuleControlRequest::RouteBind {
11925 admission_facts, ..
11926 } = bind
11927 else {
11928 panic!("expected route.bind")
11929 };
11930 assert_eq!(admission_facts, Some(expected_facts));
11931 handler
11932 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11933 .await
11934 .unwrap();
11935 assert!(route_task.await.unwrap().is_empty());
11936 assert!(matches!(
11937 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
11938 .unwrap(),
11939 ClientControlResponse::RouteOpen { .. }
11940 ));
11941
11942 let direct = handler
11943 .handle_control_frame(
11944 &route_ctx(ConnectionId::new(73)).0,
11945 route_open_frame_with_admission_facts(
11946 11,
11947 "target",
11948 unique_project_root("admission-facts"),
11949 None,
11950 Some(json!({"x": 1})),
11951 ),
11952 )
11953 .await
11954 .unwrap();
11955 assert_eq!(
11956 parse_error(&direct[0])["code"],
11957 "admission_facts_not_permitted"
11958 );
11959
11960 let different_reserved = handler
11961 .handle_control_frame(
11962 &route_ctx(ConnectionId::new(77)).0,
11963 route_open_frame_with_admission_facts(
11964 15,
11965 "target",
11966 unique_project_root("admission-facts"),
11967 Some(subc_control::ConsumerIdentity {
11968 module_id: "other".to_string(),
11969 launch_nonce: "other-nonce".to_string(),
11970 }),
11971 Some(json!({"x": 1})),
11972 ),
11973 )
11974 .await
11975 .unwrap();
11976 assert_eq!(
11977 parse_error(&different_reserved[0])["code"],
11978 "admission_facts_not_permitted"
11979 );
11980
11981 let other_target = handler
11982 .handle_control_frame(
11983 &route_ctx(ConnectionId::new(74)).0,
11984 route_open_frame_with_admission_facts(
11985 12,
11986 "other",
11987 unique_project_root("admission-facts"),
11988 Some(subc_control::ConsumerIdentity {
11989 module_id: "fed".to_string(),
11990 launch_nonce: "fed-nonce".to_string(),
11991 }),
11992 Some(json!({"x": 1})),
11993 ),
11994 )
11995 .await
11996 .unwrap();
11997 assert_eq!(
11998 parse_error(&other_target[0])["code"],
11999 "admission_facts_target_not_allowed"
12000 );
12001
12002 let nonexistent = handler
12003 .handle_control_frame(
12004 &route_ctx(ConnectionId::new(75)).0,
12005 route_open_frame_with_admission_facts(
12006 13,
12007 "missing",
12008 unique_project_root("admission-facts"),
12009 None,
12010 Some(json!({"x": 1})),
12011 ),
12012 )
12013 .await
12014 .unwrap();
12015 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
12016
12017 let described = handler
12018 .handle_control_frame(
12019 &route_ctx(ConnectionId::new(76)).0,
12020 Frame::build(
12021 FrameType::Request,
12022 control_flags(),
12023 0,
12024 0,
12025 14,
12026 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12027 )
12028 .unwrap(),
12029 )
12030 .await
12031 .unwrap();
12032 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12033 serde_json::from_slice(&described[0].body).unwrap()
12034 else {
12035 panic!("expected server.describe response")
12036 };
12037 assert!(capabilities
12038 .iter()
12039 .any(|cap| cap == "admission_facts_relay_v1"));
12040 }
12041
12042 #[tokio::test]
12043 async fn admission_facts_without_configured_carrier_are_rejected() {
12044 let registry = Arc::new(Registry::default());
12045 let forwarding = Arc::new(ForwardingTable::default());
12046 let handler = ControlHandler::with_forwarding(registry, forwarding);
12047 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
12048 hello_via_sink(
12049 &handler,
12050 &target_ctx,
12051 &mut target_rx,
12052 hello_frame("target", PROTOCOL_VERSION, 1),
12053 )
12054 .await;
12055
12056 let responses = handler
12057 .handle_control_frame(
12058 &route_ctx(ConnectionId::new(79)).0,
12059 route_open_frame_with_admission_facts(
12060 16,
12061 "target",
12062 unique_project_root("admission-facts"),
12063 None,
12064 Some(json!({"x": 1})),
12065 ),
12066 )
12067 .await
12068 .unwrap();
12069 assert_eq!(
12070 parse_error(&responses[0])["code"],
12071 "admission_facts_not_permitted"
12072 );
12073 }
12074
12075 #[tokio::test]
12076 async fn route_open_relays_consumer_capabilities_verbatim() {
12077 let registry = Arc::new(Registry::default());
12078 let forwarding = Arc::new(ForwardingTable::default());
12079 let handler =
12080 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12081 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
12082 hello_via_sink(
12083 &handler,
12084 &module_ctx,
12085 &mut module_rx,
12086 hello_frame("aft", PROTOCOL_VERSION, 7),
12087 )
12088 .await;
12089
12090 let expected = vec!["elicitation".to_string(), "roots".to_string()];
12091 let expected_for_request = expected.clone();
12092 let project_root = unique_project_root("consumer-capabilities-present");
12093 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
12094 let route_handler = handler.clone();
12095 let route_task = tokio::spawn(async move {
12096 route_handler
12097 .handle_control_frame(
12098 &client_ctx,
12099 route_open_frame_with_consumer_capabilities(
12100 401,
12101 "aft",
12102 project_root,
12103 Some(expected_for_request),
12104 ),
12105 )
12106 .await
12107 .unwrap()
12108 });
12109 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12110 .await
12111 .unwrap()
12112 .unwrap();
12113 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12114 let ModuleControlRequest::RouteBind {
12115 consumer_capabilities,
12116 ..
12117 } = bind
12118 else {
12119 panic!("expected route.bind request, got {bind:?}");
12120 };
12121 assert_eq!(consumer_capabilities, Some(expected.clone()));
12122
12123 handler
12124 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12125 .await
12126 .unwrap();
12127 let route_response = route_task.await.unwrap();
12128 assert!(route_response.is_empty());
12129 let published = client_rx.recv().await.unwrap();
12130 assert!(matches!(
12131 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12132 ClientControlResponse::RouteOpen { .. }
12133 ));
12134 }
12135
12136 #[tokio::test]
12137 async fn route_open_without_consumer_capabilities_relays_none() {
12138 let registry = Arc::new(Registry::default());
12139 let forwarding = Arc::new(ForwardingTable::default());
12140 let handler =
12141 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12142 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
12143 hello_via_sink(
12144 &handler,
12145 &module_ctx,
12146 &mut module_rx,
12147 hello_frame("aft", PROTOCOL_VERSION, 7),
12148 )
12149 .await;
12150
12151 let project_root = unique_project_root("consumer-capabilities-absent");
12152 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
12153 let route_handler = handler.clone();
12154 let route_task = tokio::spawn(async move {
12155 route_handler
12156 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
12157 .await
12158 .unwrap()
12159 });
12160 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12161 .await
12162 .unwrap()
12163 .unwrap();
12164 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12165 let ModuleControlRequest::RouteBind {
12166 consumer_capabilities,
12167 ..
12168 } = bind
12169 else {
12170 panic!("expected route.bind request, got {bind:?}");
12171 };
12172 assert_eq!(consumer_capabilities, None);
12173
12174 handler
12175 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12176 .await
12177 .unwrap();
12178 let route_response = route_task.await.unwrap();
12179 assert!(route_response.is_empty());
12180 let published = client_rx.recv().await.unwrap();
12181 assert!(matches!(
12182 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12183 ClientControlResponse::RouteOpen { .. }
12184 ));
12185 }
12186
12187 async fn bind_role_versions_for(
12191 sent: Option<BTreeMap<String, String>>,
12192 connection: u64,
12193 ) -> Option<BTreeMap<String, String>> {
12194 let registry = Arc::new(Registry::default());
12195 let forwarding = Arc::new(ForwardingTable::default());
12196 let handler =
12197 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12198 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
12199 hello_via_sink(
12200 &handler,
12201 &module_ctx,
12202 &mut module_rx,
12203 hello_frame("aft", PROTOCOL_VERSION, 7),
12204 )
12205 .await;
12206 let project_root = unique_project_root("role-versions");
12207 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
12208 let route_handler = handler.clone();
12209 let route_task = tokio::spawn(async move {
12210 route_handler
12211 .handle_control_frame(
12212 &client_ctx,
12213 route_open_frame_with_role_versions(403, "aft", project_root, sent),
12214 )
12215 .await
12216 .unwrap()
12217 });
12218 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12219 .await
12220 .expect("a well-formed route.open reaches the module as a bind")
12221 .unwrap();
12222 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12223 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
12224 panic!("expected route.bind request, got {bind:?}");
12225 };
12226 handler
12227 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12228 .await
12229 .unwrap();
12230 assert!(route_task.await.unwrap().is_empty());
12231 let published = client_rx.recv().await.unwrap();
12232 assert!(matches!(
12233 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12234 ClientControlResponse::RouteOpen { .. }
12235 ));
12236 role_versions
12237 }
12238
12239 #[tokio::test]
12240 async fn route_open_relays_role_versions_verbatim() {
12241 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
12242 assert_eq!(
12243 bind_role_versions_for(Some(sent.clone()), 141).await,
12244 Some(sent)
12245 );
12246 }
12247
12248 #[tokio::test]
12251 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
12252 assert_eq!(bind_role_versions_for(None, 143).await, None);
12253 assert_eq!(
12254 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
12255 None
12256 );
12257 }
12258
12259 #[tokio::test]
12260 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
12261 let registry = Arc::new(Registry::default());
12262 let forwarding = Arc::new(ForwardingTable::default());
12263 let handler =
12264 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12265 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
12266 hello_via_sink(
12267 &handler,
12268 &module_ctx,
12269 &mut module_rx,
12270 hello_frame("aft", PROTOCOL_VERSION, 7),
12271 )
12272 .await;
12273
12274 let nine: BTreeMap<String, String> = (0..9)
12275 .map(|index| (format!("role-{index}"), "v1".to_string()))
12276 .collect();
12277 for (label, malformed) in [
12278 (
12279 "invalid role name",
12280 role_versions(&[("Tool_Provider", "v1")]),
12281 ),
12282 ("invalid version", role_versions(&[("tool-provider", "v0")])),
12283 ("nine entries", nine),
12284 ] {
12285 let responses = tokio::time::timeout(
12288 Duration::from_secs(1),
12289 handler.handle_control_frame(
12290 &route_ctx(ConnectionId::new(148)).0,
12291 route_open_frame_with_role_versions(
12292 404,
12293 "aft",
12294 unique_project_root("role-versions-malformed"),
12295 Some(malformed),
12296 ),
12297 ),
12298 )
12299 .await
12300 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
12301 .unwrap();
12302 assert_eq!(responses.len(), 1, "{label}");
12303 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
12304 let error = parse_error(&responses[0]);
12305 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
12306 assert_eq!(
12307 error["detail"]["field"], "role_versions",
12308 "{label}: {error}"
12309 );
12310 assert!(
12311 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
12312 "{label}: a malformed declaration is terminal"
12313 );
12314 assert!(
12315 module_rx.try_recv().is_err(),
12316 "{label}: the module must never see a bind"
12317 );
12318 }
12319 }
12320
12321 #[tokio::test]
12325 async fn route_role_versions_capability_is_advertised() {
12326 let handler = ControlHandler::new(Arc::new(Registry::default()));
12327 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
12328 let ack = hello_via_sink(
12329 &handler,
12330 &ctx,
12331 &mut rx,
12332 hello_frame("m", PROTOCOL_VERSION, 1),
12333 )
12334 .await;
12335 let ack = parse_ack(&ack);
12336 assert!(
12337 ack.subc_capabilities
12338 .iter()
12339 .any(|c| c == "route-role-versions/v1"),
12340 "{:?}",
12341 ack.subc_capabilities
12342 );
12343
12344 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12345 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12346 let reply = handler
12347 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
12348 .await
12349 .unwrap()
12350 .pop()
12351 .unwrap();
12352 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12353 serde_json::from_slice(&reply.body).unwrap()
12354 else {
12355 panic!("not a server.describe reply");
12356 };
12357 assert!(
12358 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
12359 "{capabilities:?}"
12360 );
12361 }
12362
12363 #[tokio::test]
12367 async fn catalog_events_capability_is_advertised() {
12368 let handler = ControlHandler::new(Arc::new(Registry::default()));
12369 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12370 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12371 let reply = handler
12372 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
12373 .await
12374 .unwrap()
12375 .pop()
12376 .unwrap();
12377 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12378 serde_json::from_slice(&reply.body).unwrap()
12379 else {
12380 panic!("not a server.describe reply");
12381 };
12382 assert!(
12383 capabilities
12384 .iter()
12385 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
12386 "{capabilities:?}"
12387 );
12388 }
12389
12390 #[tokio::test]
12391 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
12392 let handler = ControlHandler::new(Arc::new(Registry::default()));
12393 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
12394 let ack = hello_via_sink(
12395 &handler,
12396 &module,
12397 &mut module_rx,
12398 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
12399 )
12400 .await;
12401 let ack = parse_ack(&ack);
12402 assert!(
12403 ack.subc_capabilities
12404 .iter()
12405 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12406 "{:?}",
12407 ack.subc_capabilities
12408 );
12409 assert!(ack
12410 .subc_capabilities
12411 .iter()
12412 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12413
12414 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
12415 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12416 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12417 let reply = handler
12418 .handle_control_frame(&client, frame)
12419 .await
12420 .unwrap()
12421 .pop()
12422 .unwrap();
12423 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12424 serde_json::from_slice(&reply.body).unwrap()
12425 else {
12426 panic!("not a server.describe reply");
12427 };
12428 assert!(
12429 capabilities
12430 .iter()
12431 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12432 "{capabilities:?}"
12433 );
12434 assert!(capabilities
12435 .iter()
12436 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12437 }
12438
12439 #[tokio::test]
12440 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
12441 let handler = ControlHandler::new(Arc::new(Registry::default()));
12442 let (module, mut rx) = route_ctx(ConnectionId::new(156));
12443 let ack = hello_via_sink(
12444 &handler,
12445 &module,
12446 &mut rx,
12447 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
12448 )
12449 .await;
12450 assert!(parse_ack(&ack)
12451 .subc_capabilities
12452 .iter()
12453 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12454 assert!(parse_ack(&ack)
12455 .subc_capabilities
12456 .iter()
12457 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12458 let (client, _rx) = route_ctx(ConnectionId::new(157));
12459 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12460 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12461 let reply = handler
12462 .handle_control_frame(&client, frame)
12463 .await
12464 .unwrap()
12465 .pop()
12466 .unwrap();
12467 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12468 serde_json::from_slice(&reply.body).unwrap()
12469 else {
12470 panic!("server.describe reply")
12471 };
12472 assert!(capabilities
12473 .iter()
12474 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12475 assert!(capabilities
12476 .iter()
12477 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12478 }
12479
12480 #[tokio::test]
12481 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
12482 let handler = ControlHandler::new(Arc::new(Registry::default()));
12483 let (module, mut rx) = route_ctx(ConnectionId::new(158));
12484 let ack = hello_via_sink(
12485 &handler,
12486 &module,
12487 &mut rx,
12488 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
12489 )
12490 .await;
12491 assert!(parse_ack(&ack)
12492 .subc_capabilities
12493 .iter()
12494 .any(|cap| cap == "scope-workdir/v1"));
12495 let (client, _rx) = route_ctx(ConnectionId::new(159));
12496 let frame = Frame::build(
12497 FrameType::Request,
12498 control_flags(),
12499 0,
12500 0,
12501 5,
12502 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12503 )
12504 .unwrap();
12505 let reply = handler
12506 .handle_control_frame(&client, frame)
12507 .await
12508 .unwrap()
12509 .remove(0);
12510 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12511 serde_json::from_slice(&reply.body).unwrap()
12512 else {
12513 panic!("server.describe reply")
12514 };
12515 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
12516 }
12517
12518 #[tokio::test]
12519 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
12520 let handler = ControlHandler::new(Arc::new(Registry::default()));
12521 let (module, mut rx) = route_ctx(ConnectionId::new(154));
12522 let ack = hello_via_sink(
12523 &handler,
12524 &module,
12525 &mut rx,
12526 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
12527 )
12528 .await;
12529 assert!(parse_ack(&ack)
12530 .subc_capabilities
12531 .iter()
12532 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12533 let (client, _rx) = route_ctx(ConnectionId::new(155));
12534 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12535 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12536 let reply = handler
12537 .handle_control_frame(&client, frame)
12538 .await
12539 .unwrap()
12540 .pop()
12541 .unwrap();
12542 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12543 serde_json::from_slice(&reply.body).unwrap()
12544 else {
12545 panic!("not a server.describe reply")
12546 };
12547 assert!(capabilities
12548 .iter()
12549 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12550 }
12551
12552 #[tokio::test]
12553 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
12554 let registry = Arc::new(Registry::default());
12555 let forwarding = Arc::new(ForwardingTable::default());
12556 let handler =
12557 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12558 .with_health_probe_timeout(Duration::from_secs(5));
12559 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
12560 hello_via_sink(
12561 &handler,
12562 &module_ctx,
12563 &mut module_rx,
12564 non_routable_hello_frame_with_control_ops(
12565 "mcp",
12566 300,
12567 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12568 ),
12569 )
12570 .await;
12571 assert!(registry
12572 .get_module("mcp")
12573 .unwrap()
12574 .unwrap()
12575 .manifest
12576 .provides
12577 .is_empty());
12578
12579 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
12580 let route_response = handler
12581 .handle_control_frame(
12582 &route_client_ctx,
12583 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
12584 )
12585 .await
12586 .unwrap();
12587 assert_eq!(route_response[0].header.ty, FrameType::Error);
12588 assert_eq!(
12589 parse_error(&route_response[0])["code"],
12590 "target_unavailable"
12591 );
12592 assert!(parse_error(&route_response[0])["message"]
12593 .as_str()
12594 .unwrap()
12595 .contains("does not provide the requested target"));
12596 assert!(module_rx.try_recv().is_err());
12597
12598 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
12599 let health_handler = handler.clone();
12600 let health_task = tokio::spawn(async move {
12601 health_handler
12602 .handle_control_frame(
12603 &health_client_ctx,
12604 supervisor_health_probe_frame(302, "mcp"),
12605 )
12606 .await
12607 .unwrap()
12608 });
12609 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12610 .await
12611 .unwrap()
12612 .unwrap();
12613 assert_eq!(
12614 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
12615 ModuleControlRequest::HealthCheck {}
12616 );
12617 handler
12618 .handle_control_frame(
12619 &module_ctx,
12620 health_response(health_frame.header.corr, HealthStatus::Ok),
12621 )
12622 .await
12623 .unwrap();
12624 let health_response = health_task.await.unwrap();
12625 assert_eq!(health_response[0].header.ty, FrameType::Response);
12626 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
12627 ClientControlResponse::SupervisorHealthProbe {
12628 module_id, status, ..
12629 } => {
12630 assert_eq!(module_id, "mcp");
12631 assert_eq!(status, HealthStatus::Ok);
12632 }
12633 other => panic!("unexpected health response: {other:?}"),
12634 }
12635
12636 forwarding
12641 .cleanup_connection(module_ctx.connection_id)
12642 .unwrap();
12643 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
12644 let cleanup_response = tokio::time::timeout(
12645 Duration::from_millis(200),
12646 handler.handle_control_frame(
12647 &cleanup_probe_ctx,
12648 supervisor_health_probe_frame(303, "mcp"),
12649 ),
12650 )
12651 .await
12652 .expect("probe should fail immediately when the control lane is gone")
12653 .unwrap();
12654 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
12655 assert_eq!(
12656 parse_error(&cleanup_response[0])["code"],
12657 "target_unavailable"
12658 );
12659 assert!(parse_error(&cleanup_response[0])["message"]
12660 .as_str()
12661 .unwrap()
12662 .contains("no module connection"));
12663
12664 handler
12665 .cleanup_connection(module_ctx.connection_id)
12666 .unwrap();
12667 }
12668
12669 #[tokio::test]
12670 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
12671 let registry = Arc::new(Registry::default());
12672 let supervisor_handle = SupervisorHandle::new();
12673 let supervisor =
12674 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12675 .with_handle(supervisor_handle.clone())
12676 .with_connection_file_path(
12677 std::env::temp_dir()
12678 .join(format!("subc-route-open-warming-{}", std::process::id())),
12679 );
12680 let module = supervisor
12681 .supervise_configured(
12682 ModuleSpec {
12683 module_id: "warming".to_string(),
12684 program: fake_aft_stub_path(),
12685 args: Vec::new(),
12686 env: Vec::new(),
12687 reserved: false,
12688 reserved_prefixes: Vec::new(),
12689 protocol: ModuleProtocol::Subc,
12690 overlap: Default::default(),
12691 },
12692 true,
12693 )
12694 .unwrap();
12695 assert_eq!(module.state().unwrap(), ModuleState::Running);
12696
12697 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12698 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
12699 let response = handler
12700 .handle_control_frame(
12701 &ctx,
12702 route_open_frame(304, "warming", unique_project_root("warming")),
12703 )
12704 .await
12705 .unwrap();
12706 module.stop().await.unwrap();
12707
12708 assert_eq!(response[0].header.ty, FrameType::Error);
12709 let error = parse_error(&response[0]);
12710 assert_eq!(error["code"], "module_warming");
12711 assert!(error["message"]
12712 .as_str()
12713 .unwrap()
12714 .contains("state=running, enabled=true, live=false"));
12715 }
12716
12717 #[test]
12718 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
12719 let handler = ControlHandler::new(Arc::new(Registry::default()));
12720 let capture = EventCapture::default();
12721 let _subscriber =
12722 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12723 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
12724 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
12725 let pending = (0..limit).collect::<Vec<_>>();
12726 let response = handler
12727 .route_open_capacity_refusal(
12728 &ctx,
12729 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
12730 "busy",
12731 pending.len(),
12732 limit,
12733 )
12734 .unwrap();
12735 assert_eq!(parse_error(&response)["code"], "target_unavailable");
12736 let event = capture
12737 .events()
12738 .into_iter()
12739 .find(|event| {
12740 event.target == "control"
12741 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
12742 })
12743 .expect("connection admission refusal event");
12744 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12745 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12746 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
12749 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
12750 }
12751
12752 #[test]
12753 fn route_open_target_cap_logs_admission_reason_and_capacity() {
12754 let handler = ControlHandler::new(Arc::new(Registry::default()));
12755 let capture = EventCapture::default();
12756 let _subscriber =
12757 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12758 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
12759 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
12760 let guards = (0..limit)
12761 .map(|_| {
12762 handler
12763 .route_bind_concurrency
12764 .try_admit("busy", limit)
12765 .unwrap()
12766 })
12767 .collect::<Vec<_>>();
12768 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
12769 Err(in_flight) => in_flight,
12770 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
12771 };
12772 let response = handler
12773 .route_open_target_capacity_refusal(
12774 &ctx,
12775 &RouteOpenCaller::unknown(),
12776 &route_open_frame(397, "busy", unique_project_root("target-cap")),
12777 "busy",
12778 in_flight,
12779 )
12780 .unwrap();
12781 assert_eq!(parse_error(&response)["code"], "target_unavailable");
12782 let event = capture
12783 .events()
12784 .into_iter()
12785 .find(|event| {
12786 event.target == "control"
12787 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
12788 })
12789 .expect("target admission refusal event");
12790 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12791 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12792 drop(guards);
12793 }
12794
12795 #[cfg(unix)]
12796 #[tokio::test(start_paused = true)]
12797 async fn supervisor_list_shows_a_failed_modules_next_retry() {
12798 let registry = Arc::new(Registry::default());
12799 let handle = SupervisorHandle::new();
12800 let supervisor =
12801 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
12802 .with_handle(handle.clone());
12803 let module = supervisor
12804 .spawn(crate::ModuleSpec {
12805 module_id: "retry-list".into(),
12806 program: PathBuf::from("/bin/sh"),
12807 args: vec!["-c".into(), "exit 17".into()],
12808 env: vec![],
12809 reserved: false,
12810 reserved_prefixes: vec![],
12811 protocol: ModuleProtocol::None,
12812 overlap: Default::default(),
12813 })
12814 .unwrap();
12815 let deadline = std::time::Instant::now() + Duration::from_secs(10);
12816 while module.state().unwrap() != crate::ModuleState::Failed {
12817 assert!(
12818 std::time::Instant::now() < deadline,
12819 "waiting for budget failure"
12820 );
12821 tokio::task::yield_now().await;
12822 }
12823 let handler = ControlHandler::new(registry).with_supervisor(handle);
12824 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12825 let replies = handler.handle_supervisor_list(frame).await.unwrap();
12826 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12827 assert_eq!(body["modules"][0]["state"], "failed");
12828 let retry = body["modules"][0]["next_retry_at_ms"]
12829 .as_u64()
12830 .expect("scheduled retry on the wire");
12831 let now = std::time::SystemTime::now()
12832 .duration_since(std::time::UNIX_EPOCH)
12833 .unwrap()
12834 .as_millis() as u64;
12835 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
12836 module.set_enabled(false).await.unwrap();
12837 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
12838 let replies = handler.handle_supervisor_list(frame).await.unwrap();
12839 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12840 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
12841 }
12842
12843 #[tokio::test]
12844 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
12845 let registry = Arc::new(Registry::default());
12846 let handle = SupervisorHandle::new();
12847 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
12848 .with_handle(handle.clone());
12849 supervisor
12850 .supervise_configured(
12851 crate::ModuleSpec {
12852 module_id: "drift".into(),
12853 program: PathBuf::from("disabled-module"),
12854 args: Vec::new(),
12855 env: Vec::new(),
12856 reserved: false,
12857 reserved_prefixes: Vec::new(),
12858 protocol: ModuleProtocol::Subc,
12859 overlap: Default::default(),
12860 },
12861 false,
12862 )
12863 .unwrap();
12864 let handler = ControlHandler::new(registry).with_supervisor(handle);
12865 for configured in [None, Some(false), Some(true)] {
12866 handler.capability_evaluator.configure(
12867 configured.map(|enabled| ("drift".to_string(), enabled)),
12868 BTreeMap::new(),
12869 );
12870 let frame =
12871 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12872 let replies = handler
12873 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
12874 .await
12875 .unwrap();
12876 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12877 assert!(body.get("configured_enabled").is_none());
12878 assert_eq!(body["op"], "supervisor.ack");
12879 let frame =
12880 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12881 let replies = handler.handle_supervisor_list(frame).await.unwrap();
12882 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12883 assert_eq!(
12884 body["modules"][0].get("configured_enabled"),
12885 configured.map(Value::Bool).as_ref()
12886 );
12887 }
12888 }
12889
12890 #[tokio::test(flavor = "current_thread")]
12891 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
12892 let handler = ControlHandler::new(Arc::new(Registry::default()));
12893 let capture = EventCapture::default();
12894 let _subscriber =
12895 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12896 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
12897 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12898
12899 let _ = handler
12900 .handle_client_control_request(
12901 &ctx,
12902 frame,
12903 ClientControlRequest::SupervisorSetEnabled {
12904 module_id: "broca".to_string(),
12905 enabled: false,
12906 },
12907 &mut None,
12908 )
12909 .await
12910 .unwrap();
12911
12912 let events = capture
12913 .events()
12914 .into_iter()
12915 .filter(|event| {
12916 event.target == "control"
12917 && event.fields.get("message").map(String::as_str)
12918 == Some("supervisor request received")
12919 })
12920 .collect::<Vec<_>>();
12921 assert_eq!(events.len(), 1, "one supervisor request log line");
12922 let event = &events[0];
12923 assert_eq!(event.level, tracing::Level::INFO);
12924 assert_eq!(
12925 event.fields.get("op"),
12926 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
12927 );
12928 assert_eq!(
12929 event.fields.get("module_id"),
12930 Some(&"\"broca\"".to_string())
12931 );
12932 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
12933 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
12934 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
12935 }
12936
12937 #[tokio::test(flavor = "current_thread")]
12938 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
12939 let registry = Arc::new(Registry::default());
12940 let module_connection = ConnectionId::new(303);
12941 registry
12942 .register_with_control_ops(
12943 manifest("aft", PROTOCOL_VERSION),
12944 PROTOCOL_VERSION,
12945 module_connection,
12946 module_baseline_control_ops(),
12947 )
12948 .unwrap();
12949 let handler = ControlHandler::new(registry);
12950 let capture = EventCapture::default();
12951 let _subscriber =
12952 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12953 let (ctx, _rx) = route_ctx(module_connection);
12954 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12955
12956 let _ = handler
12957 .handle_client_control_request(
12958 &ctx,
12959 frame,
12960 ClientControlRequest::SupervisorSetEnabled {
12961 module_id: "broca".to_string(),
12962 enabled: false,
12963 },
12964 &mut None,
12965 )
12966 .await
12967 .unwrap();
12968
12969 let callers = capture
12970 .events()
12971 .into_iter()
12972 .filter(|event| {
12973 event.fields.get("message").map(String::as_str)
12974 == Some("supervisor request received")
12975 })
12976 .map(|event| event.fields.get("caller").cloned())
12977 .collect::<Vec<_>>();
12978 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
12979 }
12980
12981 #[tokio::test(flavor = "current_thread")]
12982 async fn supervisor_list_does_not_log_request_received() {
12983 let handler = ControlHandler::new(Arc::new(Registry::default()));
12984 let capture = EventCapture::default();
12985 let _subscriber =
12986 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12987 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
12988 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12989
12990 handler
12991 .handle_client_control_request(
12992 &ctx,
12993 frame,
12994 ClientControlRequest::SupervisorList {},
12995 &mut None,
12996 )
12997 .await
12998 .unwrap();
12999
13000 assert!(capture.events().into_iter().all(|event| {
13001 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13002 }));
13003 }
13004
13005 #[tokio::test(flavor = "current_thread")]
13006 async fn supervisor_rescan_preview_does_not_log_request_received() {
13007 let handler = ControlHandler::new(Arc::new(Registry::default()));
13008 let capture = EventCapture::default();
13009 let _subscriber =
13010 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13011 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
13012 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13013
13014 handler
13015 .handle_client_control_request(
13016 &ctx,
13017 frame,
13018 ClientControlRequest::SupervisorRescan { preview: true },
13019 &mut None,
13020 )
13021 .await
13022 .unwrap();
13023
13024 assert!(capture.events().into_iter().all(|event| {
13025 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13026 }));
13027 }
13028
13029 struct ApproveOperator;
13030 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
13031 fn prompt(
13032 &self,
13033 _: &str,
13034 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
13035 ) -> crate::operator_confirm::ProviderResult {
13036 crate::operator_confirm::ProviderResult::Approved
13037 }
13038 }
13039
13040 #[tokio::test]
13041 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
13042 assert_eq!(
13043 ControlHandler::default().privileged_operator_prompt,
13044 PrivilegedOperatorPrompt::Log
13045 );
13046 for (initial, changed, running) in [
13047 ("log", "enforce", PrivilegedOperatorPrompt::Log),
13048 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
13049 ] {
13050 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
13051 let path = dir.join("subc.jsonc");
13052 std::fs::write(
13053 &path,
13054 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
13055 )
13056 .unwrap();
13057 let config = crate::daemon_config::load(&path).unwrap().unwrap();
13058 let registry = Arc::new(Registry::default());
13059 let handle = SupervisorHandle::new();
13060 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
13061 .with_handle(handle.clone());
13062 let handler = ControlHandler::new(registry)
13063 .with_supervisor(handle)
13064 .with_privileged_operator_prompt(config.privileged_operator_prompt)
13065 .with_supervisor_rescan(supervisor, &path, config.port);
13066 handler.forwarding.operator_confirms().configure(
13067 Arc::new(ApproveOperator),
13068 crate::operator_confirm::OperatorLimits::default(),
13069 );
13070 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
13071
13072 for (disk_mode, needs_restart) in [
13073 (Some(initial), false),
13074 (Some(changed), true),
13075 (Some(changed), true),
13076 (None, running == PrivilegedOperatorPrompt::Enforce),
13077 (Some(initial), false),
13078 ] {
13079 let mut doc = json!({"version":1});
13080 if let Some(mode) = disk_mode {
13081 doc["privileged_operator_prompt"] = json!(mode);
13082 }
13083 std::fs::write(&path, doc.to_string()).unwrap();
13084 for preview in [true, false] {
13085 let request = ClientControlRequest::SupervisorRescan { preview };
13086 let frame = Frame::build(
13087 FrameType::Request,
13088 control_flags(),
13089 0,
13090 0,
13091 1,
13092 serde_json::to_vec(&request).unwrap(),
13093 )
13094 .unwrap();
13095 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13096 assert_eq!(frames.len(), 1);
13097 let ClientControlResponse::SupervisorRescan { result } =
13098 serde_json::from_slice(&frames[0].body).unwrap()
13099 else {
13100 panic!("expected a rescan response");
13101 };
13102 let expected = if needs_restart {
13103 vec!["privileged_operator_prompt"]
13104 } else {
13105 Vec::new()
13106 };
13107 assert_eq!(result.restart_required, expected);
13108 assert_eq!(handler.privileged_operator_prompt, running);
13109 assert_eq!(
13110 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
13111 running
13112 );
13113 }
13114 }
13115 }
13116 }
13117
13118 #[tokio::test]
13119 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
13120 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
13121 let path = dir.join("subc.jsonc");
13122 let registry = Arc::new(Registry::default());
13123 let handle = SupervisorHandle::new();
13124 let supervisor =
13125 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
13126 let handler = ControlHandler::new(registry)
13127 .with_supervisor(handle)
13128 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
13129 .with_supervisor_rescan(supervisor, &path, None);
13130 handler.forwarding.operator_confirms().configure(
13131 Arc::new(ApproveOperator),
13132 crate::operator_confirm::OperatorLimits::default(),
13133 );
13134 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
13135 for preview in [true, false] {
13136 std::fs::write(
13137 &path,
13138 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
13139 )
13140 .unwrap();
13141 let request = ClientControlRequest::SupervisorRescan { preview };
13142 let frame = Frame::build(
13143 FrameType::Request,
13144 control_flags(),
13145 0,
13146 0,
13147 1,
13148 serde_json::to_vec(&request).unwrap(),
13149 )
13150 .unwrap();
13151 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13152 assert_eq!(frames.len(), 1);
13153 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
13154 assert_eq!(
13155 handler.privileged_operator_prompt,
13156 PrivilegedOperatorPrompt::Enforce
13157 );
13158 }
13159 }
13160
13161 #[tokio::test]
13166 async fn route_open_refusal_names_the_check_that_refused() {
13167 let handler = ControlHandler::new(Arc::new(Registry::default()));
13168 let capture = EventCapture::default();
13169 let _subscriber =
13170 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13171 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13172 let response = handler
13173 .handle_control_frame(
13174 &ctx,
13175 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
13176 )
13177 .await
13178 .unwrap();
13179
13180 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13181 let event = capture
13182 .events()
13183 .into_iter()
13184 .find(|event| {
13185 event.target == "control"
13186 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13187 })
13188 .expect("route.open refusal event");
13189 assert_eq!(
13190 event.fields.get("reason"),
13191 Some(&"\"not_registered\"".to_string())
13192 );
13193 }
13194
13195 #[tokio::test]
13196 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
13197 let registry = Arc::new(Registry::default());
13198 let supervisor_handle = SupervisorHandle::new();
13199 let supervisor =
13200 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13201 .with_handle(supervisor_handle.clone())
13202 .with_connection_file_path(std::env::temp_dir().join(format!(
13203 "subc-route-open-refusal-info-{}",
13204 std::process::id()
13205 )));
13206 let module = supervisor
13207 .supervise_configured(
13208 ModuleSpec {
13209 module_id: "warming".to_string(),
13210 program: fake_aft_stub_path(),
13211 args: Vec::new(),
13212 env: Vec::new(),
13213 reserved: false,
13214 reserved_prefixes: Vec::new(),
13215 protocol: ModuleProtocol::Subc,
13216 overlap: Default::default(),
13217 },
13218 true,
13219 )
13220 .unwrap();
13221 assert_eq!(module.state().unwrap(), ModuleState::Running);
13222
13223 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13224 assert!(handler
13225 .counters()
13226 .snapshot()
13227 .get("route_open_refused_by_code")
13228 .is_none());
13229 let capture = EventCapture::default();
13230 let _subscriber =
13231 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13232 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
13233 let response = handler
13234 .handle_control_frame(
13235 &ctx,
13236 route_open_frame(394, "warming", unique_project_root("refusal-info")),
13237 )
13238 .await
13239 .unwrap();
13240 module.stop().await.unwrap();
13241
13242 assert_eq!(parse_error(&response[0])["code"], "module_warming");
13243 let event = capture
13244 .events()
13245 .into_iter()
13246 .find(|event| {
13247 event.target == "control"
13248 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
13249 })
13250 .expect("route.open refusal event");
13251 assert_eq!(
13252 event.fields.get("module_id"),
13253 Some(&"\"warming\"".to_string())
13254 );
13255 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
13256 assert_eq!(
13257 event.fields.get("reason"),
13258 Some(&"\"supervised_not_registered\"".to_string())
13259 );
13260 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
13261 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
13262 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
13263 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13264 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13265 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
13266 assert_eq!(
13267 handler.counters().snapshot()["route_open_refused_by_code"],
13268 json!({ "module_warming": 1 })
13269 );
13270 }
13271
13272 const OUTAGE_START: &str = "route.open refusing module: not serving";
13273 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
13274
13275 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
13276 capture
13277 .events()
13278 .into_iter()
13279 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
13280 .collect()
13281 }
13282
13283 fn supervise_stub(
13284 registry: &Arc<Registry>,
13285 module_id: &str,
13286 enabled: bool,
13287 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
13288 let supervisor_handle = SupervisorHandle::new();
13289 let supervisor =
13290 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
13291 .with_handle(supervisor_handle.clone())
13292 .with_connection_file_path(std::env::temp_dir().join(format!(
13293 "subc-route-outage-{module_id}-{}",
13294 std::process::id()
13295 )));
13296 let module = supervisor
13297 .supervise_configured(
13298 ModuleSpec {
13299 module_id: module_id.to_string(),
13300 program: fake_aft_stub_path(),
13301 args: Vec::new(),
13302 env: Vec::new(),
13303 reserved: false,
13304 reserved_prefixes: Vec::new(),
13305 protocol: ModuleProtocol::Subc,
13306 overlap: Default::default(),
13307 },
13308 enabled,
13309 )
13310 .unwrap();
13311 (supervisor_handle, module)
13312 }
13313
13314 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
13315 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
13316 module_id: module_id.to_string(),
13317 drain_timeout_ms: Some(50),
13318 })
13319 .unwrap();
13320 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
13321 }
13322
13323 #[test]
13327 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
13328 let registry = Arc::new(Registry::default());
13329 let forwarding = Arc::new(ForwardingTable::default());
13330 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13331 let second = ControlHandler::with_forwarding(registry, forwarding);
13332 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
13333 }
13334
13335 #[tokio::test(flavor = "current_thread")]
13339 async fn route_open_unknown_module_refusals_add_no_outage_state() {
13340 let handler = ControlHandler::new(Arc::new(Registry::default()));
13341 let capture = EventCapture::default();
13342 let _subscriber =
13343 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13344 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
13345 for corr in 0..8 {
13346 let response = handler
13347 .handle_control_frame(
13348 &ctx,
13349 route_open_frame(
13350 380 + corr,
13351 &format!("nobody-{corr}"),
13352 unique_project_root("outage-unknown"),
13353 ),
13354 )
13355 .await
13356 .unwrap();
13357 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13358 }
13359
13360 assert_eq!(handler.route_outages.tracked_module_count(), 0);
13361 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
13362 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
13363 }
13364
13365 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13370 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
13371 let registry = Arc::new(Registry::default());
13372 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
13373 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13374 let capture = EventCapture::default();
13375 let _subscriber =
13376 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13377 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
13378 handler.route_outages.record_accepted("outage-restart");
13381
13382 let response = handler
13383 .handle_control_frame(
13384 &ctx,
13385 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
13386 )
13387 .await
13388 .unwrap();
13389 assert_eq!(response[0].header.ty, FrameType::Error);
13390 let starts = outage_lines(&capture, OUTAGE_START);
13391 assert_eq!(starts.len(), 1, "{starts:?}");
13392 assert_eq!(starts[0].level, tracing::Level::WARN);
13393 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
13394 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
13395 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
13396 handler.route_outages.record_accepted("outage-restart");
13397 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
13398
13399 let restart = handler
13400 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
13401 .await
13402 .unwrap();
13403 assert_eq!(
13404 restart[0].header.ty,
13405 FrameType::Response,
13406 "{:?}",
13407 parse_error(&restart[0])
13408 );
13409 handler
13410 .handle_control_frame(
13411 &ctx,
13412 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
13413 )
13414 .await
13415 .unwrap();
13416 module.stop().await.unwrap();
13417
13418 let starts = outage_lines(&capture, OUTAGE_START);
13419 assert_eq!(starts.len(), 2, "{starts:?}");
13420 assert_eq!(starts[1].level, tracing::Level::INFO);
13421 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
13422 }
13423
13424 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13428 async fn failed_operator_restart_leaves_no_operator_mark() {
13429 let registry = Arc::new(Registry::default());
13430 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
13431 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13432 let capture = EventCapture::default();
13433 let _subscriber =
13434 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13435 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
13436 handler.route_outages.record_accepted("outage-disabled");
13437
13438 let restart = handler
13439 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
13440 .await
13441 .unwrap();
13442 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
13443 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
13444
13445 handler
13446 .handle_control_frame(
13447 &ctx,
13448 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
13449 )
13450 .await
13451 .unwrap();
13452 let starts = outage_lines(&capture, OUTAGE_START);
13453 assert_eq!(starts.len(), 1, "{starts:?}");
13454 assert_eq!(starts[0].level, tracing::Level::WARN);
13455 }
13456
13457 #[tokio::test(flavor = "current_thread")]
13458 async fn route_open_unknown_module_escapes_target_module_id() {
13459 let handler = ControlHandler::new(Arc::new(Registry::default()));
13460 let capture = EventCapture::default();
13461 let _subscriber =
13462 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13463 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
13464 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13465 let response = handler
13466 .handle_control_frame(
13467 &ctx,
13468 route_open_frame(
13469 395,
13470 hostile_module_id,
13471 unique_project_root("hostile-target-module-id"),
13472 ),
13473 )
13474 .await
13475 .unwrap();
13476
13477 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13478 let event = capture
13479 .events()
13480 .into_iter()
13481 .find(|event| {
13482 event.target == "control"
13483 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13484 })
13485 .expect("route.open unknown-module refusal event");
13486 let logged = event.fields.get("module_id").expect("module_id field");
13487 assert!(!logged.bytes().any(|byte| byte < 0x20));
13488 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13489 }
13490
13491 #[tokio::test(flavor = "current_thread")]
13492 async fn route_open_module_rejection_uses_daemon_counter_key() {
13493 let registry = Arc::new(Registry::default());
13494 let forwarding = Arc::new(ForwardingTable::default());
13495 let handler =
13496 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13497 let module_connection = ConnectionId::new(95);
13498 let (module_ctx, mut module_rx) = route_ctx(module_connection);
13499 hello_via_sink(
13500 &handler,
13501 &module_ctx,
13502 &mut module_rx,
13503 hello_frame("aft", PROTOCOL_VERSION, 395),
13504 )
13505 .await;
13506
13507 let client_connection = ConnectionId::new(96);
13508 let (client_ctx, _client_rx) = route_ctx(client_connection);
13509 let capture = EventCapture::default();
13510 let _subscriber =
13511 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13512 let (route_task, bind) = relay_route_open(
13513 &handler,
13514 client_connection,
13515 &client_ctx.egress,
13516 &mut module_rx,
13517 396,
13518 "aft",
13519 "hostile-module-code",
13520 )
13521 .await;
13522 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
13523 let rejection = Frame::build(
13524 FrameType::Error,
13525 control_flags(),
13526 0,
13527 0,
13528 bind.header.corr,
13529 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
13530 )
13531 .unwrap();
13532 handler
13533 .handle_control_frame(&module_ctx, rejection)
13534 .await
13535 .unwrap();
13536
13537 let response = route_task.await.unwrap();
13538 assert_eq!(parse_error(&response[0])["code"], hostile_code);
13539 let counters = handler.counters().snapshot();
13540 assert_eq!(
13541 counters["route_open_refused_by_code"],
13542 json!({ "module_rejected": 1 })
13543 );
13544 assert!(counters["route_open_refused_by_code"]
13545 .get(hostile_code)
13546 .is_none());
13547
13548 let event = capture
13549 .events()
13550 .into_iter()
13551 .find(|event| {
13552 event.target == "control"
13553 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
13554 })
13555 .expect("route.open module-rejection refusal event");
13556 let logged = event.fields.get("module_code").expect("module_code field");
13557 assert!(!logged.bytes().any(|byte| byte < 0x20));
13558 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13559 }
13560
13561 #[tokio::test]
13562 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
13563 let registry = Arc::new(Registry::default());
13564 let supervisor_handle = SupervisorHandle::new();
13565 let missing_program = std::env::temp_dir().join(format!(
13566 "subc-route-open-missing-program-{}",
13567 std::process::id()
13568 ));
13569 let supervisor =
13570 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13571 .with_handle(supervisor_handle.clone());
13572 let module = supervisor
13573 .supervise_configured(
13574 ModuleSpec {
13575 module_id: "failed".to_string(),
13576 program: missing_program,
13577 args: Vec::new(),
13578 env: Vec::new(),
13579 reserved: false,
13580 reserved_prefixes: Vec::new(),
13581 protocol: ModuleProtocol::Subc,
13582 overlap: Default::default(),
13583 },
13584 true,
13585 )
13586 .unwrap();
13587 assert_eq!(module.state().unwrap(), ModuleState::Failed);
13588
13589 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13590 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
13591 let response = handler
13592 .handle_control_frame(
13593 &ctx,
13594 route_open_frame(305, "failed", unique_project_root("failed")),
13595 )
13596 .await
13597 .unwrap();
13598
13599 assert_eq!(response[0].header.ty, FrameType::Error);
13600 let error = parse_error(&response[0]);
13601 assert_eq!(error["code"], "target_unavailable");
13602 assert!(error["message"]
13603 .as_str()
13604 .unwrap()
13605 .contains("state=failed, enabled=true, live=false"));
13606 }
13607
13608 #[tokio::test]
13609 async fn route_open_role_mismatch_remains_target_unavailable() {
13610 let registry = Arc::new(Registry::default());
13611 let handler = ControlHandler::new(Arc::clone(®istry));
13612 handler
13613 .handle_control(
13614 ConnectionId::new(41),
13615 non_routable_hello_frame_with_control_ops("health-only", 306, None),
13616 )
13617 .unwrap();
13618
13619 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
13620 let response = handler
13621 .handle_control_frame(
13622 &ctx,
13623 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
13624 )
13625 .await
13626 .unwrap();
13627
13628 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13629 assert!(parse_error(&response[0])["message"]
13630 .as_str()
13631 .unwrap()
13632 .contains("does not provide the requested target"));
13633 }
13634
13635 #[tokio::test]
13636 async fn route_open_inactive_registration_remains_target_unavailable() {
13637 let registry = Arc::new(Registry::default());
13638 let handler = ControlHandler::new(Arc::clone(®istry));
13639 handler
13640 .handle_control(
13641 ConnectionId::new(43),
13642 hello_frame("inactive", PROTOCOL_VERSION, 308),
13643 )
13644 .unwrap();
13645 assert!(registry
13646 .set_module_state_for_test("inactive", ChannelState::Closed)
13647 .unwrap());
13648
13649 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
13650 let response = handler
13651 .handle_control_frame(
13652 &ctx,
13653 route_open_frame(309, "inactive", unique_project_root("inactive")),
13654 )
13655 .await
13656 .unwrap();
13657
13658 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13659 assert!(parse_error(&response[0])["message"]
13660 .as_str()
13661 .unwrap()
13662 .contains("is not active"));
13663 }
13664
13665 #[tokio::test]
13666 async fn late_health_reply_is_recorded_through_the_module_response_path() {
13667 let registry = Arc::new(Registry::default());
13668 let forwarding = Arc::new(ForwardingTable::default());
13669 let supervisor_handle = SupervisorHandle::new();
13670 let supervisor =
13671 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
13672 .with_forwarding(Arc::clone(&forwarding))
13673 .with_handle(supervisor_handle.clone());
13674 let module = supervisor
13675 .supervise_configured(
13676 crate::ModuleSpec {
13677 module_id: "late-health-response".to_string(),
13678 program: PathBuf::from("disabled-module"),
13679 args: Vec::new(),
13680 env: Vec::new(),
13681 reserved: false,
13682 reserved_prefixes: Vec::new(),
13683 protocol: ModuleProtocol::Subc,
13684 overlap: Default::default(),
13685 },
13686 false,
13687 )
13688 .unwrap();
13689 let handler =
13690 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13691 .with_supervisor(supervisor_handle);
13692 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
13693 handler
13694 .handle_control_frame(
13695 &module_ctx,
13696 hello_frame_with_control_ops(
13697 "late-health-response",
13698 PROTOCOL_VERSION,
13699 7,
13700 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13701 ),
13702 )
13703 .await
13704 .unwrap();
13705 let probe_started_at = Instant::now() - Duration::from_millis(80);
13706 let pending = forwarding
13707 .begin_health_probe_rpc_for(
13708 "late-health-response",
13709 MODULE_CONTROL_OP_HEALTH_CHECK,
13710 probe_started_at,
13711 Instant::now() - Duration::from_millis(1),
13712 )
13713 .unwrap();
13714 assert!(forwarding
13715 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
13716 .unwrap());
13717
13718 let responses = handler
13719 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
13720 .await
13721 .unwrap();
13722
13723 assert!(responses.is_empty());
13724 let health = module.status().unwrap().health;
13725 assert_eq!(health.late_answer_count, 1);
13726 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
13727 }
13728
13729 #[tokio::test]
13730 async fn health_probe_timeout_and_module_death_are_typed() {
13731 let registry = Arc::new(Registry::default());
13732 let forwarding = Arc::new(ForwardingTable::default());
13733 let handler =
13734 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13735 .with_health_probe_timeout(Duration::from_millis(50));
13736 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
13737 hello_via_sink(
13738 &handler,
13739 &module_ctx,
13740 &mut module_rx,
13741 hello_frame_with_control_ops(
13742 "aft",
13743 PROTOCOL_VERSION,
13744 7,
13745 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13746 ),
13747 )
13748 .await;
13749
13750 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
13751 let responses = handler
13752 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
13753 .await
13754 .unwrap();
13755 assert_eq!(responses[0].header.ty, FrameType::Error);
13756 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
13757 let _ = module_rx.try_recv();
13758
13759 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
13760 let health_handler = handler.clone();
13761 let death_task = tokio::spawn(async move {
13762 health_handler
13763 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
13764 .await
13765 .unwrap()
13766 });
13767 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13768 .await
13769 .unwrap()
13770 .unwrap();
13771 handler
13772 .cleanup_connection(module_ctx.connection_id)
13773 .unwrap();
13774 let responses = death_task.await.unwrap();
13775 assert_eq!(responses[0].header.ty, FrameType::Error);
13776 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
13777 }
13778
13779 #[test]
13780 fn hello_requires_exact_protocol_version() {
13781 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
13782 let registry = Arc::new(Registry::default());
13783 let handler = ControlHandler::new(Arc::clone(®istry));
13784 let responses = handler
13785 .handle_control(
13786 ConnectionId::new(connection),
13787 hello_frame("aft", offered, 9),
13788 )
13789 .unwrap();
13790
13791 assert_eq!(responses.len(), 1);
13792 assert_eq!(responses[0].header.ty, FrameType::Error);
13793 let error = parse_error(&responses[0]);
13794 assert_eq!(error["code"], "version_unsupported");
13795 assert!(registry.get_module("aft").unwrap().is_none());
13796 assert_eq!(registry.active_registration_count().unwrap(), 0);
13797 }
13798 }
13799
13800 #[test]
13801 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
13802 let registry = Arc::new(Registry::default());
13803 let forwarding = Arc::new(ForwardingTable::default());
13804 let handler =
13805 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13806 let module_connection = ConnectionId::new(301);
13807 let registration = registry
13808 .register_with_control_ops(
13809 manifest("aft-push", PROTOCOL_VERSION),
13810 PROTOCOL_VERSION,
13811 module_connection,
13812 module_baseline_control_ops(),
13813 )
13814 .unwrap();
13815 let (module_tx, _module_rx) = mpsc::channel(8);
13816 let endpoint = forwarding
13817 .register_module_connection(
13818 module_connection,
13819 "aft-push".to_string(),
13820 PROTOCOL_VERSION,
13821 manifest_concurrency(®istration.manifest),
13822 FrameSink::new(module_tx),
13823 )
13824 .unwrap();
13825
13826 let unknown = Frame::build(
13828 FrameType::Push,
13829 control_flags(),
13830 0,
13831 0,
13832 5,
13833 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
13834 )
13835 .unwrap();
13836 let out = handler.handle_status_update(endpoint, unknown).unwrap();
13837 assert!(
13838 out.is_empty(),
13839 "unknown push op must be ignored, got {out:?}"
13840 );
13841
13842 let malformed = Frame::build(
13844 FrameType::Push,
13845 control_flags(),
13846 0,
13847 0,
13848 6,
13849 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
13850 )
13851 .unwrap();
13852 let out = handler.handle_status_update(endpoint, malformed).unwrap();
13853 assert_eq!(out.len(), 1);
13854 assert_eq!(out[0].header.ty, FrameType::Error);
13855 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
13856 }
13857
13858 #[test]
13859 fn hello_rejected_when_connection_already_owns_client_routes() {
13860 let registry = Arc::new(Registry::default());
13861 let forwarding = Arc::new(ForwardingTable::default());
13862 let handler =
13863 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13864 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
13866 let client_connection = ConnectionId::new(202);
13867
13868 let responses = handler
13871 .handle_control(
13872 client_connection,
13873 hello_frame("aft-second", PROTOCOL_VERSION, 9),
13874 )
13875 .unwrap();
13876 assert_eq!(responses[0].header.ty, FrameType::Error);
13877 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
13878 assert!(registry.get_module("aft-second").unwrap().is_none());
13879 }
13880
13881 #[tokio::test]
13882 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
13883 let registry = Arc::new(Registry::default());
13884 let forwarding = Arc::new(ForwardingTable::default());
13885 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
13886 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
13887 hello_via_sink(
13888 &handler,
13889 &module_ctx,
13890 &mut module_rx,
13891 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
13892 )
13893 .await;
13894 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
13895 let pending = forwarding
13896 .begin_route_bind_relay_for_test(
13897 client_ctx.connection_id,
13898 client_ctx.egress.clone(),
13899 2,
13900 "alpha",
13901 )
13902 .unwrap();
13903 forwarding
13904 .complete_pending_relay(
13905 module_ctx.connection_id,
13906 pending.corr,
13907 RouteBindRelayOutcome::Accepted,
13908 )
13909 .unwrap();
13910 client_rx.try_recv().unwrap();
13911 for module_id in ["beta", "alpha"] {
13912 let replies = handler
13913 .handle_control_frame(
13914 &module_ctx,
13915 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
13916 )
13917 .await
13918 .unwrap();
13919 assert_eq!(replies.len(), 1, "second HELLO must be refused");
13920 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
13921 }
13922 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
13923 assert!(registry.get_module("beta").unwrap().is_none());
13924 assert!(matches!(
13925 forwarding
13926 .lookup_data_route(
13927 client_ctx.connection_id,
13928 pending.client_channel,
13929 pending.client_epoch,
13930 )
13931 .unwrap(),
13932 DataRoute::Client(DataRouteState::Bound(_))
13933 ));
13934 assert!(handler
13935 .hello_launch_nonces
13936 .lock()
13937 .unwrap()
13938 .presented(module_ctx.connection_id, Some("alpha-nonce")));
13939 assert!(module_rx.try_recv().is_err());
13940 }
13941
13942 #[test]
13943 fn reserved_module_hello_requires_matching_launch_nonce() {
13944 let registry = Arc::new(Registry::default());
13945 let supervisor = SupervisorHandle::new();
13946 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
13949 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
13950
13951 let no_nonce = handler
13953 .handle_control(
13954 ConnectionId::new(1),
13955 hello_frame("vault", PROTOCOL_VERSION, 1),
13956 )
13957 .unwrap();
13958 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
13959 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
13960 assert!(registry.get_module("vault").unwrap().is_none());
13961
13962 let wrong = handler
13964 .handle_control(
13965 ConnectionId::new(2),
13966 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
13967 )
13968 .unwrap();
13969 assert_eq!(wrong[0].header.ty, FrameType::Error);
13970 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
13971 assert!(registry.get_module("vault").unwrap().is_none());
13972
13973 let ok = handler
13975 .handle_control(
13976 ConnectionId::new(3),
13977 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
13978 )
13979 .unwrap();
13980 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
13981 assert!(registry.get_module("vault").unwrap().is_some());
13982 }
13983
13984 #[test]
13985 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
13986 let registry = Arc::new(Registry::default());
13987 let supervisor = SupervisorHandle::new();
13988 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13989 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13990 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
13991
13992 let squat = handler
13993 .handle_control(
13994 ConnectionId::new(1),
13995 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
13996 )
13997 .unwrap();
13998 assert_eq!(squat[0].header.ty, FrameType::Error);
13999 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
14000 assert!(parse_error(&squat[0])["message"]
14001 .as_str()
14002 .unwrap()
14003 .contains("fed:"));
14004
14005 let accepted_peer = handler
14006 .handle_control(
14007 ConnectionId::new(2),
14008 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
14009 )
14010 .unwrap();
14011 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
14012
14013 let accepted_short = handler
14014 .handle_control(
14015 ConnectionId::new(3),
14016 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
14017 )
14018 .unwrap();
14019 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
14020
14021 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
14022 let response = handler
14023 .handle_control(
14024 ConnectionId::new(conn),
14025 hello_frame(module_id, PROTOCOL_VERSION, conn),
14026 )
14027 .unwrap();
14028 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
14029 }
14030 }
14031
14032 #[test]
14033 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
14034 let registry = Arc::new(Registry::default());
14035 let supervisor = SupervisorHandle::new();
14036 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14037 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14038 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
14039 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14040
14041 let owner_nonce = handler
14042 .handle_control(
14043 ConnectionId::new(1),
14044 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
14045 )
14046 .unwrap();
14047 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
14048 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
14049 assert!(registry.get_module("fed:special").unwrap().is_none());
14050
14051 let exact_nonce = handler
14052 .handle_control(
14053 ConnectionId::new(2),
14054 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
14055 )
14056 .unwrap();
14057 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
14058 assert!(registry.get_module("fed:special").unwrap().is_some());
14059 }
14060
14061 #[test]
14062 fn non_reserved_module_ignores_launch_nonce() {
14063 let registry = Arc::new(Registry::default());
14064 let handler = ControlHandler::new(Arc::clone(®istry));
14067 let no_nonce = handler
14068 .handle_control(
14069 ConnectionId::new(1),
14070 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
14071 )
14072 .unwrap();
14073 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
14074 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
14075
14076 let echoed_nonce = handler
14077 .handle_control(
14078 ConnectionId::new(2),
14079 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
14080 )
14081 .unwrap();
14082 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
14083 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
14084 }
14085
14086 #[test]
14087 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
14088 let handler = ControlHandler::default();
14089 let conn = ConnectionId::new(1);
14090 let malformed = Frame::build(
14091 FrameType::Hello,
14092 control_flags(),
14093 0,
14094 0,
14095 3,
14096 b"{not json".to_vec(),
14097 )
14098 .unwrap();
14099
14100 let error = handler.handle_control(conn, malformed).unwrap();
14101 assert_eq!(error[0].header.ty, FrameType::Error);
14102 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
14103
14104 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14105 let pong = handler.handle_control(conn, ping).unwrap();
14106 assert_eq!(pong[0].header.ty, FrameType::Pong);
14107 assert_eq!(pong[0].header.corr, 4);
14108 }
14109
14110 #[test]
14111 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
14112 let registry = Arc::new(Registry::default());
14113 let handler = ControlHandler::new(Arc::clone(®istry));
14114
14115 handler
14116 .handle_control(
14117 ConnectionId::new(1),
14118 hello_frame("aft", PROTOCOL_VERSION, 1),
14119 )
14120 .unwrap();
14121 let duplicate = handler
14122 .handle_control(
14123 ConnectionId::new(2),
14124 hello_frame("aft", PROTOCOL_VERSION, 2),
14125 )
14126 .unwrap();
14127
14128 assert_eq!(duplicate[0].header.ty, FrameType::Error);
14129 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14130 let registration = registry.get_module("aft").unwrap().unwrap();
14131 assert_eq!(registration.connection_id, ConnectionId::new(1));
14132 }
14133
14134 #[test]
14135 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
14136 let registry = Arc::new(Registry::default());
14137 let forwarding = Arc::new(ForwardingTable::default());
14138 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
14139 let handler =
14140 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14141 .with_process_liveness(process_liveness);
14142 let (ctx, route_channel, route_epoch) =
14143 bind_liveness_route(®istry, &forwarding, "aft-dead");
14144 let responses = handler
14145 .handle_route_poll(
14146 &ctx,
14147 route_poll_frame(41, PollKind::Liveness, route_channel),
14148 route_channel,
14149 route_epoch,
14150 PollKind::Liveness,
14151 )
14152 .unwrap();
14153
14154 assert_eq!(responses.len(), 1);
14155 assert_eq!(responses[0].header.ty, FrameType::Response);
14156 assert_route_poll_liveness(&responses[0], false);
14157 }
14158
14159 #[test]
14160 fn liveness_poll_without_process_source_uses_bound_route() {
14161 let registry = Arc::new(Registry::default());
14162 let forwarding = Arc::new(ForwardingTable::default());
14163 let handler =
14164 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14165 let (ctx, route_channel, route_epoch) =
14166 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
14167 let responses = handler
14168 .handle_route_poll(
14169 &ctx,
14170 route_poll_frame(42, PollKind::Liveness, route_channel),
14171 route_channel,
14172 route_epoch,
14173 PollKind::Liveness,
14174 )
14175 .unwrap();
14176
14177 assert_route_poll_liveness(&responses[0], true);
14178 }
14179
14180 #[test]
14181 fn liveness_poll_untracked_process_source_uses_bound_route() {
14182 let registry = Arc::new(Registry::default());
14183 let forwarding = Arc::new(ForwardingTable::default());
14184 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
14185 let handler =
14186 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14187 .with_process_liveness(process_liveness);
14188 let (ctx, route_channel, route_epoch) =
14189 bind_liveness_route(®istry, &forwarding, "aft-untracked");
14190 let responses = handler
14191 .handle_route_poll(
14192 &ctx,
14193 route_poll_frame(43, PollKind::Liveness, route_channel),
14194 route_channel,
14195 route_epoch,
14196 PollKind::Liveness,
14197 )
14198 .unwrap();
14199
14200 assert_route_poll_liveness(&responses[0], true);
14201 }
14202
14203 #[tokio::test]
14204 async fn unknown_op_returns_unknown_control_op() {
14205 let handler = ControlHandler::default();
14206 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
14207 let request = Frame::build(
14208 FrameType::Request,
14209 control_flags(),
14210 0,
14211 0,
14212 55,
14213 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
14214 )
14215 .unwrap();
14216
14217 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14218
14219 assert_eq!(response.len(), 1);
14220 assert_eq!(response[0].header.ty, FrameType::Error);
14221 assert_eq!(response[0].header.corr, 55);
14222 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
14223 }
14224
14225 #[tokio::test]
14226 async fn supervisor_provenance_rejects_unknown_exact_module() {
14227 let handler = ControlHandler::default();
14228 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
14229 let request = Frame::build(
14230 FrameType::Request,
14231 control_flags(),
14232 0,
14233 0,
14234 57,
14235 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
14236 )
14237 .unwrap();
14238
14239 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14240
14241 assert_eq!(response.len(), 1);
14242 assert_eq!(response[0].header.ty, FrameType::Error);
14243 assert_eq!(response[0].header.corr, 57);
14244 let error = parse_error(&response[0]);
14245 assert_eq!(error["code"], "unknown_module");
14246 assert_eq!(error["message"], "module_id 'missing' is not supervised");
14247 }
14248
14249 #[test]
14250 fn provenance_probe_override_keeps_handler_tests_deterministic() {
14251 let expected = subc_control::RunningImageAgreement::Unavailable {
14252 reason: subc_control::RunningImageUnavailableReason::HashFailed,
14253 };
14254 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
14255 assert_eq!(handler.provenance_probe_override, Some(expected));
14256 }
14257
14258 #[test]
14259 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
14260 let verdict = reload_verdict(
14261 std::path::Path::new("/bin/new"),
14262 Some(std::path::Path::new("/bin/old")),
14263 subc_control::RunningImageAgreement::Unavailable {
14264 reason: subc_control::RunningImageUnavailableReason::HashFailed,
14265 },
14266 );
14267 assert!(matches!(
14268 verdict.path,
14269 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
14270 if configured == std::path::Path::new("/bin/new")
14271 && spawned_from == std::path::Path::new("/bin/old")
14272 ));
14273 }
14274
14275 #[test]
14276 fn launch_spec_without_running_digest_is_unavailable_not_match() {
14277 use subc_control::{
14278 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
14279 };
14280 let configured = crate::supervise::ModuleSpec {
14281 module_id: "legacy".into(),
14282 program: "/module".into(),
14283 args: Vec::new(),
14284 env: Vec::new(),
14285 reserved: false,
14286 reserved_prefixes: Vec::new(),
14287 protocol: subc_control::ModuleProtocol::Subc,
14288 overlap: Default::default(),
14289 };
14290 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
14291 assert_eq!(
14292 launch_spec_agreement(&configured, false, Some(&digest)),
14293 Agreement::Unavailable {
14294 reason: Reason::NotRunning
14295 }
14296 );
14297 assert_eq!(
14298 launch_spec_agreement(&configured, true, None),
14299 Agreement::Unavailable {
14300 reason: Reason::SpawnedSpecUnavailable
14301 }
14302 );
14303 assert_eq!(
14304 launch_spec_agreement(&configured, true, Some(&digest)),
14305 Agreement::Match
14306 );
14307 }
14308
14309 #[cfg(windows)]
14310 #[test]
14311 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
14312 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
14313 let configured = dir.join("module.exe");
14314 std::fs::write(&configured, b"executable path fixture").unwrap();
14315 let spawned = std::fs::canonicalize(&configured).unwrap();
14316 assert_ne!(
14317 configured, spawned,
14318 "fixture must exercise different path spellings"
14319 );
14320 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
14321 let lower_case = configured.to_string_lossy().to_lowercase();
14322 for spelling in [
14323 &configured,
14324 Path::new(&forward_slashes),
14325 Path::new(&lower_case),
14326 ] {
14327 let verdict = reload_verdict(
14328 spelling,
14329 Some(&spawned),
14330 subc_control::RunningImageAgreement::Unavailable {
14331 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14332 },
14333 );
14334 assert_eq!(
14335 verdict.path,
14336 ReloadPathAgreement::Match,
14337 "configured {spelling:?}, spawned {spawned:?}"
14338 );
14339 }
14340 }
14341
14342 #[cfg(windows)]
14343 #[test]
14344 fn windows_reload_path_keeps_different_existing_program_pending() {
14345 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
14346 let old = dir.join("old.exe");
14347 let new = dir.join("new.exe");
14348 std::fs::write(&old, b"same executable content").unwrap();
14349 std::fs::copy(&old, &new).unwrap();
14350 let spawned = std::fs::canonicalize(&old).unwrap();
14351 let image = subc_control::RunningImageAgreement::Unavailable {
14352 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14353 };
14354 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
14355 assert!(
14356 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
14357 if configured == new && spawned_from == spawned)
14358 );
14359 let missing = dir.join("missing.exe");
14360 assert!(matches!(
14361 reload_verdict(&missing, Some(&spawned), image).path,
14362 ReloadPathAgreement::Mismatch { .. }
14363 ));
14364 }
14365
14366 #[test]
14367 fn reload_verdict_detects_replaced_image_at_same_path() {
14368 let image = subc_control::RunningImageAgreement::Mismatch {
14369 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
14370 digest: "old".into(),
14371 },
14372 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
14373 digest: "new".into(),
14374 },
14375 };
14376 let verdict = reload_verdict(
14377 std::path::Path::new("/bin/same"),
14378 Some(std::path::Path::new("/bin/same")),
14379 image.clone(),
14380 );
14381 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14382 assert_eq!(verdict.image, image);
14383 }
14384
14385 #[test]
14386 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
14387 let image = subc_control::RunningImageAgreement::Unavailable {
14388 reason: subc_control::RunningImageUnavailableReason::NotRunning,
14389 };
14390 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
14391 assert_eq!(
14392 verdict.path,
14393 subc_control::ReloadPathAgreement::Unavailable {
14394 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
14395 }
14396 );
14397 assert_eq!(verdict.image, image);
14398
14399 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
14400 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14401 };
14402 let verdict = reload_verdict(
14403 std::path::Path::new("/bin/same"),
14404 Some(std::path::Path::new("/bin/same")),
14405 unconfirmed.clone(),
14406 );
14407 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14408 assert_eq!(verdict.image, unconfirmed);
14409 }
14410
14411 #[test]
14412 fn reload_verdict_preserves_each_image_unavailability_reason() {
14413 use subc_control::RunningImageUnavailableReason as Reason;
14414
14415 for reason in [
14416 Reason::NotRunning,
14417 Reason::UnsupportedPlatform,
14418 Reason::RunningExecutableUnreadable,
14419 Reason::SpawnedPathUnreadable,
14420 Reason::HashFailed,
14421 Reason::ProcessIdentityUnconfirmed,
14422 Reason::Unknown("future_probe_reason".to_string()),
14423 ] {
14424 let image = subc_control::RunningImageAgreement::Unavailable {
14425 reason: reason.clone(),
14426 };
14427 let verdict = reload_verdict(
14428 std::path::Path::new("/bin/same"),
14429 Some(std::path::Path::new("/bin/same")),
14430 image.clone(),
14431 );
14432 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14433 assert_eq!(verdict.image, image, "{reason:?}");
14434 }
14435 }
14436
14437 #[tokio::test]
14438 async fn malformed_control_bodies_return_invalid_control_body() {
14439 let handler = ControlHandler::default();
14440 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
14441
14442 for (corr, body) in [
14443 (56, br#"{"route_channel":1}"#.as_slice()),
14444 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
14445 (
14446 58,
14447 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
14448 ),
14449 ] {
14450 let request = Frame::build(
14451 FrameType::Request,
14452 control_flags(),
14453 0,
14454 0,
14455 corr,
14456 body.to_vec(),
14457 )
14458 .unwrap();
14459 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14460
14461 assert_eq!(response.len(), 1);
14462 assert_eq!(response[0].header.ty, FrameType::Error);
14463 assert_eq!(response[0].header.corr, corr);
14464 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
14465 }
14466 }
14467
14468 #[tokio::test]
14469 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
14470 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14471 let registry = Arc::new(Registry::default());
14472 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
14473 let router = Router::with_control_handler(Arc::clone(&control));
14474 let connection = router.begin_connection();
14475 let (ctx, mut rx) = route_ctx(connection.id());
14476
14477 router
14478 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
14479 .await
14480 .unwrap();
14481 let response = rx.recv().await.unwrap();
14482 let ack = parse_ack(&response);
14483 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14484 let channel = 1;
14485
14486 let goodbye =
14487 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
14488 router.route_for_connection(&ctx, goodbye).await.unwrap();
14489 assert!(rx.try_recv().is_err());
14490 assert!(registry.get_module("aft").unwrap().is_none());
14491
14492 router
14493 .route_for_connection(&ctx, channel_request(channel, 13))
14494 .await
14495 .unwrap();
14496 let error_frame = rx.recv().await.unwrap();
14497 assert_eq!(error_frame.header.ty, FrameType::Error);
14498 assert_eq!(error_frame.header.channel, channel);
14499 let captured = crate::router::test_log::captured_logs(&logs);
14500 assert_eq!(
14501 captured
14502 .lines()
14503 .filter(|line| {
14504 line.contains("module_id=aft")
14505 && line.contains("reason=explicit_goodbye")
14506 && line.contains("module registration ended")
14507 })
14508 .count(),
14509 1,
14510 "unexpected GOODBYE registry log: {captured}"
14511 );
14512 }
14513
14514 #[tokio::test]
14515 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
14516 let registry = Arc::new(Registry::default());
14517 let handler = ControlHandler::new(registry).with_capability_config(
14518 [("prov".to_string(), true), ("cons".to_string(), true)],
14519 BTreeMap::new(),
14520 );
14521 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
14522 register_capability_manifest(
14523 &handler,
14524 &provider_ctx,
14525 &mut provider_rx,
14526 capability_manifest("prov", &["thing/v1"], &[]),
14527 1,
14528 )
14529 .await;
14530 let mut consumer = capability_manifest("cons", &[], &[]);
14531 consumer.capabilities.as_mut().unwrap().requires.push(
14532 subc_protocol::manifest::CapabilityRequirement {
14533 capability: "thing/v1".to_string(),
14534 need: subc_protocol::manifest::CapabilityNeed::Required,
14535 },
14536 );
14537 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
14538 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
14539 assert_eq!(
14540 handler.capability_evaluator.verdict("cons", "thing/v1"),
14541 Some(CapabilityVerdict::Provided)
14542 );
14543 let (mut client_rx, _) = open_route_for_capability_test(
14544 &handler,
14545 &provider_ctx,
14546 &mut provider_rx,
14547 703,
14548 3,
14549 "prov",
14550 None,
14551 )
14552 .await;
14553 let goodbye =
14554 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14555 handler
14556 .handle_control_frame(&provider_ctx, goodbye)
14557 .await
14558 .unwrap();
14559 assert_eq!(
14560 handler.capability_evaluator.verdict("cons", "thing/v1"),
14561 Some(CapabilityVerdict::NeverProvided)
14562 );
14563 let closed = client_rx
14564 .try_recv()
14565 .expect("GOODBYE pushes route.closed before route GOODBYE");
14566 assert!(
14567 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
14568 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
14569 );
14570 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
14571 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14572 }
14573
14574 struct RouteClosedFixture {
14575 handler: ControlHandler,
14576 #[cfg(unix)]
14577 supervisor: Supervisor,
14578 module_ctx: RouteCtx,
14579 #[cfg(unix)]
14580 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14581 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14582 route: Arc<crate::forwarding::RouteBinding>,
14583 }
14584
14585 async fn route_closed_fixture() -> RouteClosedFixture {
14586 let registry = Arc::new(Registry::default());
14587 let handler = ControlHandler::new(Arc::clone(®istry));
14588 #[cfg(unix)]
14589 let supervisor = Supervisor::new(registry, RestartPolicy::default())
14590 .with_forwarding(Arc::clone(&handler.forwarding));
14591 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
14592 register_capability_manifest(
14593 &handler,
14594 &module_ctx,
14595 &mut module_rx,
14596 capability_manifest("prov", &[], &[]),
14597 1,
14598 )
14599 .await;
14600 let (client_rx, bind) = open_route_for_capability_test(
14601 &handler,
14602 &module_ctx,
14603 &mut module_rx,
14604 702,
14605 2,
14606 "prov",
14607 None,
14608 )
14609 .await;
14610 let ModuleControlRequest::RouteBind {
14611 route_channel: channel,
14612 epoch: route_epoch,
14613 ..
14614 } = bind
14615 else {
14616 panic!("expected route.bind");
14617 };
14618 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
14619 handler
14620 .forwarding
14621 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
14622 .unwrap()
14623 else {
14624 panic!("the acknowledged route must be bound");
14625 };
14626 RouteClosedFixture {
14627 handler,
14628 #[cfg(unix)]
14629 supervisor,
14630 module_ctx,
14631 #[cfg(unix)]
14632 module_rx,
14633 client_rx,
14634 route,
14635 }
14636 }
14637
14638 impl RouteClosedFixture {
14639 #[cfg(unix)]
14640 async fn cut(&mut self) {
14641 let notices = async {
14642 let mut draining = self.module_rx.recv().await.unwrap();
14643 assert!(matches!(
14644 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
14645 &draining.body
14646 )
14647 .unwrap(),
14648 subc_protocol::session::ModuleControlCommand::Draining {
14649 reason: RouteCloseReason::Restart,
14650 ..
14651 }
14652 ));
14653 draining.flushed.take().unwrap().send(()).unwrap();
14654 let mut closing = self.client_rx.recv().await.unwrap();
14655 assert!(matches!(
14656 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
14657 ClientControlPush::RouteClosing {
14658 reason: RouteCloseReason::Restart,
14659 ..
14660 }
14661 ));
14662 closing.flushed.take().unwrap().send(()).unwrap();
14663 };
14664 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
14665 result.unwrap();
14666 }
14667
14668 fn disconnect(&mut self) -> ClientControlPush {
14669 self.handler
14670 .cleanup_connection(self.module_ctx.connection_id)
14671 .unwrap();
14672 let closed = self.client_rx.try_recv().unwrap();
14673 assert_eq!(closed.header.ty, FrameType::Push);
14674 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
14675 assert!(matches!(
14676 &push,
14677 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
14678 if channels == &[self.route.client_channel]
14679 ));
14680 assert_eq!(
14681 self.client_rx.try_recv().unwrap().header.ty,
14682 FrameType::Goodbye
14683 );
14684 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
14685 push
14686 }
14687 }
14688
14689 #[cfg(unix)]
14690 #[tokio::test(start_paused = true)]
14691 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
14692 let mut fixture = route_closed_fixture().await;
14693 fixture.cut().await;
14694 assert!(matches!(
14695 fixture.disconnect(),
14696 ClientControlPush::RouteClosed {
14697 reason: RouteCloseReason::Restart,
14698 drained: true,
14699 ..
14700 }
14701 ));
14702 }
14703
14704 #[cfg(unix)]
14705 #[tokio::test(start_paused = true)]
14706 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
14707 let mut fixture = route_closed_fixture().await;
14708 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
14709 let started = tokio::time::Instant::now();
14710 fixture.cut().await;
14711 assert!(started.elapsed() >= Duration::from_secs(2));
14712 assert_eq!(fixture.route.flow.in_flight(), 1);
14713 assert!(matches!(
14714 fixture.disconnect(),
14715 ClientControlPush::RouteClosed {
14716 reason: RouteCloseReason::Restart,
14717 drained: false,
14718 ..
14719 }
14720 ));
14721 }
14722
14723 #[tokio::test]
14724 async fn module_connection_crash_reports_crash_and_not_drained() {
14725 let mut fixture = route_closed_fixture().await;
14726 assert!(matches!(
14730 fixture.disconnect(),
14731 ClientControlPush::RouteClosed {
14732 reason: RouteCloseReason::Crash,
14733 drained: false,
14734 ..
14735 }
14736 ));
14737 }
14738
14739 #[tokio::test]
14740 async fn dropping_router_connection_releases_registration() {
14741 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14742 let registry = Arc::new(Registry::default());
14743 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
14744 let router = Router::with_control_handler(Arc::clone(&control));
14745 let connection = router.begin_connection();
14746 let connection_id = connection.id();
14747 let (ctx, mut rx) = route_ctx(connection_id);
14748
14749 router
14750 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
14751 .await
14752 .unwrap();
14753 let response = rx.recv().await.unwrap();
14754 let ack = parse_ack(&response);
14755 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14756 assert!(registry.get_module("aft").unwrap().is_some());
14757
14758 drop(connection);
14759
14760 assert!(registry.get_module("aft").unwrap().is_none());
14761 assert_eq!(registry.active_registration_count().unwrap(), 0);
14762
14763 control
14764 .cleanup_connection(ConnectionId::new(u64::MAX))
14765 .unwrap();
14766 let captured = crate::router::test_log::captured_logs(&logs);
14767 println!("captured registry lifecycle logs:\n{captured}");
14768 let events: Vec<_> = captured
14769 .lines()
14770 .filter(|line| {
14771 line.contains("module registered module_id=aft ")
14772 || line.contains("module registration ended")
14773 })
14774 .collect();
14775 assert_eq!(
14776 events.len(),
14777 2,
14778 "unexpected registry lifecycle logs: {captured}"
14779 );
14780 assert!(events[0].contains("module registered module_id=aft "));
14781 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
14782 assert!(events[1].contains(&format!(
14783 "module_id=aft connection_id={}",
14784 connection_id.get()
14785 )));
14786 assert!(events[1].contains("reason=connection_closed"));
14787 assert!(events[1].contains("module registration ended"));
14788 assert_eq!(
14789 captured
14790 .lines()
14791 .filter(|line| line.contains("module registered module_id=aft "))
14792 .count(),
14793 1,
14794 "legacy registration admission line must appear once: {captured}"
14795 );
14796 }
14797
14798 #[tokio::test]
14799 async fn hello_registration_keeps_legacy_module_registered_line_once() {
14800 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14801 let registry = Arc::new(Registry::default());
14802 let control = ControlHandler::new(Arc::clone(®istry));
14803 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
14804 hello_via_sink(
14805 &control,
14806 &ctx,
14807 &mut rx,
14808 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
14809 )
14810 .await;
14811
14812 let captured = crate::router::test_log::captured_logs(&logs);
14813 let admissions: Vec<_> = captured
14814 .lines()
14815 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
14816 .collect();
14817 assert_eq!(
14818 admissions.len(),
14819 1,
14820 "legacy admission line must remain exactly once: {captured}"
14821 );
14822 assert!(admissions[0].contains("routable_provider=true"));
14823 assert!(admissions[0].contains("connection_id=777"));
14824 }
14825
14826 fn capability_manifest(
14827 module_id: &str,
14828 provides: &[&str],
14829 must_never_reach: &[&str],
14830 ) -> ModuleManifest {
14831 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
14832 manifest.capabilities = Some(CapabilityDeclarations {
14833 provides: provides
14834 .iter()
14835 .map(|capability| (*capability).to_string())
14836 .collect(),
14837 requires: Vec::new(),
14838 must_never_reach: must_never_reach
14839 .iter()
14840 .map(|capability| (*capability).to_string())
14841 .collect(),
14842 });
14843 manifest
14844 }
14845
14846 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
14847 Frame::build(
14848 FrameType::Hello,
14849 control_flags(),
14850 0,
14851 0,
14852 corr,
14853 serde_json::to_vec(&ModuleHelloBody {
14854 protocol_ver: manifest.protocol_ver,
14855 manifest,
14856 control_ops: None,
14857 launch_nonce: None,
14858 })
14859 .expect("capability test HELLO serializes"),
14860 )
14861 .expect("capability test HELLO frame builds")
14862 }
14863
14864 fn catalog_update_with_capabilities_frame(
14865 corr: u64,
14866 capabilities: CapabilityDeclarations,
14867 ) -> Frame {
14868 Frame::build(
14869 FrameType::Request,
14870 control_flags(),
14871 0,
14872 0,
14873 corr,
14874 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14875 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
14876 capabilities: Some(capabilities),
14877 ready: None,
14878 })
14879 .expect("capability catalog.update serializes"),
14880 )
14881 .expect("capability catalog.update frame builds")
14882 }
14883
14884 async fn register_capability_manifest(
14885 handler: &ControlHandler,
14886 ctx: &RouteCtx,
14887 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14888 manifest: ModuleManifest,
14889 corr: u64,
14890 ) {
14891 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
14892 }
14893
14894 async fn open_route_for_capability_test(
14895 handler: &ControlHandler,
14896 target_ctx: &RouteCtx,
14897 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14898 client_connection_id: u64,
14899 corr: u64,
14900 target_module_id: &str,
14901 consumer_identity: Option<ConsumerIdentity>,
14902 ) -> (
14903 mpsc::Receiver<crate::router::OutboundFrame>,
14904 ModuleControlRequest,
14905 ) {
14906 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
14907 let route_handler = handler.clone();
14908 let target_module_id = target_module_id.to_string();
14909 let route_task = tokio::spawn(async move {
14910 route_handler
14911 .handle_control_frame(
14912 &client_ctx,
14913 route_open_frame_with_admission_facts(
14914 corr,
14915 &target_module_id,
14916 unique_project_root("admission-facts"),
14917 consumer_identity,
14918 None,
14919 ),
14920 )
14921 .await
14922 .expect("capability test route.open succeeds")
14923 });
14924 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
14925 .await
14926 .expect("capability test route.open must reach route.bind")
14927 .expect("target control receiver stays open");
14928 let bind_request: ModuleControlRequest =
14929 serde_json::from_slice(&bind.body).expect("route.bind decodes");
14930 handler
14931 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
14932 .await
14933 .expect("capability test route.bind ACK succeeds");
14934 assert!(route_task.await.expect("route.open task joins").is_empty());
14935 let opened = client_rx
14936 .recv()
14937 .await
14938 .expect("successful route.open publishes a response");
14939 assert!(matches!(
14940 serde_json::from_slice::<ClientControlResponse>(&opened.body),
14941 Ok(ClientControlResponse::RouteOpen { .. })
14942 ));
14943 (client_rx, bind_request)
14944 }
14945
14946 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
14947 assert_eq!(frame.header.ty, FrameType::Push);
14948 assert_eq!(frame.header.channel, 0);
14949 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
14950 .expect("route.closed control push decodes");
14951 let ClientControlPush::RouteClosed { channels, .. } = &push else {
14952 panic!("expected route.closed");
14953 };
14954 assert_eq!(channels.len(), 1, "exactly one violating route closed");
14955 let channels = channels.clone();
14956 assert_eq!(
14957 push,
14958 ClientControlPush::RouteClosed {
14959 module_id: target_module_id.to_string(),
14960 channels,
14961 reason: RouteCloseReason::CapabilityDenied,
14962 drained: false,
14963 abandoned: 0,
14964 excluded_subscriptions: 0,
14965 terminal: Some(false),
14966 }
14967 );
14968 }
14969
14970 #[tokio::test]
14971 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
14972 let registry = Arc::new(Registry::default());
14973 let forwarding = Arc::new(ForwardingTable::default());
14974 let supervisor = SupervisorHandle::new();
14975 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14976 let handler =
14977 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14978 .with_supervisor(supervisor);
14979 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
14980 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
14981 register_capability_manifest(
14982 &handler,
14983 &target_ctx,
14984 &mut target_rx,
14985 capability_manifest("target", &["credentials-provider/v1"], &[]),
14986 1,
14987 )
14988 .await;
14989 register_capability_manifest(
14990 &handler,
14991 &opener_ctx,
14992 &mut opener_rx,
14993 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14994 2,
14995 )
14996 .await;
14997
14998 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
14999 let replies = handler
15000 .handle_control_frame(
15001 &client_ctx,
15002 route_open_frame_with_admission_facts(
15003 3,
15004 "target",
15005 unique_project_root("admission-facts"),
15006 Some(ConsumerIdentity {
15007 module_id: "opener".to_string(),
15008 launch_nonce: "opener-nonce".to_string(),
15009 }),
15010 None,
15011 ),
15012 )
15013 .await
15014 .expect("denied route.open returns a typed frame");
15015 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15016 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15017 assert!(
15018 target_rx.try_recv().is_err(),
15019 "forbidden route.open must not relay route.bind"
15020 );
15021 }
15022
15023 #[tokio::test]
15024 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
15025 let registry = Arc::new(Registry::default());
15026 let forwarding = Arc::new(ForwardingTable::default());
15027 let supervisor = SupervisorHandle::new();
15028 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15029 let handler =
15030 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15031 .with_supervisor(supervisor);
15032 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
15033 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
15034 register_capability_manifest(
15035 &handler,
15036 &target_ctx,
15037 &mut target_rx,
15038 capability_manifest("target", &["credentials-provider/v1"], &[]),
15039 1,
15040 )
15041 .await;
15042 register_capability_manifest(
15043 &handler,
15044 &old_opener_ctx,
15045 &mut old_opener_rx,
15046 capability_manifest("opener", &[], &[]),
15047 2,
15048 )
15049 .await;
15050 let (mut client_rx, _) = open_route_for_capability_test(
15051 &handler,
15052 &target_ctx,
15053 &mut target_rx,
15054 712,
15055 3,
15056 "target",
15057 Some(ConsumerIdentity {
15058 module_id: "opener".to_string(),
15059 launch_nonce: "opener-nonce".to_string(),
15060 }),
15061 )
15062 .await;
15063 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15064
15065 handler
15066 .cleanup_connection(old_opener_ctx.connection_id)
15067 .expect("old opener registration cleans up");
15068 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
15069 register_capability_manifest(
15070 &handler,
15071 &new_opener_ctx,
15072 &mut new_opener_rx,
15073 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15074 4,
15075 )
15076 .await;
15077
15078 assert_capability_denied_push(
15079 client_rx
15080 .try_recv()
15081 .expect("HELLO deny addition must emit route.closed")
15082 .frame,
15083 "target",
15084 );
15085 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15086 assert!(matches!(
15087 target_rx.try_recv(),
15088 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15089 ));
15090 }
15091
15092 #[tokio::test]
15093 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
15094 let registry = Arc::new(Registry::default());
15095 let forwarding = Arc::new(ForwardingTable::default());
15096 let supervisor = SupervisorHandle::new();
15097 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15098 let handler =
15099 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15100 .with_supervisor(supervisor);
15101 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
15102 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
15103 register_capability_manifest(
15104 &handler,
15105 &target_ctx,
15106 &mut target_rx,
15107 capability_manifest("target", &[], &[]),
15108 1,
15109 )
15110 .await;
15111 register_capability_manifest(
15112 &handler,
15113 &opener_ctx,
15114 &mut opener_rx,
15115 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15116 2,
15117 )
15118 .await;
15119 let (mut client_rx, _) = open_route_for_capability_test(
15120 &handler,
15121 &target_ctx,
15122 &mut target_rx,
15123 722,
15124 3,
15125 "target",
15126 Some(ConsumerIdentity {
15127 module_id: "opener".to_string(),
15128 launch_nonce: "opener-nonce".to_string(),
15129 }),
15130 )
15131 .await;
15132 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15133
15134 let replies = handler
15135 .handle_control_frame(
15136 &target_ctx,
15137 catalog_update_with_capabilities_frame(
15138 4,
15139 CapabilityDeclarations {
15140 provides: vec!["credentials-provider/v1".to_string()],
15141 requires: Vec::new(),
15142 must_never_reach: Vec::new(),
15143 },
15144 ),
15145 )
15146 .await
15147 .expect("claim catalog.update succeeds");
15148 assert!(matches!(
15149 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
15150 Ok(ModuleControlResponseToModule::CatalogUpdate {})
15151 ));
15152 assert_capability_denied_push(
15153 client_rx
15154 .try_recv()
15155 .expect("claim addition must emit route.closed")
15156 .frame,
15157 "target",
15158 );
15159 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15160 assert!(matches!(
15161 target_rx.try_recv(),
15162 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15163 ));
15164 }
15165
15166 #[tokio::test]
15167 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
15168 let registry = Arc::new(Registry::default());
15169 let forwarding = Arc::new(ForwardingTable::default());
15170 let supervisor = SupervisorHandle::new();
15171 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15172 let handler =
15173 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15174 .with_supervisor(supervisor);
15175 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
15176 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
15177 register_capability_manifest(
15178 &handler,
15179 &target_ctx,
15180 &mut target_rx,
15181 capability_manifest("target", &["credentials-provider/v1"], &[]),
15182 1,
15183 )
15184 .await;
15185 register_capability_manifest(
15186 &handler,
15187 &opener_ctx,
15188 &mut opener_rx,
15189 capability_manifest("opener", &[], &[]),
15190 2,
15191 )
15192 .await;
15193 let (mut client_rx, _) = open_route_for_capability_test(
15194 &handler,
15195 &target_ctx,
15196 &mut target_rx,
15197 732,
15198 3,
15199 "target",
15200 Some(ConsumerIdentity {
15201 module_id: "opener".to_string(),
15202 launch_nonce: "opener-nonce".to_string(),
15203 }),
15204 )
15205 .await;
15206 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15207
15208 handler
15209 .handle_control_frame(
15210 &target_ctx,
15211 catalog_update_with_capabilities_frame(
15212 4,
15213 CapabilityDeclarations {
15214 provides: Vec::new(),
15215 requires: Vec::new(),
15216 must_never_reach: Vec::new(),
15217 },
15218 ),
15219 )
15220 .await
15221 .expect("claim removal catalog.update succeeds");
15222 assert_eq!(
15223 forwarding.active_binding_count().unwrap(),
15224 1,
15225 "removing an attested target claim must leave the route census unchanged"
15226 );
15227 assert!(
15228 client_rx.try_recv().is_err(),
15229 "claim removal must not emit route.closed capability_denied"
15230 );
15231 assert!(
15232 target_rx.try_recv().is_err(),
15233 "claim removal must not send the target a route GOODBYE"
15234 );
15235 }
15236
15237 #[tokio::test]
15240 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
15241 let registry = Arc::new(Registry::default());
15242 let forwarding = Arc::new(ForwardingTable::default());
15243 let supervisor = SupervisorHandle::new();
15244 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15245 let handler =
15246 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15247 .with_supervisor(supervisor);
15248 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
15249 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
15250 register_capability_manifest(
15251 &handler,
15252 &target_ctx,
15253 &mut target_rx,
15254 capability_manifest("target", &["credentials-provider/v1"], &[]),
15255 1,
15256 )
15257 .await;
15258 register_capability_manifest(
15259 &handler,
15260 &opener_ctx,
15261 &mut opener_rx,
15262 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15263 2,
15264 )
15265 .await;
15266
15267 let (_client_rx, bind) = open_route_for_capability_test(
15268 &handler,
15269 &target_ctx,
15270 &mut target_rx,
15271 742,
15272 3,
15273 "target",
15274 None,
15275 )
15276 .await;
15277 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
15278 panic!("direct scope-honesty route must bind");
15279 };
15280 assert_eq!(principal, Some(Principal::Direct));
15281 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15282 }
15283
15284 #[tokio::test]
15287 async fn must_never_reach_self_route_is_capability_forbidden() {
15288 let registry = Arc::new(Registry::default());
15289 let forwarding = Arc::new(ForwardingTable::default());
15290 let supervisor = SupervisorHandle::new();
15291 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
15292 let handler =
15293 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15294 .with_supervisor(supervisor);
15295 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
15296 register_capability_manifest(
15297 &handler,
15298 &self_ctx,
15299 &mut self_rx,
15300 capability_manifest(
15301 "self-provider",
15302 &["credentials-provider/v1"],
15303 &["credentials-provider/v1"],
15304 ),
15305 1,
15306 )
15307 .await;
15308
15309 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
15310 let replies = handler
15311 .handle_control_frame(
15312 &client_ctx,
15313 route_open_frame_with_admission_facts(
15314 2,
15315 "self-provider",
15316 unique_project_root("admission-facts"),
15317 Some(ConsumerIdentity {
15318 module_id: "self-provider".to_string(),
15319 launch_nonce: "self-nonce".to_string(),
15320 }),
15321 None,
15322 ),
15323 )
15324 .await
15325 .expect("self-route refusal returns a typed frame");
15326 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15327 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15328 assert!(
15329 self_rx.try_recv().is_err(),
15330 "self denial must not relay route.bind"
15331 );
15332 }
15333
15334 #[test]
15335 fn unsupported_channel_zero_frame_returns_error() {
15336 let handler = ControlHandler::default();
15337 let request = Frame::build(
15338 FrameType::Request,
15339 control_flags(),
15340 0,
15341 0,
15342 21,
15343 b"opaque".to_vec(),
15344 )
15345 .unwrap();
15346
15347 let response = handler
15348 .handle_control(ConnectionId::new(1), request)
15349 .unwrap();
15350
15351 assert_eq!(response[0].header.ty, FrameType::Error);
15352 assert_eq!(
15353 parse_error(&response[0])["code"],
15354 "unsupported_control_frame"
15355 );
15356 }
15357
15358 mod swap {
15363 use super::*;
15364
15365 const INCUMBENT: ConnectionId = ConnectionId::new(30);
15366 const CANDIDATE: ConnectionId = ConnectionId::new(40);
15367
15368 struct Swap {
15369 registry: Arc<Registry>,
15370 forwarding: Arc<ForwardingTable>,
15371 handler: ControlHandler,
15372 incumbent_ctx: RouteCtx,
15373 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15374 candidate_ctx: RouteCtx,
15375 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15376 }
15377
15378 async fn swap_with_incumbent() -> Swap {
15379 let registry = Arc::new(Registry::default());
15380 let forwarding = Arc::new(ForwardingTable::default());
15381 let handler =
15382 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15383 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
15384 hello_via_sink(
15385 &handler,
15386 &incumbent_ctx,
15387 &mut incumbent_rx,
15388 hello_frame("aft", PROTOCOL_VERSION, 7),
15389 )
15390 .await;
15391 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
15392 Swap {
15393 registry,
15394 forwarding,
15395 handler,
15396 incumbent_ctx,
15397 incumbent_rx,
15398 candidate_ctx,
15399 candidate_rx,
15400 }
15401 }
15402
15403 fn register_candidate(swap: &Swap, ready: Option<bool>) {
15404 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
15405 candidate_manifest.ready = ready;
15406 let registration = swap
15407 .registry
15408 .register_candidate_with_control_ops(
15409 candidate_manifest,
15410 PROTOCOL_VERSION,
15411 CANDIDATE,
15412 module_baseline_control_ops(),
15413 )
15414 .unwrap();
15415 swap.forwarding
15416 .register_candidate_module_connection(
15417 CANDIDATE,
15418 "aft".to_string(),
15419 PROTOCOL_VERSION,
15420 manifest_concurrency(®istration.manifest),
15421 swap.candidate_ctx.egress.clone(),
15422 )
15423 .unwrap();
15424 }
15425
15426 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
15427 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
15428 swap.registry.promote_candidate("aft").unwrap().unwrap();
15429 cutover.incumbent.unwrap()
15430 }
15431
15432 #[tokio::test]
15441 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
15442 let mut swap = swap_with_incumbent().await;
15443 let hold = Arc::new(std::sync::Barrier::new(2));
15444 let handler = swap
15445 .handler
15446 .clone()
15447 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
15448 handler
15449 .supervisor
15450 .open_swap("aft", "candidate-nonce".to_string());
15451 let hello = {
15452 let handler = handler.clone();
15453 let candidate_ctx = swap.candidate_ctx.clone();
15454 std::thread::spawn(move || {
15455 handler
15456 .handle_hello(
15457 candidate_ctx.connection_id,
15458 Some(candidate_ctx.egress.clone()),
15459 hello_frame_with_nonce(
15460 "aft",
15461 PROTOCOL_VERSION,
15462 8,
15463 Some("candidate-nonce"),
15464 ),
15465 )
15466 .unwrap()
15467 })
15468 };
15469
15470 hold.wait();
15471 assert!(swap
15474 .registry
15475 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
15476 .unwrap()
15477 .is_some());
15478 assert!(swap
15479 .registry
15480 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15481 .unwrap()
15482 .is_none());
15483 assert!(swap
15484 .forwarding
15485 .module_endpoint_for_connection(CANDIDATE)
15486 .unwrap()
15487 .is_none());
15488
15489 hold.wait();
15490 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
15491 assert_eq!(
15492 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
15493 FrameType::HelloAck
15494 );
15495 let published = swap
15496 .registry
15497 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15498 .unwrap()
15499 .expect("candidate is published once its HELLO completes");
15500 assert_eq!(published.connection_id, CANDIDATE);
15501 assert!(swap
15502 .forwarding
15503 .module_endpoint_for_connection(CANDIDATE)
15504 .unwrap()
15505 .is_some());
15506 }
15507
15508 fn keyed_total(counters: &Value, key: &str) -> u64 {
15509 counters[key]
15510 .as_object()
15511 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
15512 .unwrap_or(0)
15513 }
15514
15515 #[tokio::test]
15516 async fn replacement_logs_old_end_and_new_admission_once() {
15517 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15518 let mut swap = swap_with_incumbent().await;
15519 swap.handler
15520 .supervisor
15521 .open_swap("aft", "candidate-nonce".to_string());
15522 hello_via_sink(
15523 &swap.handler,
15524 &swap.candidate_ctx,
15525 &mut swap.candidate_rx,
15526 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
15527 )
15528 .await;
15529 cutover(&swap);
15530 swap.handler.cleanup_connection(INCUMBENT).unwrap();
15531
15532 let captured = crate::router::test_log::captured_logs(&logs);
15533 println!("captured replacement registry logs:\n{captured}");
15534 let new_admissions: Vec<_> = captured
15535 .lines()
15536 .filter(|line| {
15537 line.contains("connection_id=40")
15538 && line.contains("swap candidate registered; not routable until cutover")
15539 })
15540 .collect();
15541 assert_eq!(
15542 new_admissions.len(),
15543 1,
15544 "unexpected admission logs: {captured}"
15545 );
15546 assert!(new_admissions[0].contains("module_id=aft"));
15547 assert!(new_admissions[0].contains("connection_id=40"));
15548 assert!(
15549 new_admissions[0].contains("swap candidate registered; not routable until cutover")
15550 );
15551 assert!(new_admissions[0].contains("ready=true"));
15552
15553 let old_admissions: Vec<_> = captured
15554 .lines()
15555 .filter(|line| {
15556 line.contains("module registered module_id=aft ")
15557 && line.contains("connection_id=30")
15558 })
15559 .collect();
15560 assert_eq!(
15561 old_admissions.len(),
15562 1,
15563 "unexpected incumbent admission: {captured}"
15564 );
15565
15566 let old_ends: Vec<_> = captured
15567 .lines()
15568 .filter(|line| {
15569 line.contains("connection_id=30") && line.contains("module registration ended")
15570 })
15571 .collect();
15572 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
15573 assert!(old_ends[0].contains("module_id=aft"));
15574 assert!(old_ends[0].contains("connection_id=30"));
15575 assert!(old_ends[0].contains("reason=replaced"));
15576 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
15577 assert_eq!(
15578 captured
15579 .lines()
15580 .filter(|line| line.contains("module registration promoted"))
15581 .count(),
15582 1,
15583 "unexpected promotion logs: {captured}"
15584 );
15585 }
15586
15587 #[tokio::test]
15593 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
15594 let mut swap = swap_with_incumbent().await;
15595 let handler = swap.handler.clone();
15596
15597 let cotenant = ConnectionId::new(31);
15599 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
15600 let (cotenant_task, cotenant_bind) = relay_route_open(
15601 &handler,
15602 cotenant,
15603 &cotenant_ctx.egress,
15604 &mut swap.incumbent_rx,
15605 100,
15606 "aft",
15607 "swap-cotenant",
15608 )
15609 .await;
15610 handler
15611 .handle_control_frame(
15612 &swap.incumbent_ctx,
15613 route_bind_ack(cotenant_bind.header.corr),
15614 )
15615 .await
15616 .unwrap();
15617 assert!(cotenant_task.await.unwrap().is_empty());
15618 let (cotenant_channel, cotenant_epoch) =
15619 published_route(&cotenant_rx.recv().await.unwrap());
15620
15621 let caller = ConnectionId::new(32);
15623 let (caller_ctx, mut caller_rx) = route_ctx(caller);
15624 let (caller_task, caller_bind) = relay_route_open(
15625 &handler,
15626 caller,
15627 &caller_ctx.egress,
15628 &mut swap.incumbent_rx,
15629 101,
15630 "aft",
15631 "swap-caller",
15632 )
15633 .await;
15634 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
15635
15636 register_candidate(&swap, None);
15637 cutover(&swap);
15638
15639 let ack = handler
15641 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
15642 .await;
15643 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
15644 if module_loop_error.is_some() {
15645 handler.cleanup_connection(INCUMBENT).unwrap();
15648 }
15649
15650 assert!(
15652 cotenant_rx.try_recv().is_err(),
15653 "the co-tenant route on the incumbent was torn down by one late ack: \
15654 {module_loop_error:?}"
15655 );
15656 assert!(matches!(
15657 swap.forwarding
15658 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
15659 .unwrap(),
15660 DataRoute::Client(DataRouteState::Bound(_))
15661 ));
15662 assert_eq!(module_loop_error, None);
15663 assert!(swap
15664 .registry
15665 .get_module_by_connection(INCUMBENT)
15666 .unwrap()
15667 .is_some());
15668
15669 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
15671 .await
15672 .expect("the incumbent is told to drop the abandoned binding")
15673 .unwrap()
15674 .frame;
15675 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15676 assert_eq!(goodbye.header.channel, abandoned_channel);
15677 assert_eq!(goodbye.header.epoch, abandoned_epoch);
15678 assert!(swap.incumbent_rx.try_recv().is_err());
15679
15680 let response = caller_task.await.unwrap();
15682 assert_eq!(response.len(), 1);
15683 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15684 assert!(caller_rx.try_recv().is_err());
15685
15686 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
15690 let counters = handler.counters().snapshot();
15691 assert_eq!(
15692 keyed_total(&counters, "route_open_accepted_by_principal"),
15693 1
15694 );
15695 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
15696 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
15697 }
15698
15699 #[tokio::test]
15703 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
15704 let mut swap = swap_with_incumbent().await;
15705 register_candidate(&swap, None);
15706 let incumbent = cutover(&swap);
15707 swap.forwarding
15708 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
15709 .unwrap()
15710 .expect("the incumbent is still registered");
15711
15712 let client = ConnectionId::new(33);
15713 let (client_ctx, mut client_rx) = route_ctx(client);
15714 let route_handler = swap.handler.clone();
15715 let open_ctx = RouteCtx {
15716 connection_id: client,
15717 egress: client_ctx.egress.clone(),
15718 };
15719 let mut route_task = tokio::spawn(async move {
15720 route_handler
15721 .handle_control_frame(
15722 &open_ctx,
15723 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
15724 )
15725 .await
15726 .unwrap()
15727 });
15728 let bind = tokio::select! {
15729 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
15730 response = &mut route_task => {
15731 let response = response.unwrap();
15732 panic!(
15733 "post-cutover route.open was refused instead of relayed to the candidate: {}",
15734 parse_error(&response[0])["code"]
15735 );
15736 }
15737 };
15738 swap.handler
15739 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
15740 .await
15741 .unwrap();
15742 assert!(route_task.await.unwrap().is_empty());
15743 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
15744 match swap
15745 .forwarding
15746 .lookup_data_route(client, channel, epoch)
15747 .unwrap()
15748 {
15749 DataRoute::Client(DataRouteState::Bound(route)) => {
15750 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
15751 }
15752 other => panic!("expected a bound route on the candidate, got {other:?}"),
15753 }
15754 assert!(swap.incumbent_rx.try_recv().is_err());
15755 }
15756
15757 #[tokio::test]
15762 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
15763 let swap = swap_with_incumbent().await;
15764 register_candidate(&swap, Some(false));
15765 let update = Frame::build(
15766 FrameType::Request,
15767 control_flags(),
15768 0,
15769 0,
15770 55,
15771 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
15772 provides: manifest("aft", PROTOCOL_VERSION).provides,
15773 capabilities: None,
15774 ready: Some(true),
15775 })
15776 .unwrap(),
15777 )
15778 .unwrap();
15779
15780 let replies = swap
15781 .handler
15782 .handle_control_frame(&swap.candidate_ctx, update)
15783 .await
15784 .unwrap();
15785
15786 assert_eq!(replies.len(), 1);
15787 assert_eq!(
15788 replies[0].header.ty,
15789 FrameType::Response,
15790 "candidate catalog.update was refused: {:?}",
15791 serde_json::from_slice::<Value>(&replies[0].body).ok()
15792 );
15793 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
15794 assert_eq!(
15795 swap.registry
15796 .get_module("aft")
15797 .unwrap()
15798 .unwrap()
15799 .connection_id,
15800 INCUMBENT
15801 );
15802 }
15803 }
15804
15805 mod swap_admission {
15809 use super::*;
15810
15811 const INCUMBENT_NONCE: &str = "incumbent-nonce";
15812 const CANDIDATE_NONCE: &str = "candidate-nonce";
15813
15814 fn handler_with_incumbent(
15815 module_id: &str,
15816 reserved: bool,
15817 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
15818 let registry = Arc::new(Registry::default());
15819 let supervisor = SupervisorHandle::new();
15820 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
15821 if reserved {
15822 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
15823 }
15824 let handler =
15825 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
15826 let incumbent = handler
15827 .handle_control(
15828 ConnectionId::new(1),
15829 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
15830 )
15831 .unwrap();
15832 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
15833 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
15834 (registry, supervisor, handler)
15835 }
15836
15837 #[test]
15843 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
15844 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
15845
15846 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
15847 let replies = handler
15848 .handle_control(
15849 ConnectionId::new(connection),
15850 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
15851 )
15852 .unwrap();
15853 assert_eq!(replies[0].header.ty, FrameType::Error);
15854 assert_eq!(
15855 parse_error(&replies[0])["code"],
15856 "swap_token_invalid",
15857 "nonce {nonce:?}"
15858 );
15859 }
15860 assert!(registry.get_candidate("aft").unwrap().is_none());
15861 assert_eq!(
15862 registry.get_module("aft").unwrap().unwrap().connection_id,
15863 ConnectionId::new(1)
15864 );
15865
15866 let admitted = handler
15869 .handle_control(
15870 ConnectionId::new(4),
15871 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
15872 )
15873 .unwrap();
15874 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
15875 assert_eq!(
15876 registry
15877 .get_candidate("aft")
15878 .unwrap()
15879 .unwrap()
15880 .connection_id,
15881 ConnectionId::new(4)
15882 );
15883 assert_eq!(
15884 registry.get_module("aft").unwrap().unwrap().connection_id,
15885 ConnectionId::new(1),
15886 "the candidate must not take the active slot"
15887 );
15888 let replayed = handler
15889 .handle_control(
15890 ConnectionId::new(5),
15891 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
15892 )
15893 .unwrap();
15894 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
15895
15896 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
15900 let squatter = handler
15901 .handle_control(
15902 ConnectionId::new(6),
15903 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
15904 )
15905 .unwrap();
15906 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
15907 assert!(
15908 registry.get_module("aft").unwrap().is_none(),
15909 "a squatter took the active slot of an id being swapped"
15910 );
15911 }
15912
15913 #[test]
15918 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
15919 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
15920
15921 let replies = handler
15922 .handle_control(
15923 ConnectionId::new(2),
15924 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15925 )
15926 .unwrap();
15927
15928 assert_eq!(
15929 replies[0].header.ty,
15930 FrameType::HelloAck,
15931 "reserved candidate refused: {:?}",
15932 serde_json::from_slice::<Value>(&replies[0].body).ok()
15933 );
15934 assert_eq!(
15935 registry
15936 .get_candidate("vault")
15937 .unwrap()
15938 .unwrap()
15939 .connection_id,
15940 ConnectionId::new(2)
15941 );
15942 }
15943
15944 #[test]
15947 fn without_an_open_swap_the_ordinary_gates_decide() {
15948 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
15949 supervisor.close_swap("vault");
15950
15951 let candidate = handler
15952 .handle_control(
15953 ConnectionId::new(2),
15954 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15955 )
15956 .unwrap();
15957 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
15958 let duplicate = handler
15959 .handle_control(
15960 ConnectionId::new(3),
15961 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
15962 )
15963 .unwrap();
15964 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15965 assert!(registry.get_candidate("vault").unwrap().is_none());
15966 }
15967 }
15968
15969 mod scopes {
15973 mod idle_tests {
15974 use super::*;
15975 include!("scope_idle_tests.rs");
15976 }
15977 mod route_open_timings {
15978 use super::*;
15979 include!("route_open_timing_tests.rs");
15980 }
15981 mod route_open_refusal_log {
15982 use super::*;
15983 include!("route_open_refusal_log_tests.rs");
15984 }
15985 mod activity {
15986 include!("activity_scope_tests.rs");
15987 }
15988 mod delta {
15989 include!("scope_handler_delta_tests.rs");
15990 }
15991 mod call_modes {
15992 include!("scope_call_mode_tests.rs");
15993 }
15994 #[tokio::test]
15995 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
15996 let rig = rig().await;
15997 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
15998 "s".into(),
15999 1,
16000 "tool".into(),
16001 ))
16002 .unwrap();
16003 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
16004 let (direct, _rx) = wide_ctx(999);
16005 for (ctx, expected) in [
16006 (&rig.owner, FrameType::Response),
16007 (&direct, FrameType::Error),
16008 ] {
16009 let frame =
16010 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
16011 .unwrap();
16012 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
16013 assert_eq!(replies[0].header.ty, expected);
16014 if expected == FrameType::Response {
16015 let reply: subc_control::ScopeServedByReply =
16016 serde_json::from_slice(&replies[0].body).unwrap();
16017 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
16018 } else {
16019 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
16020 }
16021 }
16022 let reply = call(
16023 &rig.handler,
16024 &rig.owner,
16025 &ModuleControlRequestFromModule::ScopeDescribe {
16026 owner: Principal::Reserved {
16027 module_id: OWNER.into(),
16028 },
16029 scope_ref: "s".into(),
16030 },
16031 )
16032 .await;
16033 assert!(matches!(
16034 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
16035 ModuleControlResponseToModule::ScopeDescribe { .. }
16036 ));
16037 }
16038
16039 use subc_protocol::scope::{
16040 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
16041 ScopeStatus,
16042 };
16043
16044 use super::*;
16045
16046 const OWNER: &str = "prefrontal-core";
16047
16048 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
16049 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
16050 }
16051
16052 async fn call(
16053 handler: &ControlHandler,
16054 ctx: &RouteCtx,
16055 request: &ModuleControlRequestFromModule,
16056 ) -> Frame {
16057 let body = serde_json::to_vec(request).unwrap();
16058 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
16059 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
16060 assert_eq!(replies.len(), 1, "{replies:?}");
16061 replies.pop().unwrap()
16062 }
16063
16064 async fn sync(
16065 handler: &ControlHandler,
16066 ctx: &RouteCtx,
16067 generation: u64,
16068 scopes: Vec<ScopeRecord>,
16069 ) -> Result<ModuleControlResponseToModule, String> {
16070 let reply = call(
16071 handler,
16072 ctx,
16073 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
16074 )
16075 .await;
16076 match reply.header.ty {
16077 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
16078 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
16079 }
16080 }
16081
16082 async fn describe(
16083 handler: &ControlHandler,
16084 ctx: &RouteCtx,
16085 owner: &str,
16086 scope_ref: &str,
16087 ) -> ModuleControlResponseToModule {
16088 let reply = call(
16089 handler,
16090 ctx,
16091 &ModuleControlRequestFromModule::ScopeDescribe {
16092 owner: Principal::Reserved {
16093 module_id: owner.to_string(),
16094 },
16095 scope_ref: scope_ref.to_string(),
16096 },
16097 )
16098 .await;
16099 assert_eq!(
16100 reply.header.ty,
16101 FrameType::Response,
16102 "{:?}",
16103 parse_error(&reply)
16104 );
16105 serde_json::from_slice(&reply.body).unwrap()
16106 }
16107
16108 #[tokio::test]
16109 async fn scope_describe_never_echoes_bind_proof_or_hash() {
16110 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16111 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16112 let mut rig = rig().await;
16113 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16114 .await;
16115 let _bound = rig
16116 .bound(
16117 Some(AFT),
16118 PLEXUS,
16119 Some(
16120 rig.selector("s", Some(1))
16121 .with_bind_proof(Some(PROOF.into())),
16122 ),
16123 )
16124 .await;
16125 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
16126 let output = serde_json::to_string(&reply).unwrap();
16127 assert!(output.contains("\"status\":\"live\""));
16128 assert!(!output.contains(PROOF));
16129 assert!(!output.contains(HASH));
16130 }
16131
16132 async fn module(
16134 handler: &ControlHandler,
16135 connection: u64,
16136 module_id: &str,
16137 nonce: Option<&str>,
16138 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16139 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
16140 hello_via_sink(
16141 handler,
16142 &ctx,
16143 &mut rx,
16144 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
16145 )
16146 .await;
16147 (ctx, rx)
16148 }
16149
16150 #[tokio::test]
16153 async fn a_client_connection_cannot_sync_or_describe() {
16154 let handler = ControlHandler::new(Arc::new(Registry::default()));
16155 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
16156 for request in [
16157 ModuleControlRequestFromModule::ScopeSync {
16158 generation: 1,
16159 scopes: vec![head("s", 1)],
16160 },
16161 ModuleControlRequestFromModule::ScopeDescribe {
16162 owner: Principal::Direct,
16163 scope_ref: "s".to_string(),
16164 },
16165 ] {
16166 let reply = call(&handler, &ctx, &request).await;
16167 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
16168 }
16169 assert!(
16170 !handler
16171 .scopes
16172 .read()
16173 .unwrap()
16174 .describe(
16175 &Principal::Reserved {
16176 module_id: OWNER.to_string()
16177 },
16178 "s"
16179 )
16180 .owner_synced
16181 );
16182 }
16183
16184 #[tokio::test]
16187 async fn a_module_without_a_supervised_launch_cannot_sync() {
16188 let handler = ControlHandler::new(Arc::new(Registry::default()));
16189 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
16190 assert_eq!(
16191 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
16192 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16193 );
16194 }
16195
16196 #[tokio::test]
16197 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
16198 let supervisor = SupervisorHandle::new();
16199 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16200 let handler = ControlHandler::new(Arc::new(Registry::default()))
16201 .with_supervisor(supervisor.clone());
16202 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16203 sync(&handler, &incumbent, 1, vec![head("s", 1)])
16204 .await
16205 .expect("the current launch syncs");
16206
16207 supervisor.open_swap(OWNER, "n2".to_string());
16210 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
16211 assert_eq!(
16212 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16213 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16214 );
16215 sync(&handler, &incumbent, 2, vec![head("s", 1)])
16216 .await
16217 .expect("the serving owner syncs during the swap");
16218
16219 supervisor.close_swap(OWNER);
16222 assert_eq!(
16223 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16224 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16225 );
16226 sync(&handler, &incumbent, 3, vec![head("s", 1)])
16227 .await
16228 .expect("the serving owner syncs after the rollback");
16229 handler.cleanup_connection(candidate.connection_id).unwrap();
16230
16231 supervisor.open_swap(OWNER, "n3".to_string());
16236 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
16237 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
16238 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
16239 .await
16240 .expect("the promoted launch takes authority");
16241 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16242 panic!("unexpected reply {reply:?}");
16243 };
16244 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
16245 assert_eq!(
16246 sync(&handler, &incumbent, 4, Vec::new()).await,
16247 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16248 );
16249 }
16250
16251 #[tokio::test]
16254 async fn closing_the_authority_connection_frees_sync_authority() {
16255 let supervisor = SupervisorHandle::new();
16256 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16257 let handler = ControlHandler::new(Arc::new(Registry::default()))
16258 .with_supervisor(supervisor.clone());
16259 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16260 sync(&handler, &first, 10, vec![head("s", 1)])
16261 .await
16262 .unwrap();
16263 handler.cleanup_connection(first.connection_id).unwrap();
16264
16265 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
16266 sync(&handler, &second, 1, vec![head("s", 1)])
16267 .await
16268 .expect("the next connection takes the released authority");
16269 }
16270
16271 #[tokio::test]
16272 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
16273 let supervisor = SupervisorHandle::new();
16274 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16275 let handler =
16276 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
16277 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
16278 sync(&handler, &first, 10, vec![head("s", 1)])
16279 .await
16280 .unwrap();
16281 handler
16282 .handle_control_frame(
16283 &first,
16284 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
16285 )
16286 .await
16287 .unwrap();
16288 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
16289 sync(&handler, &second, 1, vec![head("s", 1)])
16290 .await
16291 .expect("GOODBYE releases authority even if the old socket remains open");
16292 }
16293
16294 #[tokio::test]
16295 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
16296 let registry = Arc::new(Registry::default());
16297 let supervisor_handle = SupervisorHandle::new();
16298 let supervisor =
16299 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
16300 .with_handle(supervisor_handle.clone())
16301 .with_daemon_incarnation("incarnation-7".to_string());
16302 supervisor
16305 .supervise_configured(
16306 ModuleSpec {
16307 module_id: OWNER.to_string(),
16308 program: PathBuf::from("/nonexistent/prefrontal-core"),
16309 args: Vec::new(),
16310 env: Vec::new(),
16311 reserved: false,
16312 reserved_prefixes: Vec::new(),
16313 protocol: ModuleProtocol::Subc,
16314 overlap: Default::default(),
16315 },
16316 false,
16317 )
16318 .unwrap();
16319 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
16320 let handler =
16321 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
16322 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
16323
16324 let ModuleControlResponseToModule::ScopeDescribe {
16326 status,
16327 daemon_incarnation,
16328 owner_synced,
16329 owner_configured,
16330 scope,
16331 ..
16332 } = describe(&handler, &reader, OWNER, "s").await
16333 else {
16334 panic!("not a describe reply");
16335 };
16336 assert_eq!(status, ScopeStatus::NotLive);
16337 assert_eq!(daemon_incarnation, "incarnation-7");
16338 assert!(!owner_synced);
16339 assert!(owner_configured);
16340 assert!(scope.is_none());
16341
16342 let ModuleControlResponseToModule::ScopeDescribe {
16345 status,
16346 owner_configured,
16347 ..
16348 } = describe(&handler, &reader, "ghost", "s").await
16349 else {
16350 panic!("not a describe reply");
16351 };
16352 assert_eq!(status, ScopeStatus::NotLive);
16353 assert!(!owner_configured);
16354
16355 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
16357 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
16358 let ModuleControlResponseToModule::ScopeDescribe {
16359 status,
16360 scope_epoch,
16361 owner_synced,
16362 scope,
16363 ..
16364 } = describe(&handler, &reader, OWNER, "s").await
16365 else {
16366 panic!("not a describe reply");
16367 };
16368 assert_eq!(status, ScopeStatus::Live);
16369 assert_eq!(scope_epoch, Some(4));
16370 assert!(owner_synced);
16371 let stamp = scope.expect("a live scope carries its stamp");
16372 assert!(
16373 stamp.owner_authorized,
16374 "prefrontal-core is the default authority"
16375 );
16376 assert_eq!(stamp.kind, ScopeKind::Head);
16377 }
16378
16379 #[tokio::test]
16380 async fn scope_authority_owners_decides_owner_authorized() {
16381 let supervisor = SupervisorHandle::new();
16382 supervisor.set_spawn_nonce("broca", "b1".to_string());
16383 let handler = ControlHandler::new(Arc::new(Registry::default()))
16384 .with_supervisor(supervisor)
16385 .with_scope_authority_owners(vec!["broca".to_string()]);
16386 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
16387 let mut gated = head("s", 1);
16388 gated.attributes.agent_id = Some("agent".to_string());
16389 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
16390 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
16391 describe(&handler, &broca, "broca", "s").await
16392 else {
16393 panic!("not a describe reply");
16394 };
16395 assert!(scope.unwrap().owner_authorized);
16396 }
16397
16398 #[tokio::test]
16402 async fn scope_ops_and_the_scopes_capability_are_advertised() {
16403 let handler = ControlHandler::new(Arc::new(Registry::default()));
16404 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
16405 let ack = hello_via_sink(
16406 &handler,
16407 &ctx,
16408 &mut rx,
16409 hello_frame("m", PROTOCOL_VERSION, 1),
16410 )
16411 .await;
16412 let ack = parse_ack(&ack);
16413 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
16414 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
16415 }
16416 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
16417
16418 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
16419 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16420 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16421 let reply = handler
16422 .handle_control_frame(&client, frame)
16423 .await
16424 .unwrap()
16425 .pop()
16426 .unwrap();
16427 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16428 serde_json::from_slice(&reply.body).unwrap()
16429 else {
16430 panic!("not a server.describe reply");
16431 };
16432 assert!(
16433 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
16434 "{capabilities:?}"
16435 );
16436 }
16437
16438 const PLEXUS: &str = "plexus";
16441 const OTHER: &str = "other";
16442 const AFT: &str = "aft";
16443 const BROCA: &str = "broca";
16444 const MAGIC: &str = "magic-context";
16445
16446 fn nonce(module_id: &str) -> String {
16447 format!("nonce-{module_id}")
16448 }
16449
16450 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16451 let (tx, rx) = mpsc::channel(64);
16452 (
16453 RouteCtx {
16454 connection_id: ConnectionId::new(connection),
16455 egress: FrameSink::new(tx),
16456 },
16457 rx,
16458 )
16459 }
16460
16461 struct Rig {
16465 handler: ControlHandler,
16466 forwarding: Arc<ForwardingTable>,
16467 owner: RouteCtx,
16468 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16469 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
16470 generation: u64,
16471 next_connection: u64,
16472 _supervisor: Supervisor,
16473 }
16474
16475 async fn rig() -> Rig {
16476 rig_with_flow_support(true).await
16477 }
16478
16479 async fn rig_with_flow_support(flow_support: bool) -> Rig {
16480 rig_with_scope_capabilities(flow_support, false).await
16481 }
16482
16483 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
16484 let mut provides = Vec::new();
16485 if flow_support {
16486 provides.push("flow-scopes/v1");
16487 }
16488 if run_support {
16489 provides.push("agent-run-scopes/v1");
16490 }
16491 rig_with_target_capabilities(&provides).await
16492 }
16493
16494 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
16495 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
16496 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16497 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16500 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
16501 Frame::build(
16502 FrameType::Hello,
16503 control_flags(),
16504 0,
16505 0,
16506 connection,
16507 serde_json::to_vec(&body).unwrap(),
16508 )
16509 .unwrap()
16510 }
16511
16512 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
16513 let registry = Arc::new(Registry::default());
16514 let forwarding = Arc::new(ForwardingTable::default());
16515 let supervisor_handle = SupervisorHandle::new();
16516 let supervisor =
16517 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
16518 .with_handle(supervisor_handle.clone());
16519 supervisor
16520 .supervise_configured(
16521 ModuleSpec {
16522 module_id: OWNER.to_string(),
16523 program: PathBuf::from("/nonexistent/prefrontal-core"),
16524 args: Vec::new(),
16525 env: Vec::new(),
16526 reserved: false,
16527 reserved_prefixes: Vec::new(),
16528 protocol: ModuleProtocol::Subc,
16529 overlap: Default::default(),
16530 },
16531 false,
16532 )
16533 .unwrap();
16534 for module_id in [OWNER, AFT, BROCA, MAGIC] {
16535 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
16536 }
16537 let handler =
16538 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16539 .with_supervisor(supervisor_handle);
16540 let (owner, mut owner_rx) = wide_ctx(1);
16541 hello_via_sink(
16542 &handler,
16543 &owner,
16544 &mut owner_rx,
16545 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
16546 )
16547 .await;
16548 let mut modules = BTreeMap::new();
16549 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
16550 let (ctx, mut rx) = wide_ctx(connection);
16551 let hello = hello_with_capabilities(module_id, connection, provides);
16552 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
16553 modules.insert(module_id.to_string(), (ctx, rx));
16554 }
16555 Rig {
16556 handler,
16557 forwarding,
16558 owner,
16559 _owner_rx: owner_rx,
16560 modules,
16561 generation: 0,
16562 next_connection: 100,
16563 _supervisor: supervisor,
16564 }
16565 }
16566
16567 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
16568 ScopeCarrier::new(Principal::Reserved {
16569 module_id: module_id.to_string(),
16570 })
16571 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
16572 }
16573
16574 fn session(scope_epoch: u64) -> ScopeRecord {
16577 let mut record = head("s", scope_epoch);
16578 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
16579 record.attributes.agent_id = Some("agent-1".to_string());
16580 record.attributes.delegates = true;
16581 record
16582 }
16583
16584 impl Rig {
16585 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
16586 self.generation += 1;
16587 sync(&self.handler, &self.owner, self.generation, scopes)
16588 .await
16589 .expect("the owner's sync is accepted");
16590 }
16591
16592 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16593 ScopeSelector::new(
16594 Principal::Reserved {
16595 module_id: OWNER.to_string(),
16596 },
16597 scope_ref.to_string(),
16598 scope_epoch,
16599 )
16600 }
16601
16602 fn open_frame(
16603 &mut self,
16604 opener: Option<&str>,
16605 target: &str,
16606 scope: Option<ScopeSelector>,
16607 ) -> (
16608 RouteCtx,
16609 mpsc::Receiver<crate::router::OutboundFrame>,
16610 Frame,
16611 ) {
16612 self.next_connection += 1;
16613 let (ctx, rx) = wide_ctx(self.next_connection);
16614 let root = unique_project_root("scoped-open");
16615 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
16616 target: RouteTarget::ToolProvider {
16617 module_id: target.to_string(),
16618 },
16619 identity: BindIdentity::new(
16620 root.path().to_path_buf(),
16621 "unit".to_string(),
16622 "session".to_string(),
16623 ),
16624 consumer_identity: opener.map(|module_id| ConsumerIdentity {
16625 module_id: module_id.to_string(),
16626 launch_nonce: nonce(module_id),
16627 }),
16628 consumer_capabilities: None,
16629 role_versions: None,
16630 admission_facts: None,
16631 scope,
16632 })
16633 .unwrap();
16634 let frame = Frame::build(
16635 FrameType::Request,
16636 control_flags(),
16637 0,
16638 0,
16639 self.next_connection,
16640 body,
16641 )
16642 .unwrap();
16643 (ctx, rx, frame)
16644 }
16645
16646 async fn refused(
16648 &mut self,
16649 opener: Option<&str>,
16650 target: &str,
16651 scope: Option<ScopeSelector>,
16652 ) -> String {
16653 self.refusal_body(opener, target, scope).await["code"]
16654 .as_str()
16655 .unwrap()
16656 .to_string()
16657 }
16658
16659 async fn refusal_body(
16660 &mut self,
16661 opener: Option<&str>,
16662 target: &str,
16663 scope: Option<ScopeSelector>,
16664 ) -> Value {
16665 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
16666 let replies = tokio::time::timeout(
16667 Duration::from_secs(2),
16668 self.handler.handle_control_frame(&ctx, frame),
16669 )
16670 .await
16671 .expect("the open must be refused before waiting for a bind ack")
16672 .unwrap();
16673 assert_eq!(replies.len(), 1, "{replies:?}");
16674 assert_eq!(replies[0].header.ty, FrameType::Error);
16675 let (_, module_rx) = self.modules.get_mut(target).unwrap();
16676 assert!(
16677 module_rx.try_recv().is_err(),
16678 "a refused open relays nothing"
16679 );
16680 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
16681 parse_error(&replies[0])
16682 }
16683
16684 async fn relayed(
16686 &mut self,
16687 opener: Option<&str>,
16688 target: &str,
16689 scope: Option<ScopeSelector>,
16690 ) -> Relayed {
16691 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
16692 let handler = self.handler.clone();
16693 let task_ctx = ctx.clone();
16694 let task = tokio::spawn(async move {
16695 handler
16696 .handle_control_frame(&task_ctx, frame)
16697 .await
16698 .unwrap()
16699 });
16700 let (_, module_rx) = self.modules.get_mut(target).unwrap();
16701 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
16702 .await
16703 .expect("the target receives the relayed route.bind")
16704 .unwrap()
16705 .frame;
16706 Relayed {
16707 target: target.to_string(),
16708 client: ctx,
16709 client_rx: rx,
16710 task,
16711 bind,
16712 }
16713 }
16714
16715 async fn ack(&self, relayed: &Relayed) {
16716 let (module, _) = &self.modules[&relayed.target];
16717 self.handler
16718 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
16719 .await
16720 .unwrap();
16721 }
16722
16723 async fn bound(
16725 &mut self,
16726 opener: Option<&str>,
16727 target: &str,
16728 scope: Option<ScopeSelector>,
16729 ) -> Bound {
16730 let relayed = self.relayed(opener, target, scope).await;
16731 self.ack(&relayed).await;
16732 let Relayed {
16733 target,
16734 client,
16735 mut client_rx,
16736 task,
16737 bind,
16738 } = relayed;
16739 assert!(
16740 task.await.unwrap().is_empty(),
16741 "the open is answered by commit"
16742 );
16743 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
16744 Bound {
16745 target,
16746 client,
16747 client_rx,
16748 channel,
16749 epoch,
16750 bind,
16751 }
16752 }
16753
16754 fn live(&self, route: &Bound) -> bool {
16755 matches!(
16756 self.forwarding
16757 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16758 .unwrap(),
16759 DataRoute::Client(DataRouteState::Bound(_))
16760 )
16761 }
16762 }
16763
16764 struct Relayed {
16765 target: String,
16766 client: RouteCtx,
16767 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16768 task: tokio::task::JoinHandle<Vec<Frame>>,
16769 bind: Frame,
16770 }
16771
16772 struct Bound {
16773 target: String,
16774 client: RouteCtx,
16775 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16776 channel: u16,
16777 epoch: u32,
16778 bind: Frame,
16779 }
16780
16781 impl Bound {
16782 fn closed_reason(&mut self) -> RouteCloseReason {
16785 let mut reason = None;
16786 let mut goodbye = false;
16787 while let Ok(outbound) = self.client_rx.try_recv() {
16788 let frame = outbound.frame;
16789 match frame.header.ty {
16790 FrameType::Goodbye => {
16791 assert_eq!(
16792 (frame.header.channel, frame.header.epoch),
16793 (self.channel, self.epoch)
16794 );
16795 goodbye = true;
16796 }
16797 FrameType::Push => {
16798 let ClientControlPush::RouteClosed {
16799 reason: r,
16800 module_id,
16801 ..
16802 } = serde_json::from_slice(&frame.body).unwrap()
16803 else {
16804 panic!("unexpected push");
16805 };
16806 assert_eq!(module_id, self.target);
16807 reason = Some(r);
16808 }
16809 other => panic!("unexpected frame {other:?}"),
16810 }
16811 }
16812 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
16813 reason.expect("the client is told why the route closed")
16814 }
16815
16816 fn untouched(&mut self) -> bool {
16817 self.client_rx.try_recv().is_err()
16818 }
16819
16820 fn stamp(&self) -> Option<ScopeStamp> {
16821 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
16822 ModuleControlRequest::RouteBind { scope, .. } => scope,
16823 other => panic!("expected a route.bind, got {other:?}"),
16824 }
16825 }
16826 }
16827
16828 #[tokio::test]
16829 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
16830 ) {
16831 let mut rig = rig().await;
16832 let mut record = session(1);
16833 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
16834 record.child_owners = vec![Principal::Reserved {
16835 module_id: MAGIC.to_string(),
16836 }];
16837 rig.sync(vec![record]).await;
16838 let scope = || Some(rig_selector("s", Some(1)));
16839
16840 rig.bound(Some(OWNER), PLEXUS, scope()).await;
16843 rig.bound(Some(AFT), OTHER, scope()).await;
16844 rig.bound(Some(BROCA), PLEXUS, scope()).await;
16845
16846 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
16850 assert_eq!(
16851 rig.refused(opener, target, scope()).await,
16852 error_codes::SCOPE_NOT_CARRIER,
16853 "{opener:?} -> {target}"
16854 );
16855 }
16856 }
16857
16858 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16859 ScopeSelector::new(
16860 Principal::Reserved {
16861 module_id: OWNER.to_string(),
16862 },
16863 scope_ref.to_string(),
16864 scope_epoch,
16865 )
16866 }
16867
16868 #[tokio::test]
16869 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
16870 let mut rig = rig().await;
16871 rig.sync(vec![session(1)]).await;
16872 for opener in [OWNER, AFT] {
16873 assert_eq!(
16874 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
16875 .await,
16876 error_codes::SCOPE_EPOCH_REQUIRED,
16877 "{opener}"
16878 );
16879 }
16880 }
16881
16882 #[tokio::test]
16883 async fn admission_separates_not_synced_not_live_and_ended() {
16884 let mut rig = rig().await;
16885 let code = rig
16887 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16888 .await;
16889 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
16890 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
16891
16892 let ghost = ScopeSelector::new(
16894 Principal::Reserved {
16895 module_id: "ghost".to_string(),
16896 },
16897 "s".to_string(),
16898 Some(1),
16899 );
16900 assert_eq!(
16901 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
16902 error_codes::SCOPE_NOT_LIVE
16903 );
16904
16905 rig.sync(vec![session(2)]).await;
16906 assert_eq!(
16907 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
16908 .await,
16909 error_codes::SCOPE_NOT_LIVE
16910 );
16911 for epoch in [1, 3] {
16912 assert_eq!(
16913 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
16914 .await,
16915 error_codes::SCOPE_ENDED,
16916 "epoch {epoch}"
16917 );
16918 }
16919 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
16921 .await;
16922 }
16923
16924 #[tokio::test]
16925 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
16926 let mut rig = rig().await;
16927 rig.sync(vec![session(1)]).await;
16928 let route = rig
16929 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16930 .await;
16931 let stamp = route.stamp().expect("a scoped bind carries the stamp");
16932 assert_eq!(stamp.scope_ref, "s");
16933 assert_eq!(stamp.scope_epoch, 1);
16934 assert_eq!(stamp.kind, ScopeKind::Head);
16935 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
16936 assert!(stamp.attributes.delegates);
16937 assert!(stamp.owner_authorized);
16938 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16939 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
16940
16941 let (broca, mut broca_rx) = wide_ctx(50);
16944 hello_via_sink(
16945 &rig.handler,
16946 &broca,
16947 &mut broca_rx,
16948 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
16949 )
16950 .await;
16951 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
16952 .await
16953 .unwrap();
16954 let own = ScopeSelector::new(
16955 Principal::Reserved {
16956 module_id: BROCA.to_string(),
16957 },
16958 "b".to_string(),
16959 Some(1),
16960 );
16961 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
16962 assert!(!route.stamp().unwrap().owner_authorized);
16963 }
16964
16965 #[tokio::test]
16966 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
16967 let mut rig = rig().await;
16968 let mut record = head("s", 1);
16969 record.carriers = vec![carrier(AFT, None)];
16970 let flow_id = "Flow:run-7/step_2!~";
16971 record.attributes.flow_id = Some(flow_id.to_string());
16972 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
16973 .await
16974 .unwrap();
16975 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16976 panic!("not a sync reply");
16977 };
16978 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
16979 let route = rig
16980 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16981 .await;
16982 assert!(rig.live(&route), "the stamped bind committed");
16983 let stamp = route.stamp().expect("a flow scope carries a stamp");
16984 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
16985 assert_eq!(stamp.attributes.agent_id, None);
16986 assert!(!stamp.attributes.delegates);
16987 assert!(stamp.owner_authorized);
16988 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16989 assert_eq!(unscoped.stamp(), None);
16990 }
16991
16992 #[tokio::test]
16993 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
16994 let mut rig = rig_with_flow_support(false).await;
16995 let mut record = session(1);
16996 record.attributes.flow_id = Some("flow:7".to_string());
16997 rig.sync(vec![record]).await;
16998 for opener in [OWNER, AFT] {
16999 let body = rig
17000 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17001 .await;
17002 assert_eq!(body["code"], "target_flow_unsupported");
17003 let message = body["message"].as_str().unwrap();
17004 for required in [PLEXUS, "flow-scopes/v1"] {
17005 assert!(message.contains(required), "{message}");
17006 }
17007 }
17008 }
17009
17010 #[tokio::test]
17011 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
17012 let mut rig = rig_with_flow_support(true).await;
17013 let mut record = session(1);
17014 record.attributes.flow_id = Some("flow:7".to_string());
17015 rig.sync(vec![record]).await;
17016 for opener in [OWNER, AFT] {
17017 let route = rig
17018 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17019 .await;
17020 assert!(rig.live(&route));
17021 assert_eq!(
17022 route.stamp().unwrap().attributes.flow_id.as_deref(),
17023 Some("flow:7")
17024 );
17025 }
17026 }
17027
17028 #[tokio::test]
17029 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
17030 use std::future::Future;
17031
17032 let mut rig = rig_with_flow_support(true).await;
17033 let mut record = session(1);
17034 record.attributes.flow_id = Some("flow:7".to_string());
17035 rig.sync(vec![record]).await;
17036 let (client, mut client_rx, frame) =
17037 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
17038 for _ in 0..64 {
17041 client.egress.try_send(route_bind_ack(1)).unwrap();
17042 }
17043 let handler = rig.handler.clone();
17044 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17045 std::future::poll_fn(|cx| {
17046 assert!(open.as_mut().poll(cx).is_pending());
17047 std::task::Poll::Ready(())
17048 })
17049 .await;
17050 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17051
17052 let old_connection = rig.modules[PLEXUS].0.connection_id;
17053 rig.handler.cleanup_connection(old_connection).unwrap();
17054 let (replacement, mut replacement_rx) = wide_ctx(200);
17055 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
17056 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17057 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17058 let hello = Frame::build(
17059 FrameType::Hello,
17060 control_flags(),
17061 0,
17062 0,
17063 200,
17064 serde_json::to_vec(&body).unwrap(),
17065 )
17066 .unwrap();
17067 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
17068
17069 client_rx.try_recv().unwrap();
17070 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17071 .await
17072 .expect("the replacement is refused without waiting for a bind ack")
17073 .unwrap();
17074 assert_eq!(replies.len(), 1);
17075 let body = parse_error(&replies[0]);
17076 assert_eq!(body["code"], "target_flow_unsupported");
17077 for required in [PLEXUS, "flow-scopes/v1"] {
17078 assert!(body["message"].as_str().unwrap().contains(required));
17079 }
17080 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
17081 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17082 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17083 assert!(rig
17084 .handler
17085 .registry
17086 .get_module_by_connection(replacement.connection_id)
17087 .unwrap()
17088 .is_some());
17089 }
17090
17091 #[tokio::test]
17092 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
17093 use std::future::Future;
17094 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17095 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17096 const NEXT_HASH: &str =
17097 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17098 let mut rig = rig().await;
17099 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
17100 record.carriers.push(ScopeCarrier::new(Principal::Direct));
17101 rig.sync(vec![record.clone()]).await;
17102 let (client, mut rx, frame) = rig.open_frame(
17103 None,
17104 PLEXUS,
17105 Some(
17106 rig.selector("s", Some(1))
17107 .with_bind_proof(Some(PROOF.into())),
17108 ),
17109 );
17110 for _ in 0..64 {
17113 client.egress.try_send(route_bind_ack(1)).unwrap();
17114 }
17115 let handler = rig.handler.clone();
17116 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17117 std::future::poll_fn(|cx| {
17118 assert!(open.as_mut().poll(cx).is_pending());
17119 std::task::Poll::Ready(())
17120 })
17121 .await;
17122 record.bind_proof_sha256 = Some(NEXT_HASH.into());
17123 rig.sync(vec![record]).await;
17124 rx.try_recv().unwrap();
17125 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17126 .await
17127 .unwrap()
17128 .unwrap();
17129 assert_eq!(
17130 parse_error(&replies[0])["code"],
17131 error_codes::SCOPE_BIND_PROOF_MISMATCH
17132 );
17133 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
17134 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17135 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17136 }
17137
17138 #[tokio::test]
17139 async fn scope_bind_proof_reserved_openers_ignore_proof() {
17140 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17141 let mut rig = rig().await;
17142 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17143 .await;
17144 for opener in [OWNER, AFT] {
17145 for proof in [None, Some("malformed+proof=")] {
17146 let route = rig
17147 .bound(
17148 Some(opener),
17149 PLEXUS,
17150 Some(
17151 rig.selector("s", Some(1))
17152 .with_bind_proof(proof.map(str::to_string)),
17153 ),
17154 )
17155 .await;
17156 assert!(rig.live(&route));
17157 }
17158 }
17159 }
17160
17161 #[tokio::test]
17162 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
17163 use std::future::Future;
17164 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17165 const NEXT_HASH: &str =
17166 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17167 let mut rig = rig().await;
17168 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17169 .await;
17170 let (client, mut rx, frame) =
17171 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
17172 for _ in 0..64 {
17173 client.egress.try_send(route_bind_ack(1)).unwrap();
17174 }
17175 let handler = rig.handler.clone();
17176 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17177 std::future::poll_fn(|cx| {
17178 assert!(open.as_mut().poll(cx).is_pending());
17179 std::task::Poll::Ready(())
17180 })
17181 .await;
17182 rig.sync(vec![
17183 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
17184 ])
17185 .await;
17186 rx.try_recv().unwrap();
17187 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17188 let bind = std::future::poll_fn(|cx| {
17189 assert!(open.as_mut().poll(cx).is_pending());
17190 module_rx.poll_recv(cx)
17191 })
17192 .await
17193 .unwrap()
17194 .frame;
17195 handler
17196 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
17197 .await
17198 .unwrap();
17199 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17200 .await
17201 .unwrap()
17202 .unwrap();
17203 assert_eq!(replies.len(), 1);
17204 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
17207 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17208 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
17209 let route = rig
17210 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
17211 .await;
17212 assert!(rig.live(&route));
17213 }
17214
17215 #[tokio::test]
17216 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
17217 {
17218 let mut rig = rig_with_flow_support(false).await;
17219 rig.sync(vec![session(1)]).await;
17220 let route = rig
17221 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17222 .await;
17223 assert!(rig.live(&route));
17224 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
17225 let mut record = session(1);
17226 record.attributes.flow_id = Some("flow:7".to_string());
17227 rig.sync(vec![record]).await;
17228 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
17229 assert!(rig.live(&unscoped));
17230 assert_eq!(unscoped.stamp(), None);
17231 }
17232
17233 #[tokio::test]
17234 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
17235 let mut rig = rig().await;
17236 let mut record = session(1);
17237 record.attributes.flow_id = Some("flow:7".to_string());
17238 rig.sync(vec![record.clone()]).await;
17239 let before = rig.forwarding.published_scope_tag(OWNER, "s");
17240 let mut owner_route = rig
17241 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17242 .await;
17243 let mut carrier_route = rig
17244 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
17245 .await;
17246 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17247 rig.sync(vec![record.clone()]).await;
17248 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17249 assert!(rig.live(&owner_route) && owner_route.untouched());
17250 assert!(rig.live(&carrier_route) && carrier_route.untouched());
17251
17252 record.attributes.flow_id = Some("flow:8".to_string());
17253 rig.sync(vec![record]).await;
17254 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
17255 let before = before.unwrap();
17256 assert_eq!(after.scope_epoch, before.scope_epoch);
17257 assert!(after.version > before.version);
17258 for route in [&mut owner_route, &mut carrier_route] {
17259 assert!(!rig.live(route));
17260 assert_eq!(
17261 route.closed_reason(),
17262 RouteCloseReason::ScopeDelegationChanged
17263 );
17264 }
17265 assert!(rig.live(&unscoped) && unscoped.untouched());
17266 for target in [PLEXUS, OTHER] {
17269 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
17270 let goodbye = module_rx
17271 .try_recv()
17272 .expect("the provider sees the drain")
17273 .frame;
17274 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17275 assert!(module_rx.try_recv().is_err());
17276 }
17277 let rebound = rig
17278 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17279 .await;
17280 assert_eq!(
17281 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
17282 Some("flow:8")
17283 );
17284 }
17285
17286 #[tokio::test]
17291 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
17292 let mut rig = rig().await;
17293 rig.sync(vec![session(1)]).await;
17294 let mut cotenant = rig
17295 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17296 .await;
17297
17298 let mut changed = session(1);
17299 changed.child_owners.push(Principal::Reserved {
17300 module_id: MAGIC.to_string(),
17301 });
17302 let mut ended = None;
17303 for (code, next) in [
17304 (error_codes::SCOPE_CHANGED, vec![changed]),
17305 (error_codes::SCOPE_ENDED, Vec::new()),
17306 ] {
17307 let relayed = rig
17308 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17309 .await;
17310 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
17311 ended = Some(next.is_empty());
17312 rig.sync(next).await;
17313 rig.ack(&relayed).await;
17314 let replies = relayed.task.await.unwrap();
17315 assert_eq!(replies.len(), 1, "{replies:?}");
17316 assert_eq!(parse_error(&replies[0])["code"], code);
17317 assert_eq!(
17318 subc_protocol::error_codes::is_retryable_route_open(code),
17319 code == error_codes::SCOPE_CHANGED
17320 );
17321 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17322 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17324 let mut goodbyes = Vec::new();
17327 while let Ok(outbound) = plexus_rx.try_recv() {
17328 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
17329 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
17330 }
17331 assert!(
17332 goodbyes.contains(&(bind_channel, bind_epoch)),
17333 "{goodbyes:?}"
17334 );
17335 assert!(rig
17336 .handler
17337 .registry
17338 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
17339 .unwrap()
17340 .is_some());
17341 }
17342 assert_eq!(ended, Some(true));
17343 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
17346 }
17347
17348 #[tokio::test]
17351 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
17352 struct Case {
17353 name: &'static str,
17354 change: fn(&mut ScopeRecord),
17355 closed: [Option<RouteCloseReason>; 4],
17358 }
17359 use RouteCloseReason::*;
17360 let cases = [
17361 Case {
17362 name: "a carrier entry removed",
17363 change: |r| {
17364 r.carriers.retain(|c| {
17365 c.principal
17366 != Principal::Reserved {
17367 module_id: AFT.to_string(),
17368 }
17369 })
17370 },
17371 closed: [None, Some(ScopeCarrierRemoved), None, None],
17372 },
17373 Case {
17374 name: "a target removed from a carrier",
17375 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
17376 closed: [None, None, None, Some(ScopeCarrierRemoved)],
17377 },
17378 Case {
17379 name: "a bare carrier narrowed to targets",
17380 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
17381 closed: [None, Some(ScopeCarrierRemoved), None, None],
17382 },
17383 Case {
17384 name: "delegates turned off",
17385 change: |r| r.attributes.delegates = false,
17386 closed: [Some(ScopeDelegationChanged); 4],
17387 },
17388 Case {
17389 name: "agent_id changed",
17390 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
17391 closed: [Some(ScopeDelegationChanged); 4],
17392 },
17393 Case {
17394 name: "a carrier added, child owners changed, the record re-sent",
17395 change: |r| {
17396 r.carriers.push(carrier(MAGIC, None));
17397 r.child_owners.push(Principal::Reserved {
17398 module_id: MAGIC.to_string(),
17399 });
17400 },
17401 closed: [None; 4],
17402 },
17403 Case {
17404 name: "a target added",
17405 change: |r| {
17406 r.carriers[1]
17407 .targets
17408 .as_mut()
17409 .unwrap()
17410 .push("third".to_string())
17411 },
17412 closed: [None; 4],
17413 },
17414 Case {
17415 name: "delegates turned on",
17416 change: |r| r.attributes.delegates = true,
17417 closed: [None; 4],
17418 },
17419 ];
17420 for case in cases {
17421 let mut rig = rig().await;
17422 rig.sync(vec![session(1)]).await;
17423 let scope = || Some(rig_selector("s", Some(1)));
17424 let mut routes = [
17425 rig.bound(Some(OWNER), PLEXUS, scope()).await,
17426 rig.bound(Some(AFT), PLEXUS, scope()).await,
17427 rig.bound(Some(BROCA), PLEXUS, scope()).await,
17428 rig.bound(Some(BROCA), OTHER, scope()).await,
17429 ];
17430 let mut record = session(1);
17431 (case.change)(&mut record);
17432 rig.sync(vec![record]).await;
17433 for (index, expected) in case.closed.iter().enumerate() {
17434 let route = &mut routes[index];
17435 match expected {
17436 Some(reason) => {
17437 assert!(!rig.live(route), "{}: route {index} still live", case.name);
17438 assert_eq!(
17439 route.closed_reason(),
17440 *reason,
17441 "{}: route {index}",
17442 case.name
17443 );
17444 }
17445 None => {
17446 assert!(rig.live(route), "{}: route {index} closed", case.name);
17447 assert!(
17448 route.untouched(),
17449 "{}: route {index} was told something",
17450 case.name
17451 );
17452 }
17453 }
17454 }
17455 }
17456 }
17457
17458 #[tokio::test]
17459 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
17460 for next in [Vec::new(), vec![session(2)]] {
17462 let mut rig = rig().await;
17463 rig.sync(vec![session(1)]).await;
17464 let mut route = rig
17465 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17466 .await;
17467 rig.sync(next).await;
17468 assert!(!rig.live(&route));
17469 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
17470 }
17471
17472 let mut rig = rig().await;
17475 let mut child = session(1);
17476 child.scope_ref = "child".to_string();
17477 child.kind = ScopeKind::Worker;
17478 child.parent = Some(ScopeParent::new(
17479 Principal::Reserved {
17480 module_id: OWNER.to_string(),
17481 },
17482 "s".to_string(),
17483 1,
17484 ));
17485 rig.sync(vec![session(1), child.clone()]).await;
17486 let mut child_route = rig
17487 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
17488 .await;
17489 assert_eq!(
17490 child_route.stamp().unwrap().parent_state,
17491 Some(ParentState::Linked)
17492 );
17493 rig.sync(vec![child]).await;
17494 assert!(!rig.live(&child_route));
17495 assert_eq!(
17496 child_route.closed_reason(),
17497 RouteCloseReason::ScopeParentEnded
17498 );
17499 }
17500
17501 #[tokio::test]
17502 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
17503 ) {
17504 let mut rig = rig().await;
17505 rig.sync(vec![session(1)]).await;
17506 let mut route = rig
17507 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17508 .await;
17509 let before = rig.forwarding.published_scope_tag(OWNER, "s");
17510 rig.sync(vec![session(1)]).await;
17511 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17512 assert!(rig.live(&route) && route.untouched());
17513
17514 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
17518 .forwarding
17519 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17520 .unwrap()
17521 else {
17522 panic!("the route is bound");
17523 };
17524 binding.flow.acquire_tagged(9, false).await.unwrap();
17525 let mut widened = session(1);
17526 widened.carriers.push(carrier(MAGIC, None));
17527 rig.sync(vec![widened]).await;
17528 assert!(rig.live(&route) && route.untouched());
17529 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17530 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
17531 assert_eq!(binding.flow.in_flight(), 1);
17534 binding
17535 .flow
17536 .acquire_tagged(10, false)
17537 .await
17538 .expect("the flow is still open");
17539 }
17540
17541 #[tokio::test]
17544 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
17545 let mut rig = rig().await;
17546 rig.sync(vec![session(1)]).await;
17547 let mut on_incumbent = rig
17548 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17549 .await;
17550
17551 let (candidate, _candidate_rx) = wide_ctx(9);
17554 let registration = rig
17555 .handler
17556 .registry
17557 .register_candidate_with_control_ops(
17558 manifest(PLEXUS, PROTOCOL_VERSION),
17559 PROTOCOL_VERSION,
17560 candidate.connection_id,
17561 module_baseline_control_ops(),
17562 )
17563 .unwrap();
17564 rig.forwarding
17565 .register_candidate_module_connection(
17566 candidate.connection_id,
17567 PLEXUS.to_string(),
17568 PROTOCOL_VERSION,
17569 manifest_concurrency(®istration.manifest),
17570 candidate.egress.clone(),
17571 )
17572 .unwrap();
17573 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
17574 rig.handler
17575 .registry
17576 .promote_candidate(PLEXUS)
17577 .unwrap()
17578 .unwrap();
17579 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
17580
17581 rig.sync(Vec::new()).await;
17582 assert!(!rig.live(&on_incumbent));
17583 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
17584 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17585 let goodbye = incumbent_rx
17586 .try_recv()
17587 .expect("the superseded endpoint is told")
17588 .frame;
17589 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17590 }
17591 }
17592}
17593
17594#[cfg(test)]
17595mod concurrency_default_exposure_tests {
17596 use super::*;
17597
17598 fn hello_body(role_json: &str) -> Vec<u8> {
17599 format!(
17600 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
17601 )
17602 .into_bytes()
17603 }
17604
17605 fn manifest_from(body: &[u8]) -> ModuleManifest {
17606 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
17607 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
17608 .expect("manifest parses")
17609 }
17610
17611 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
17612
17613 #[test]
17614 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
17615 let body = hello_body(&format!(
17616 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
17617 ));
17618 let manifest = manifest_from(&body);
17619 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17622 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
17623 }
17624
17625 #[test]
17626 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
17627 let body = hello_body(&format!(
17628 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
17629 ));
17630 let manifest = manifest_from(&body);
17631 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17632 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17633 }
17634
17635 #[test]
17636 fn non_management_roles_are_never_reported() {
17637 let body = hello_body(
17638 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
17639 );
17640 let manifest = manifest_from(&body);
17641 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17642 }
17643}