1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_confirm_operations,
25 validate_hello_display_name, validate_hello_event_declarations,
26 validate_hello_flow_operations, validate_hello_rich_ui_descriptor,
27 validate_hello_role_declarations, validate_hello_self_signal_declarations,
28 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
29 ProviderRole,
30 },
31 peer::{ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY, OPERATOR_APP_CAPABILITY},
32 scope::{
33 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
34 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
35 SCOPE_SYNC_OP,
36 },
37 session::{
38 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
39 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
40 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
41 ROLE_VERSIONS_FIELD,
42 },
43 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
44 Priority, RouteTarget, PROTOCOL_VERSION,
45};
46use tokio::time::{timeout_at, Instant};
47use tracing::{debug, info, instrument::WithSubscriber, warn};
48
49use crate::{
50 caller_identity::CallerIdentity,
51 capability_requirements::{
52 log_duplicate_claim_events, log_manifest_change, log_requirement_events,
53 CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
54 RequirementStatus, RuntimeModule,
55 },
56 child_roster::DaemonShutdownFlag,
57 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
58 forwarding::{
59 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
60 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
61 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
62 },
63 observability::{
64 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
65 },
66 privileged_prompt::{prompt_text, PrivilegedOperation},
67 provenance::{
68 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
69 },
70 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
71 router::{daemon_error_flags, RouteCtx, RouterError},
72 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
73 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
74 stderr_tail::{CaptureState, TailEntry},
75 supervise::{
76 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
77 SupervisorHandle, SwapHelloAdmission,
78 },
79 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
80};
81
82pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
88
89const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
90const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
91const CAP_PING_PONG: &str = "ping_pong_v1";
92const CAP_SESSION_ATTACH: &str = "session_attach_v1";
93const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
94
95const SUBC_CONTROL_OPS: &[&str] = &[
96 ops::DAEMON_SHUTDOWN_INTENT,
97 ops::ACTIVITY_RECORD,
98 ops::ACTIVITY_QUERY,
99 ops::ACTIVITY_SUBSCRIBE,
100 ops::SERVER_DESCRIBE,
101 ops::CATALOG_LIST,
102 ops::ROUTE_OPEN,
103 ops::ROUTE_POLL,
104 ops::ROUTE_CLOSING,
105 ops::ROUTE_CLOSED,
106 ops::SUPERVISOR_LIST,
107 ops::SUPERVISOR_RESTART,
108 ops::SUPERVISOR_SWAP,
109 ops::SUPERVISOR_RELOAD,
110 ops::SUPERVISOR_RESCAN,
111 ops::SUPERVISOR_RELEASE_RESERVED,
112 ops::SUPERVISOR_SET_ENABLED,
113 ops::SUPERVISOR_HEALTH_PROBE,
114 ops::SUPERVISOR_HEALTH,
115 ops::SUPERVISOR_STDERR_TAIL,
116 ops::SUPERVISOR_TERMINALS,
117 ops::SUPERVISOR_ROUTES,
118 ops::SUPERVISOR_PROVENANCE,
119 ops::SUPERVISOR_SPAWN_SNAPSHOT,
120 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
121];
122
123const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
130
131mod rescan_checked;
132use rescan_checked::{RescanMode, RescanPlan};
133
134const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
135 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
136 "supervisor.live_roots",
137 SCOPE_SYNC_OP,
138 SCOPE_APPLY_OP,
139 SCOPE_DESCRIBE_OP,
140 "operator.confirm",
141];
142
143const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
147
148const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
149
150pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
158
159pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
174
175pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
193
194const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
195const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
196
197fn reload_verdict(
198 configured: &Path,
199 spawned_from: Option<&Path>,
200 image: subc_control::RunningImageAgreement,
201) -> PendingReloadVerdict {
202 let path = match spawned_from {
203 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
204 ReloadPathAgreement::Match
205 }
206 Some(spawned_from) => ReloadPathAgreement::Mismatch {
207 configured: configured.to_path_buf(),
208 spawned_from: spawned_from.to_path_buf(),
209 },
210 None => ReloadPathAgreement::Unavailable {
211 reason: if matches!(
212 image,
213 subc_control::RunningImageAgreement::Unavailable {
214 reason: subc_control::RunningImageUnavailableReason::NotRunning
215 }
216 ) {
217 ReloadPathUnavailableReason::NotRunning
218 } else {
219 ReloadPathUnavailableReason::SpawnedPathUnavailable
220 },
221 },
222 };
223 PendingReloadVerdict {
224 path,
225 image,
226 spec: None,
227 }
228}
229
230fn launch_spec_agreement(
231 configured: &crate::supervise::ModuleSpec,
232 running: bool,
233 spawned: Option<&crate::supervise::LaunchSpecDigest>,
234) -> subc_control::LaunchSpecAgreement {
235 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
236 if !running {
237 return Agreement::Unavailable {
238 reason: Reason::NotRunning,
239 };
240 }
241 match spawned {
242 None => Agreement::Unavailable {
243 reason: Reason::SpawnedSpecUnavailable,
244 },
245 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
246 Agreement::Match
247 }
248 Some(_) => Agreement::Mismatch,
249 }
250}
251
252fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
253 if configured == spawned_from {
254 return true;
255 }
256 #[cfg(windows)]
257 {
258 match (
264 std::fs::canonicalize(configured),
265 std::fs::canonicalize(spawned_from),
266 ) {
267 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
268 _ => false,
269 }
270 }
271 #[cfg(not(windows))]
272 {
273 false
274 }
275}
276
277#[derive(Clone)]
278struct DaemonProvenanceFacts {
279 build: DaemonBuildProvenance,
280 pid: Option<u32>,
281 started_at_ms: Option<u64>,
282 start_clock: Option<crate::clock::StartClock>,
283 executable_path: Option<PathBuf>,
284 executable_identity: Option<SpawnedFileIdentity>,
285 process_start_time: Option<u64>,
286 probe: ExecutableIdentityProbe,
287}
288
289impl Default for DaemonProvenanceFacts {
290 fn default() -> Self {
291 Self {
292 build: DaemonBuildProvenance {
293 build_git_sha: None,
294 build_lock_digest: None,
295 },
296 pid: None,
297 started_at_ms: None,
298 start_clock: None,
299 executable_path: None,
300 executable_identity: None,
301 process_start_time: None,
302 probe: ExecutableIdentityProbe::default(),
303 }
304 }
305}
306
307#[derive(Debug, Clone)]
308struct SupervisorRescanContext {
309 supervisor: Supervisor,
310 config_path: PathBuf,
311 configured_port: Option<u16>,
312 storage_config: Option<crate::daemon_config::StorageConfig>,
313 admission_facts_carrier_module_id: Option<String>,
314 admission_facts_targets: Option<Vec<String>>,
315 scope_authority_owners: Vec<String>,
316 privileged_operator_prompt: PrivilegedOperatorPrompt,
317 generation: Arc<std::sync::atomic::AtomicU64>,
323}
324
325const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
346 "reloading",
347 "supervisor_not_live",
348 "registration_not_active",
349 "no_forwarding_connection",
350 "relay_send_failed",
351];
352
353#[derive(Clone)]
355pub struct ControlHandler {
356 registry: Arc<Registry>,
357 activity: Option<crate::activity::ActivityLedger>,
358 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
359 forwarding: Arc<ForwardingTable>,
360 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
361 supervisor: SupervisorHandle,
362 subc_capabilities: Arc<[String]>,
363 route_bind_relay_timeout: Duration,
367 route_bind_relay_timeouts: BTreeMap<String, Duration>,
371 route_bind_breakers: RouteBindBreakers,
374 route_bind_concurrency: RouteBindConcurrency,
377 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
382 route_bind_breaker_threshold: u32,
384 route_bind_breaker_cooldown: Duration,
386 health_probe_timeout: Duration,
387 storage_config: Option<crate::daemon_config::StorageConfig>,
390 machine_id: Option<crate::machine_id::MachineId>,
394 admission_facts_carrier_module_id: Option<String>,
395 admission_facts_targets: Option<Vec<String>>,
396 scopes: Arc<RwLock<ScopeTable>>,
400 scope_deadline_added: Arc<tokio::sync::Notify>,
404 scope_authority_owners: Vec<String>,
407 privileged_operator_prompt: PrivilegedOperatorPrompt,
409 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
412 build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
413 observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
414 local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
415 pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
416 pub(crate) operator_apps:
417 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
418 pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
423 pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
429 #[cfg(test)]
431 pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
432 #[cfg(test)]
433 pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
434 #[allow(dead_code)]
438 #[cfg(windows)]
439 pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
440 #[allow(dead_code)]
441 #[cfg(windows)]
442 pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
443 #[allow(dead_code)]
447 #[cfg(windows)]
448 pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
449 #[cfg(all(test, windows))]
452 pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
453 #[cfg(test)]
454 caller_lookup: Option<TestCallerLookup>,
455 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
458 rescan: Option<SupervisorRescanContext>,
459 connected_clients: ConnectedClients,
460 counters: DaemonCounters,
461 local_endpoint_unavailable: Option<String>,
464 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
465 daemon_shutdown: DaemonShutdownFlag,
470 daemon_provenance: DaemonProvenanceFacts,
471 #[cfg(test)]
472 control_dispatch_delay: Option<Duration>,
473 #[cfg(test)]
474 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
475 #[cfg(test)]
479 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
480 #[cfg(test)]
486 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
487}
488
489impl fmt::Debug for ControlHandler {
490 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
491 f.debug_struct("ControlHandler")
492 .field("registry", &self.registry)
493 .field("forwarding", &self.forwarding)
494 .field("process_liveness", &self.process_liveness.is_some())
495 .field("supervisor", &self.supervisor)
496 .field("subc_capabilities", &self.subc_capabilities)
497 .finish()
498 }
499}
500
501struct RouteOpenRequest {
502 target: RouteTarget,
503 identity: BindIdentity,
504 consumer_identity: Option<ConsumerIdentity>,
505 consumer_capabilities: Option<Vec<String>>,
506 role_versions: Option<BTreeMap<String, String>>,
507 busy_refusals: Option<bool>,
508 admission_facts: Option<serde_json::Value>,
509 scope: Option<ScopeSelector>,
510}
511
512#[derive(Clone, Copy)]
513enum RouteOpenPhase {
514 Admission,
515 ProjectRoot,
516 EgressReservation,
517 BindSend,
518 BindAckWait,
519}
520
521struct RouteOpenTimings {
525 module_id: String,
526 principal: String,
527 admission: Duration,
528 project_root: Duration,
529 egress_reservation: Duration,
530 bind_send: Duration,
531 bind_ack_wait: Duration,
532 phase: Option<RouteOpenPhase>,
533 phase_started_at: Instant,
534}
535
536impl RouteOpenTimings {
537 fn new() -> Self {
538 Self {
539 module_id: String::new(),
540 principal: "unresolved".to_string(),
542 admission: Duration::ZERO,
543 project_root: Duration::ZERO,
544 egress_reservation: Duration::ZERO,
545 bind_send: Duration::ZERO,
546 bind_ack_wait: Duration::ZERO,
547 phase: Some(RouteOpenPhase::Admission),
548 phase_started_at: Instant::now(),
549 }
550 }
551
552 fn finish(&mut self) {
553 let elapsed = self.phase_started_at.elapsed();
554 match self.phase.take() {
555 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
556 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
557 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
558 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
559 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
560 None => {}
561 }
562 }
563
564 fn enter(&mut self, phase: RouteOpenPhase) {
565 self.finish();
566 self.phase = Some(phase);
567 self.phase_started_at = Instant::now();
568 }
569}
570
571struct RouteBindReservationGuard {
572 forwarding: Arc<ForwardingTable>,
573 endpoint: ModuleEndpointId,
574 relay_corr: u64,
575 armed: bool,
576}
577
578struct ModuleControlRpcGuard {
579 forwarding: Arc<ForwardingTable>,
580 endpoint: ModuleEndpointId,
581 corr: u64,
582 armed: bool,
583}
584
585impl ModuleControlRpcGuard {
586 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
587 Self {
588 forwarding,
589 endpoint,
590 corr,
591 armed: true,
592 }
593 }
594
595 fn disarm(&mut self) {
596 self.armed = false;
597 }
598}
599
600impl Drop for ModuleControlRpcGuard {
601 fn drop(&mut self) {
602 if self.armed {
603 let _ = self
604 .forwarding
605 .cancel_module_control_rpc(self.endpoint, self.corr);
606 }
607 }
608}
609
610impl RouteBindReservationGuard {
611 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
612 Self {
613 forwarding,
614 endpoint,
615 relay_corr,
616 armed: true,
617 }
618 }
619
620 fn release_and_disarm(&mut self) {
621 if !self.armed {
622 return;
623 }
624 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
625 self.endpoint,
626 self.relay_corr,
627 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
628 ) {
629 send_goodbye_target_best_effort(
630 &self.forwarding.counters(),
631 &target,
632 "canceled route.bind",
633 );
634 }
635 self.armed = false;
636 }
637
638 fn disarm(&mut self) {
639 self.armed = false;
640 }
641}
642
643impl Drop for RouteBindReservationGuard {
644 fn drop(&mut self) {
645 self.release_and_disarm();
646 }
647}
648
649#[derive(Debug, Clone, Default)]
679pub(crate) struct RouteBindBreakers {
680 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
681}
682
683#[derive(Debug, Clone, Default)]
684pub(crate) struct RouteBindConcurrency {
685 modules: Arc<Mutex<HashMap<String, usize>>>,
686}
687
688struct RouteBindConcurrencyGuard {
689 concurrency: RouteBindConcurrency,
690 module_id: String,
691}
692
693impl RouteBindConcurrency {
694 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
697 let mut modules = self
698 .modules
699 .lock()
700 .expect("route.bind concurrency mutex poisoned");
701 let in_flight = modules.entry(module_id.to_string()).or_default();
702 if *in_flight >= limit {
703 return Err(*in_flight);
704 }
705 *in_flight += 1;
706 Ok(RouteBindConcurrencyGuard {
707 concurrency: self.clone(),
708 module_id: module_id.to_string(),
709 })
710 }
711}
712
713impl Drop for RouteBindConcurrencyGuard {
714 fn drop(&mut self) {
715 let mut modules = self
716 .concurrency
717 .modules
718 .lock()
719 .expect("route.bind concurrency mutex poisoned");
720 let remove = {
721 let in_flight = modules
722 .get_mut(&self.module_id)
723 .expect("admitted route.bind has a concurrency entry");
724 *in_flight -= 1;
725 *in_flight == 0
726 };
727 if remove {
728 modules.remove(&self.module_id);
729 }
730 }
731}
732
733#[derive(Debug, Default)]
734struct ModuleBreakerState {
735 consecutive_timeouts: u32,
737 cooldown_until: Option<Instant>,
740 probe_in_flight: Option<Arc<()>>,
745}
746
747enum RouteBindAdmission<'a> {
749 Admitted {
750 guard: RouteBindBreakerGuard<'a>,
751 probe: bool,
754 },
755 Refused {
756 consecutive_timeouts: u32,
757 retry_in: Duration,
761 probe_in_flight: bool,
762 },
763}
764
765struct RouteBindBreakerGuard<'a> {
772 breakers: RouteBindBreakers,
773 module_id: &'a str,
774 probe_token: Option<Arc<()>>,
775 settled: bool,
776}
777
778impl RouteBindBreakerGuard<'_> {
779 fn record_accepted(&mut self) -> bool {
783 self.settled = true;
784 self.breakers.record_accepted(self.module_id)
785 }
786
787 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
790 self.settled = true;
791 self.breakers.record_timeout(
792 self.module_id,
793 self.probe_token.as_ref(),
794 threshold,
795 cooldown,
796 )
797 }
798
799 fn record_inconclusive(&mut self) {
808 self.settled = true;
809 self.breakers
810 .record_inconclusive(self.module_id, self.probe_token.as_ref());
811 }
812}
813
814impl Drop for RouteBindBreakerGuard<'_> {
815 fn drop(&mut self) {
816 if !self.settled {
817 self.breakers
818 .record_inconclusive(self.module_id, self.probe_token.as_ref());
819 }
820 }
821}
822
823struct BreakerOpened {
827 consecutive_timeouts: u32,
828 reopened_after_probe: bool,
831}
832
833impl RouteBindBreakers {
834 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
835 self.modules
836 .lock()
837 .expect("route.bind breaker mutex poisoned")
838 }
839
840 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
843 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
844 probe: probe_token.is_some(),
845 guard: RouteBindBreakerGuard {
846 breakers: self.clone(),
847 module_id,
848 probe_token,
849 settled: false,
850 },
851 };
852
853 let mut modules = self.lock();
854 let Some(state) = modules.get_mut(module_id) else {
855 return admitted(None);
856 };
857 let Some(cooldown_until) = state.cooldown_until else {
858 return admitted(None);
859 };
860 if state.probe_in_flight.is_some() {
861 return RouteBindAdmission::Refused {
862 consecutive_timeouts: state.consecutive_timeouts,
863 retry_in: Duration::ZERO,
864 probe_in_flight: true,
865 };
866 }
867 let now = Instant::now();
868 if now < cooldown_until {
869 return RouteBindAdmission::Refused {
870 consecutive_timeouts: state.consecutive_timeouts,
871 retry_in: cooldown_until - now,
872 probe_in_flight: false,
873 };
874 }
875 let token = Arc::new(());
876 state.probe_in_flight = Some(Arc::clone(&token));
877 admitted(Some(token))
878 }
879
880 fn record_accepted(&self, module_id: &str) -> bool {
881 self.lock()
882 .remove(module_id)
883 .is_some_and(|state| state.cooldown_until.is_some())
884 }
885
886 fn record_timeout(
887 &self,
888 module_id: &str,
889 probe_token: Option<&Arc<()>>,
890 threshold: u32,
891 cooldown: Duration,
892 ) -> Option<BreakerOpened> {
893 let mut modules = self.lock();
894 let state = modules.entry(module_id.to_string()).or_default();
895 let was_open = state.cooldown_until.is_some();
896 let was_probe = Self::owns_probe(state, probe_token);
897 if was_probe {
898 state.probe_in_flight = None;
899 }
900 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
901 if state.consecutive_timeouts < threshold {
902 return None;
903 }
904 state.cooldown_until = Some(Instant::now() + cooldown);
905 Some(BreakerOpened {
906 consecutive_timeouts: state.consecutive_timeouts,
907 reopened_after_probe: was_open && was_probe,
908 })
909 }
910
911 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
912 state
916 .probe_in_flight
917 .as_ref()
918 .zip(token)
919 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
920 }
921
922 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
923 if let Some(state) = self.lock().get_mut(module_id) {
924 if Self::owns_probe(state, probe_token) {
925 state.probe_in_flight = None;
926 }
927 }
928 }
929
930 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
945 self.lock()
946 .remove(module_id)
947 .map(|state| state.consecutive_timeouts)
948 .filter(|discarded| *discarded > 0)
949 }
950
951 fn open_snapshot(&self) -> Option<serde_json::Value> {
958 let now = Instant::now();
959 let modules = self.lock();
960 let open = modules
961 .iter()
962 .filter_map(|(module_id, state)| {
963 let cooldown_until = state.cooldown_until?;
964 Some((
965 module_id.clone(),
966 serde_json::json!({
967 "consecutive_timeouts": state.consecutive_timeouts,
968 "cooldown_remaining_ms":
969 cooldown_until.saturating_duration_since(now).as_millis() as u64,
970 "probe_in_flight": state.probe_in_flight.is_some(),
971 }),
972 ))
973 })
974 .collect::<serde_json::Map<String, serde_json::Value>>();
975 (!open.is_empty()).then_some(serde_json::Value::Object(open))
976 }
977}
978
979impl ControlHandler {
980 pub fn new(registry: Arc<Registry>) -> Self {
981 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
982 }
983
984 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
985 let counters = forwarding.counters();
986 let route_bind_breakers = forwarding.route_bind_breakers();
990 let route_bind_concurrency = forwarding.route_bind_concurrency();
991 let route_outages = forwarding.route_outages();
992 Self {
993 registry,
994 activity: None,
995 activity_streams: Arc::new(Mutex::new(HashMap::new())),
996 forwarding,
997 process_liveness: None,
998 supervisor: SupervisorHandle::new(),
999 subc_capabilities: Arc::from([
1000 CAP_MANIFEST_REGISTRATION.to_string(),
1001 CAP_CHANNEL_LIFECYCLE.to_string(),
1002 CAP_PING_PONG.to_string(),
1003 CAP_SESSION_ATTACH.to_string(),
1004 CAP_ADMISSION_FACTS_RELAY.to_string(),
1005 CAP_SCOPES_V1.to_string(),
1006 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1007 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1008 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1009 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1010 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1011 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1012 subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1013 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1014 subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1015 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1016 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1017 ]),
1018 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1019 route_bind_relay_timeouts: BTreeMap::new(),
1020 route_bind_breakers,
1021 route_bind_concurrency,
1022 route_outages,
1023 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1024 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1025 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1026 storage_config: None,
1027 machine_id: None,
1028 admission_facts_carrier_module_id: None,
1029 admission_facts_targets: None,
1030 scopes: Arc::new(RwLock::new(ScopeTable::new(
1031 crate::daemon_config::default_scope_authority_owners(),
1032 ))),
1033 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1034 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1035 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1036 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1037 build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1038 observed_peers: Arc::new(Mutex::new(HashMap::new())),
1039 local_connections: Arc::new(Mutex::new(HashSet::new())),
1040 operator_app_policy: Default::default(),
1041 operator_apps: Arc::new(Mutex::new(HashMap::new())),
1042 launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1043 app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1044 #[cfg(test)]
1045 app_open_test_platform: None,
1046 #[cfg(test)]
1047 app_open_test_launch: None,
1048 #[cfg(windows)]
1049 windows_verify: None,
1050 #[cfg(windows)]
1051 windows_flash: None,
1052 #[cfg(windows)]
1053 windows_track: None,
1054 #[cfg(all(test, windows))]
1055 windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1056 #[cfg(test)]
1057 caller_lookup: None,
1058 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1059 rescan: None,
1060 connected_clients: ConnectedClients::new(),
1061 local_endpoint_unavailable: None,
1062 counters,
1063 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1064 daemon_shutdown: DaemonShutdownFlag::default(),
1065 daemon_provenance: DaemonProvenanceFacts::default(),
1066 #[cfg(test)]
1067 control_dispatch_delay: None,
1068 #[cfg(test)]
1069 provenance_probe_override: None,
1070 #[cfg(test)]
1071 route_bind_relay_sent_hold: None,
1072 #[cfg(test)]
1073 hello_registry_to_forwarding_hold: None,
1074 }
1075 }
1076
1077 pub fn with_storage_config(
1080 mut self,
1081 storage_config: Option<crate::daemon_config::StorageConfig>,
1082 ) -> Self {
1083 self.storage_config = storage_config;
1084 self
1085 }
1086
1087 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1090 self.activity = Some(crate::activity::ActivityLedger::open(
1091 run_dir.join("activity.jsonl"),
1092 incarnation,
1093 ));
1094 self.supervisor.set_activity(self.activity.clone());
1095 self
1096 }
1097
1098 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1101 self.machine_id = machine_id;
1102 self
1103 }
1104
1105 pub fn with_admission_facts_config(
1109 mut self,
1110 carrier_module_id: Option<String>,
1111 targets: Option<Vec<String>>,
1112 ) -> Self {
1113 self.admission_facts_carrier_module_id = carrier_module_id;
1114 self.admission_facts_targets = targets;
1115 self
1116 }
1117
1118 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1122 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1123 self.scope_authority_owners = owners;
1124 self
1125 }
1126
1127 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1129 self.privileged_operator_prompt = mode;
1130 self
1131 }
1132
1133 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1136 self.route_bind_relay_timeout = timeout;
1137 self
1138 }
1139
1140 pub fn with_route_bind_relay_timeouts(
1146 mut self,
1147 timeouts: impl IntoIterator<Item = (String, Duration)>,
1148 ) -> Self {
1149 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1150 self
1151 }
1152
1153 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1159 self.route_bind_relay_timeouts
1160 .get(module_id)
1161 .copied()
1162 .unwrap_or(self.route_bind_relay_timeout)
1163 }
1164
1165 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1173 self.route_bind_breaker_threshold = threshold.max(1);
1174 self.route_bind_breaker_cooldown = cooldown;
1175 self
1176 }
1177
1178 #[cfg(test)]
1179 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1180 self.health_probe_timeout = timeout;
1181 self
1182 }
1183
1184 #[cfg(test)]
1185 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1186 self.control_dispatch_delay = Some(delay);
1187 self
1188 }
1189
1190 #[cfg(test)]
1194 pub(crate) fn with_route_bind_relay_sent_hold(
1195 mut self,
1196 hold: Arc<tokio::sync::Barrier>,
1197 ) -> Self {
1198 self.route_bind_relay_sent_hold = Some(hold);
1199 self
1200 }
1201
1202 #[cfg(test)]
1207 pub(crate) fn with_hello_registry_to_forwarding_hold(
1208 mut self,
1209 hold: Arc<std::sync::Barrier>,
1210 ) -> Self {
1211 self.hello_registry_to_forwarding_hold = Some(hold);
1212 self
1213 }
1214
1215 #[cfg(test)]
1216 fn hold_between_registry_and_forwarding(&self) {
1217 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1218 hold.wait();
1219 hold.wait();
1220 }
1221 }
1222
1223 pub fn with_process_liveness(
1224 mut self,
1225 process_liveness: Arc<dyn ModuleProcessLiveness>,
1226 ) -> Self {
1227 self.process_liveness = Some(process_liveness);
1228 self
1229 }
1230
1231 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1232 self.supervisor = supervisor;
1233 self.supervisor.set_activity(self.activity.clone());
1234 self
1235 }
1236
1237 pub fn with_daemon_provenance(
1238 mut self,
1239 pid: u32,
1240 started_at_ms: u64,
1241 executable_path: Option<PathBuf>,
1242 build_git_sha: Option<String>,
1243 build_lock_digest: Option<String>,
1244 ) -> Self {
1245 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1246 let process_start_time = process_start_time(pid);
1247 self.daemon_provenance = DaemonProvenanceFacts {
1248 build: DaemonBuildProvenance {
1249 build_git_sha,
1250 build_lock_digest,
1251 },
1252 pid: Some(pid),
1253 started_at_ms: Some(started_at_ms),
1254 start_clock: None,
1255 executable_path,
1256 executable_identity,
1257 process_start_time,
1258 probe: ExecutableIdentityProbe::default(),
1259 };
1260 self
1261 }
1262
1263 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1264 self.daemon_provenance.start_clock = Some(clock);
1265 self
1266 }
1267
1268 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1270 self.scopes
1271 .write()
1272 .unwrap_or_else(|poisoned| poisoned.into_inner())
1273 .set_wall_clock(Arc::new(clock));
1274 self
1275 }
1276
1277 #[cfg(test)]
1278 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1279 self.provenance_probe_override = Some(result);
1280 self
1281 }
1282
1283 pub fn with_capability_config(
1287 self,
1288 modules: impl IntoIterator<Item = (String, bool)>,
1289 reserved_capabilities: BTreeMap<String, String>,
1290 ) -> Self {
1291 self.capability_evaluator
1292 .configure(modules, reserved_capabilities);
1293 self
1294 }
1295
1296 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1300 self.daemon_shutdown = flag;
1301 self
1302 }
1303
1304 pub fn with_supervisor_rescan(
1305 mut self,
1306 supervisor: Supervisor,
1307 config_path: impl Into<PathBuf>,
1308 configured_port: Option<u16>,
1309 ) -> Self {
1310 self.rescan = Some(SupervisorRescanContext {
1311 supervisor,
1312 config_path: config_path.into(),
1313 configured_port,
1314 storage_config: self.storage_config.clone(),
1315 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1316 admission_facts_targets: self.admission_facts_targets.clone(),
1317 scope_authority_owners: self.scope_authority_owners.clone(),
1318 privileged_operator_prompt: self.privileged_operator_prompt,
1319 generation: Arc::default(),
1320 });
1321 self
1322 }
1323
1324 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1325 self.connected_clients = connected_clients;
1326 self
1327 }
1328
1329 pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1332 self.local_endpoint_unavailable = reason;
1333 self
1334 }
1335
1336 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1337 Arc::clone(&self.forwarding)
1338 }
1339
1340 pub(crate) fn counters(&self) -> DaemonCounters {
1341 self.counters.clone()
1342 }
1343
1344 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1347 tokio::spawn(async move {
1348 loop {
1349 self.capability_evaluator
1350 .wait_for_change_or_deadline()
1351 .await;
1352 self.refresh_capability_requirements();
1353 }
1354 });
1355 }
1356
1357 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1363 tokio::spawn(async move {
1364 loop {
1365 let has_deadlines = self
1366 .scopes
1367 .read()
1368 .unwrap_or_else(|poisoned| poisoned.into_inner())
1369 .has_deadlines();
1370 if !has_deadlines {
1371 self.scope_deadline_added.notified().await;
1372 continue;
1373 }
1374 tokio::time::sleep(Duration::from_secs(1)).await;
1375 if let Err(error) = self.sweep_expired_scopes() {
1376 warn!(%error, "scope expiry sweep failed");
1377 }
1378 }
1379 });
1380 }
1381
1382 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1383 let mut table = self
1384 .scopes
1385 .write()
1386 .unwrap_or_else(|poisoned| poisoned.into_inner());
1387 let swept = table.sweep_expired();
1388 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1389 drop(table);
1390 self.close_scope_drained_routes(drained);
1391 Ok(())
1392 }
1393
1394 fn publish_scope_changes(
1395 &self,
1396 changes: &[crate::scopes::ScopeTagChange],
1397 expired: &[crate::scopes::ScopeExpired],
1398 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1399 if changes.is_empty() {
1400 return Ok(Vec::new());
1401 }
1402 let drained = self
1403 .forwarding
1404 .publish_scope_changes(changes)
1405 .map_err(RouterError::Forwarding)?;
1406 let mut counts = HashMap::new();
1407 for route in &drained {
1408 *counts
1409 .entry((
1410 route.scope.owner.as_str(),
1411 route.scope.scope_ref.as_str(),
1412 route.scope.tag.scope_epoch,
1413 ))
1414 .or_insert(0usize) += 1;
1415 }
1416 for scope in expired {
1417 let routes_closed = counts
1418 .get(&(
1419 scope.owner.as_str(),
1420 scope.scope_ref.as_str(),
1421 scope.scope_epoch,
1422 ))
1423 .copied()
1424 .unwrap_or(0);
1425 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1426 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1427 routes_closed, "scope expired");
1428 }
1429 Ok(drained)
1430 }
1431
1432 fn runtime_capability_snapshot(
1433 &self,
1434 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1435 let runtime = self
1436 .supervisor
1437 .list()
1438 .into_iter()
1439 .map(|module| {
1440 let status = module.status().map_err(|err| {
1441 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1442 })?;
1443 Ok(RuntimeModule {
1444 module_id: status.module_id,
1445 state: status.state,
1446 enabled: status.enabled,
1447 })
1448 })
1449 .collect::<Result<Vec<_>, RouterError>>()?;
1450 let registrations = self.registry.list_registered_modules().map_err(|err| {
1456 RouterError::backend(
1457 0,
1458 0,
1459 format!("failed to list capability registrations: {err}"),
1460 )
1461 })?;
1462 let registrations = registrations
1463 .into_iter()
1464 .map(|registration| RegisteredModule {
1465 module_id: registration.manifest.module_id,
1466 module_version: registration.manifest.module_version,
1467 capabilities: registration.manifest.capabilities,
1468 })
1469 .collect();
1470 Ok((runtime, registrations))
1471 }
1472
1473 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1480 let cached_registration = RegisteredModule {
1481 module_id: registration.manifest.module_id.clone(),
1482 module_version: registration.manifest.module_version.clone(),
1483 capabilities: registration.manifest.capabilities.clone(),
1484 };
1485 let change = self.capability_evaluator.record_hello(&cached_registration);
1486 log_manifest_change(&cached_registration.module_id, &change);
1487 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1488 self.enforce_capability_denies();
1489 }
1490 self.refresh_capability_requirements();
1491 }
1492
1493 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1497 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1498 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1499 self.supervisor.set_swap_promotion_observer(observer);
1500 }
1501
1502 pub fn refresh_capability_requirements(&self) {
1503 match self.runtime_capability_snapshot() {
1504 Ok((runtime, registrations)) => {
1505 log_requirement_events(
1506 self.capability_evaluator
1507 .evaluate_now(&runtime, ®istrations),
1508 &self.daemon_shutdown,
1509 );
1510 }
1511 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1512 }
1513 }
1514
1515 fn enforce_capability_denies(&self) {
1519 let registrations = match self.registry.list_registered_modules() {
1520 Ok(snapshot) => snapshot,
1521 Err(err) => {
1522 warn!(error = %err, "failed to read registrations for capability deny census");
1523 return;
1524 }
1525 };
1526 let manifests = registrations
1527 .into_iter()
1528 .map(|registration| {
1529 (
1530 registration.manifest.module_id.clone(),
1531 registration.manifest,
1532 )
1533 })
1534 .collect::<BTreeMap<_, _>>();
1535 let census = match self.forwarding.route_census(None) {
1536 Ok(census) => census,
1537 Err(err) => {
1538 warn!(error = %err, "failed to read route census for capability deny enforcement");
1539 return;
1540 }
1541 };
1542
1543 for (target_module_id, routes) in census {
1544 let Some(target_manifest) = manifests.get(&target_module_id) else {
1545 continue;
1546 };
1547 let mut closed_routes = Vec::new();
1548 let mut goodbyes = Vec::new();
1549 for route in routes {
1550 let Principal::Reserved {
1551 module_id: opening_module_id,
1552 } = &route.principal
1553 else {
1554 continue;
1555 };
1556 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1557 continue;
1558 };
1559 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1560 continue;
1561 };
1562
1563 match self.forwarding.release_client_route(
1564 route.goodbye_target.connection_id,
1565 route.goodbye_target.channel,
1566 route.goodbye_target.epoch,
1567 ) {
1568 Ok(RouteRelease::Removed(module_goodbye)) => {
1569 warn!(
1570 opening_module_id,
1571 target_module_id,
1572 capability,
1573 "force-closing route because an attested capability deny edge now matches"
1574 );
1575 goodbyes.push(module_goodbye);
1576 goodbyes.push(route.goodbye_target.clone());
1577 closed_routes.push(route);
1578 }
1579 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1580 Err(err) => warn!(
1581 opening_module_id,
1582 target_module_id,
1583 capability,
1584 error = %err,
1585 "failed to force-close capability-denied route"
1586 ),
1587 }
1588 }
1589
1590 if closed_routes.is_empty() {
1591 continue;
1592 }
1593 send_route_control_pushes(
1594 &self.forwarding,
1595 closed_routes,
1596 ClientControlPush::RouteClosed {
1597 module_id: target_module_id,
1598 channels: Vec::new(),
1599 reason: RouteCloseReason::CapabilityDenied,
1600 drained: false,
1601 abandoned: 0,
1602 excluded_subscriptions: 0,
1603 terminal: Some(false),
1604 },
1605 );
1606 self.emit_route_goodbyes(goodbyes);
1609 }
1610 }
1611
1612 fn not_ready_reason(
1618 &self,
1619 registration: &crate::registry::ModuleRegistration,
1620 ) -> Option<NotReadyReason> {
1621 if !registration.ready {
1622 return Some(NotReadyReason {
1623 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1624 capability: None,
1625 });
1626 }
1627 self.first_unprovided_required_capability(registration)
1628 .map(|capability| NotReadyReason {
1629 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1630 capability: Some(capability),
1631 })
1632 }
1633
1634 fn first_unprovided_required_capability(
1652 &self,
1653 registration: &crate::registry::ModuleRegistration,
1654 ) -> Option<String> {
1655 let required = registration
1656 .manifest
1657 .capabilities
1658 .iter()
1659 .flat_map(|declarations| declarations.requires.iter())
1660 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1661 .map(|requirement| requirement.capability.as_str())
1662 .collect::<BTreeSet<_>>();
1663 if required.is_empty() {
1664 return None;
1665 }
1666 let module_id = registration.manifest.module_id.as_str();
1667 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1668 if required
1669 .iter()
1670 .any(|capability| verdict(capability).is_none())
1671 {
1672 self.refresh_capability_requirements();
1673 }
1674 required
1675 .into_iter()
1676 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1677 .map(str::to_string)
1678 }
1679
1680 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1681 self.capability_evaluator
1682 .statuses()
1683 .into_iter()
1684 .map(capability_requirement_status)
1685 .collect()
1686 }
1687
1688 fn deregister_connection(
1695 &self,
1696 connection_id: ConnectionId,
1697 reason: RegistrationEndReason,
1698 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1699 self.registry
1700 .deregister_connection_with_reason(connection_id, reason)
1701 }
1702
1703 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1704 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1705 return None;
1706 }
1707 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1708 parse_client_control_request(&frame.body)
1709 else {
1710 return None;
1711 };
1712 Some(target_module_id(&target).to_string())
1713 }
1714
1715 pub(crate) fn route_open_capacity_refusal(
1716 &self,
1717 ctx: &RouteCtx,
1718 frame: &Frame,
1719 target_module_id: &str,
1720 in_flight: usize,
1721 limit: usize,
1722 ) -> Result<Frame, RouterError> {
1723 let caller = self.route_open_caller_from_frame(frame);
1726 self.route_open_admission_refusal_frame(
1727 ctx,
1728 &caller,
1729 frame,
1730 target_module_id,
1731 "open_admission_full",
1732 (in_flight, limit),
1733 format!(
1734 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1735 ),
1736 )
1737 }
1738
1739 fn route_open_target_capacity_refusal(
1740 &self,
1741 ctx: &RouteCtx,
1742 caller: &RouteOpenCaller,
1743 frame: &Frame,
1744 target_module_id: &str,
1745 in_flight: usize,
1746 ) -> Result<Frame, RouterError> {
1747 self.route_open_admission_refusal_frame(
1748 ctx,
1749 caller,
1750 frame,
1751 target_module_id,
1752 "target_binds_full",
1753 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1754 format!(
1755 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1756 ),
1757 )
1758 }
1759
1760 #[allow(clippy::too_many_arguments)]
1768 fn route_open_admission_refusal_frame(
1769 &self,
1770 ctx: &RouteCtx,
1771 caller: &RouteOpenCaller,
1772 frame: &Frame,
1773 target_module_id: &str,
1774 reason: &'static str,
1775 (in_flight, limit): (usize, usize),
1776 message: impl Into<String>,
1777 ) -> Result<Frame, RouterError> {
1778 let code = error_codes::TARGET_UNAVAILABLE;
1779 self.counters.increment_route_open_refused(code);
1780 info!(
1781 target: "control",
1782 code,
1783 reason,
1784 module_id = ?target_module_id,
1785 connection_id = ctx.connection_id.get(),
1786 principal = %caller.principal,
1787 scope = %caller.scope,
1788 harness = %caller.harness,
1789 in_flight,
1790 limit,
1791 "route.open refused"
1792 );
1793 control_error_frame(frame, code, message.into())
1794 }
1795
1796 #[cfg(test)]
1802 pub fn handle_control(
1803 &self,
1804 connection_id: ConnectionId,
1805 frame: Frame,
1806 ) -> Result<Vec<Frame>, RouterError> {
1807 match frame.header.ty {
1808 FrameType::Ping => Ok(vec![pong(&frame)?]),
1809 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1810 FrameType::Goodbye => self.handle_goodbye(connection_id),
1811 ty => Ok(vec![control_error_frame(
1812 &frame,
1813 "unsupported_control_frame",
1814 format!("unsupported channel-0 frame {ty:?}"),
1815 )?]),
1816 }
1817 }
1818
1819 pub async fn handle_control_frame(
1820 &self,
1821 ctx: &RouteCtx,
1822 frame: Frame,
1823 ) -> Result<Vec<Frame>, RouterError> {
1824 self.handle_control_frame_timed(ctx, frame, None).await
1825 }
1826
1827 #[cfg(any(test, feature = "test-support"))]
1828 pub fn with_test_operator_app_policy(
1829 mut self,
1830 policy: crate::operator_apps::TestOperatorAppPolicy,
1831 ) -> Self {
1832 self.operator_app_policy = policy.into();
1833 self
1834 }
1835
1836 #[cfg(all(windows, any(test, feature = "test-support")))]
1840 pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
1841 self.windows_verify = Some(verify);
1842 self
1843 }
1844
1845 #[cfg(all(windows, any(test, feature = "test-support")))]
1847 pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
1848 self.windows_flash = Some(flash);
1849 self
1850 }
1851
1852 #[cfg(all(windows, any(test, feature = "test-support")))]
1856 pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
1857 self.windows_track = Some(track);
1858 self
1859 }
1860
1861 pub(crate) fn record_captured_peer(
1862 &self,
1863 connection: ConnectionId,
1864 peer: subc_os::local_peer::CapturedPeer,
1865 ) {
1866 self.record_observed_peer(connection, peer.observation().clone());
1867 #[cfg(windows)]
1868 if let Some(stamp) = self.stamp_windows_launch(&peer) {
1869 self.operator_apps
1870 .lock()
1871 .unwrap_or_else(|p| p.into_inner())
1872 .insert(connection, stamp);
1873 }
1874 if let Some(stamp) = self.operator_app_policy.stamp(peer) {
1875 self.operator_apps
1876 .lock()
1877 .unwrap_or_else(|p| p.into_inner())
1878 .insert(connection, stamp);
1879 }
1880 }
1881
1882 pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
1883 self.local_connections
1884 .lock()
1885 .unwrap_or_else(|p| p.into_inner())
1886 .insert(connection);
1887 }
1888
1889 fn connection_operator_app(
1892 &self,
1893 stamp: Option<&crate::operator_apps::OperatorAppStamp>,
1894 ) -> Option<subc_protocol::peer::OperatorApp> {
1895 stamp
1896 .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
1897 .map(|stamp| stamp.app.clone())
1898 }
1899
1900 fn handle_connection_describe(
1901 &self,
1902 ctx: &RouteCtx,
1903 frame: Frame,
1904 ) -> Result<Vec<Frame>, RouterError> {
1905 if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
1906 return Ok(vec![control_error_frame(
1907 &frame,
1908 "invalid_control_body",
1909 "malformed connection.describe body",
1910 )?]);
1911 }
1912 let local = self
1913 .local_connections
1914 .lock()
1915 .unwrap_or_else(|p| p.into_inner())
1916 .contains(&ctx.connection_id);
1917 let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
1918 let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
1919 Ok(vec![control_response_body_frame(
1920 &frame,
1921 &subc_control::ConnectionDescribeReply::new(local, operator_app),
1922 ops::CONNECTION_DESCRIBE,
1923 )?])
1924 }
1925
1926 pub(crate) fn record_observed_peer(
1927 &self,
1928 connection: ConnectionId,
1929 peer: subc_os::local_peer::ObservedPeer,
1930 ) {
1931 use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
1932 let peer = match peer {
1933 OsPeer::Unobserved => ObservedPeer::unobserved(),
1934 OsPeer::Observed(peer) => {
1935 let observed = match peer.user {
1936 PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
1937 PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
1938 };
1939 observed
1940 .with_exe(
1941 peer.exe
1942 .and_then(|path| path.into_os_string().into_string().ok()),
1943 )
1944 .with_signing_id(peer.signing_id)
1945 }
1946 };
1947 self.observed_peers
1948 .lock()
1949 .unwrap_or_else(|p| p.into_inner())
1950 .insert(connection, peer);
1951 }
1952
1953 pub(crate) fn record_caller_endpoints(
1954 &self,
1955 connection: ConnectionId,
1956 endpoints: Option<(SocketAddr, SocketAddr)>,
1957 ) {
1958 let mut callers = self
1959 .caller_endpoints
1960 .lock()
1961 .unwrap_or_else(|p| p.into_inner());
1962 callers.remove(&connection);
1963 if let Some(endpoints) = endpoints {
1964 callers.insert(connection, endpoints);
1965 }
1966 }
1967
1968 #[cfg(test)]
1969 pub(crate) fn with_caller_lookup_for_test(
1970 mut self,
1971 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1972 ) -> Self {
1973 self.caller_lookup = Some(Arc::new(lookup));
1974 self
1975 }
1976
1977 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1978 let endpoints = self
1979 .caller_endpoints
1980 .lock()
1981 .unwrap_or_else(|p| p.into_inner())
1982 .get(&connection)
1983 .copied();
1984 #[cfg(test)]
1985 let lookup = self.caller_lookup.clone();
1986 tokio::task::spawn_blocking(move || {
1987 #[cfg(test)]
1988 if let Some(lookup) = lookup {
1989 return lookup(endpoints);
1990 }
1991 endpoints
1992 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1993 .unwrap_or_default()
1994 })
1995 .await
1996 .unwrap_or_default()
1997 }
1998
1999 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
2000 frame.header.channel == 0
2001 && frame.header.ty == FrameType::Request
2002 && self
2003 .forwarding
2004 .module_endpoint_for_connection(ctx.connection_id)
2005 .is_ok_and(|endpoint| endpoint.is_none())
2006 && parse_client_control_request(&frame.body)
2007 .ok()
2008 .is_some_and(|request| privileged_operation(&request).is_some())
2009 }
2010
2011 pub(crate) async fn handle_control_frame_timed(
2012 &self,
2013 ctx: &RouteCtx,
2014 frame: Frame,
2015 dispatch_started_at: Option<StdInstant>,
2016 ) -> Result<Vec<Frame>, RouterError> {
2017 match frame.header.ty {
2018 FrameType::Ping => Ok(vec![pong(&frame)?]),
2019 FrameType::Hello => {
2020 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
2021 }
2022 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2023 FrameType::Cancel => {
2024 if self
2027 .forwarding
2028 .operator_confirms()
2029 .cancel(ctx.connection_id, frame.header.corr)
2030 || self
2031 .supervisor
2032 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2033 || self.cancel_activity_subscription(ctx, &frame)?
2034 {
2035 Ok(Vec::new())
2036 } else {
2037 Ok(vec![control_error_frame(
2038 &frame,
2039 "unknown_subscription",
2040 "no subscription has this correlation id",
2041 )?])
2042 }
2043 }
2044 FrameType::Request => {
2045 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2048 if op
2049 .as_ref()
2050 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2051 {
2052 return self.handle_activity_record(ctx, frame).await;
2053 }
2054 if op.as_ref().is_some_and(|probe| {
2055 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2056 }) {
2057 return self.handle_activity_read(ctx, frame).await;
2058 }
2059 if op
2060 .as_ref()
2061 .is_some_and(|probe| probe.op == "operator.confirm")
2062 {
2063 return self.handle_operator_confirm(ctx, frame);
2064 }
2065 if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2066 return self
2067 .observe_control_wait(
2068 ctx,
2069 None,
2070 ops::APP_OPEN,
2071 self.handle_app_open(ctx, frame),
2072 )
2073 .await;
2074 }
2075 if op
2076 .as_ref()
2077 .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2078 {
2079 return self.handle_connection_describe(ctx, frame);
2080 }
2081 if op
2082 .as_ref()
2083 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2084 {
2085 return self.handle_scope_served_by(ctx.connection_id, frame);
2086 }
2087 if self
2088 .forwarding
2089 .module_endpoint_for_connection(ctx.connection_id)
2090 .map_err(RouterError::Forwarding)?
2091 .is_some()
2092 {
2093 if !is_known_module_request_op(&frame.body) {
2094 return Ok(vec![control_error_frame(
2095 &frame,
2096 "unsupported_control_frame",
2097 "module-originated channel-0 REQUEST is not supported",
2098 )?]);
2099 }
2100 let request = match parse_module_control_request_from_module(&frame.body) {
2101 Ok(request) => request,
2102 Err((err, ControlRequestBodyError::UnknownOp)) => {
2103 return Ok(vec![control_error_frame(
2104 &frame,
2105 "unsupported_control_frame",
2106 format!("unsupported module-originated channel-0 REQUEST: {err}"),
2107 )?])
2108 }
2109 Err((err, ControlRequestBodyError::InvalidBody)) => {
2110 return Ok(vec![control_error_frame(
2111 &frame,
2112 "invalid_control_body",
2113 format!("malformed module control body: {err}"),
2114 )?])
2115 }
2116 };
2117 let op = module_control_request_op(&request);
2118 let corr = frame.header.corr;
2119 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2120 let result =
2121 self.handle_module_control_request(ctx.connection_id, frame, request);
2122 log_slow_control_dispatch(
2123 dispatch_started_at,
2124 op,
2125 ctx.connection_id,
2126 corr,
2127 None,
2128 );
2129 return result;
2130 }
2131
2132 if is_known_module_request_op(&frame.body) {
2133 return Ok(vec![control_error_frame(
2134 &frame,
2135 "not_registered",
2136 "catalog.update requires an active module registration owned by this connection",
2137 )?]);
2138 }
2139
2140 let request = match parse_client_control_request(&frame.body) {
2141 Ok(request) => request,
2142 Err((err, ControlRequestBodyError::UnknownOp)) => {
2143 return Ok(vec![control_error_frame(
2144 &frame,
2145 "unknown_control_op",
2146 format!("unknown client control op: {err}"),
2147 )?])
2148 }
2149 Err((err, ControlRequestBodyError::InvalidBody)) => {
2150 return Ok(vec![control_error_frame(
2151 &frame,
2152 "invalid_control_body",
2153 format!("malformed client control body: {err}"),
2154 )?])
2155 }
2156 };
2157 let op = client_control_request_op(&request);
2158 let corr = frame.header.corr;
2159 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2160 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2161 #[cfg(test)]
2162 if let Some(delay) = self.control_dispatch_delay {
2163 tokio::time::sleep(delay).await;
2164 }
2165 let result = self
2166 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2167 .await;
2168 if let Some(timings) = route_open_timings.as_mut() {
2169 timings.finish();
2170 }
2171 log_slow_control_dispatch(
2172 dispatch_started_at,
2173 op,
2174 ctx.connection_id,
2175 corr,
2176 route_open_timings.as_ref(),
2177 );
2178 result
2179 }
2180 FrameType::Push => {
2181 let Some(endpoint) = self
2182 .forwarding
2183 .module_endpoint_for_connection(ctx.connection_id)
2184 .map_err(RouterError::Forwarding)?
2185 else {
2186 return Ok(vec![control_error_frame(
2187 &frame,
2188 "unsupported_control_frame",
2189 "client-originated channel-0 PUSH is not supported",
2190 )?]);
2191 };
2192 self.handle_status_update(endpoint, frame)
2193 }
2194 FrameType::Response | FrameType::Error
2195 if self
2196 .forwarding
2197 .module_endpoint_for_connection(ctx.connection_id)
2198 .map_err(RouterError::Forwarding)?
2199 .is_some() =>
2200 {
2201 self.handle_module_relay_response(ctx.connection_id, frame)
2202 }
2203 ty => Ok(vec![control_error_frame(
2204 &frame,
2205 "unsupported_control_frame",
2206 format!("unsupported channel-0 frame {ty:?}"),
2207 )?]),
2208 }
2209 }
2210
2211 pub fn cleanup_connection(
2212 &self,
2213 connection_id: ConnectionId,
2214 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2215 self.local_connections
2216 .lock()
2217 .unwrap_or_else(|p| p.into_inner())
2218 .remove(&connection_id);
2219 self.caller_endpoints
2220 .lock()
2221 .unwrap_or_else(|p| p.into_inner())
2222 .remove(&connection_id);
2223 self.build_class_warned
2224 .lock()
2225 .unwrap_or_else(|p| p.into_inner())
2226 .remove(&connection_id);
2227 self.observed_peers
2228 .lock()
2229 .unwrap_or_else(|p| p.into_inner())
2230 .remove(&connection_id);
2231 self.operator_apps
2232 .lock()
2233 .unwrap_or_else(|p| p.into_inner())
2234 .remove(&connection_id);
2235 self.forwarding
2236 .operator_confirms()
2237 .connection_closed(connection_id);
2238 self.cleanup_connection_with_end_reason(
2239 connection_id,
2240 RegistrationEndReason::ConnectionClosed,
2241 )
2242 }
2243
2244 fn cleanup_connection_with_end_reason(
2245 &self,
2246 connection_id: ConnectionId,
2247 requested_reason: RegistrationEndReason,
2248 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2249 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2250 self.registry
2251 .get_module_by_connection(connection_id)?
2252 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2253 .and_then(|module| module.registration_end_reason().ok().flatten())
2254 .unwrap_or(requested_reason)
2255 } else {
2256 requested_reason
2257 };
2258 let crash_closed = self
2259 .registry
2260 .get_module_by_connection(connection_id)?
2261 .and_then(|registration| {
2262 self.forwarding
2263 .module_endpoint_for_connection(connection_id)
2264 .ok()
2265 .flatten()
2266 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2267 .map(|routes| (registration.manifest.module_id, routes))
2268 });
2269 let crash_closed = crash_closed.map(|(module_id, routes)| {
2270 let terminal = match self.supervisor.get(&module_id) {
2271 None => false,
2272 Some(module) => match module.will_recover_after_connection_loss() {
2273 Ok(will_recover) => !will_recover,
2274 Err(err) => {
2275 warn!(
2276 %module_id,
2277 error = %err,
2278 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2279 );
2280 false
2281 }
2282 },
2283 };
2284 let (reason, drained) = match self
2288 .forwarding
2289 .daemon_drain_result_for_connection(connection_id)
2290 {
2291 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2292 Ok(None) => (RouteCloseReason::Crash, false),
2293 Err(err) => {
2294 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2295 (RouteCloseReason::Crash, false)
2296 }
2297 };
2298 (module_id, routes, reason, drained, terminal)
2299 });
2300 let registrations = self.deregister_connection(connection_id, end_reason);
2301 let cleanup = if crash_closed.is_some() {
2302 self.forwarding.cleanup_connection_counted(connection_id)
2303 } else {
2304 self.forwarding
2310 .cleanup_connection(connection_id)
2311 .map(|released| crate::forwarding::ConnectionCleanup {
2312 released,
2313 abandoned_relays: 0,
2314 })
2315 };
2316 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2321 let abandoned = cleanup
2322 .as_ref()
2323 .map_or(0, |cleanup| cleanup.abandoned_relays);
2324 send_route_control_pushes(
2325 &self.forwarding,
2326 routes,
2327 ClientControlPush::RouteClosed {
2328 module_id,
2329 channels: Vec::new(),
2330 reason,
2331 drained,
2332 abandoned,
2333 excluded_subscriptions: 0,
2334 terminal: Some(terminal),
2335 },
2336 );
2337 }
2338 if let Ok(cleanup) = cleanup {
2339 self.emit_route_goodbyes(cleanup.released);
2340 }
2341 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2345 crate::supervise::notify_registration_release();
2346 self.capability_evaluator.wake_deadline_loop();
2347 self.refresh_capability_requirements();
2348 }
2349 self.supervisor.remove_spawn_subscribers(connection_id);
2350 self.activity_streams
2351 .lock()
2352 .unwrap_or_else(|p| p.into_inner())
2353 .retain(|(id, _), task| {
2354 if *id == connection_id {
2355 task.abort();
2356 false
2357 } else {
2358 true
2359 }
2360 });
2361 self.hello_launch_nonces
2364 .lock()
2365 .unwrap_or_else(|poisoned| poisoned.into_inner())
2366 .forget(connection_id);
2367 self.scopes
2368 .write()
2369 .unwrap_or_else(|poisoned| poisoned.into_inner())
2370 .release_connection(connection_id);
2371 registrations
2372 }
2373
2374 pub(crate) fn handle_route_goodbye(
2375 &self,
2376 connection_id: ConnectionId,
2377 route_channel: u16,
2378 route_epoch: u32,
2379 ) -> Result<bool, RouterError> {
2380 debug!(
2381 connection_id = connection_id.get(),
2382 route_channel, route_epoch, "handling route GOODBYE"
2383 );
2384 let RouteRelease::Removed(released_route) = self
2385 .forwarding
2386 .release_client_route(connection_id, route_channel, route_epoch)
2387 .map_err(RouterError::Forwarding)?
2388 else {
2389 return Ok(false);
2390 };
2391 self.emit_route_goodbyes(vec![released_route]);
2392 Ok(true)
2393 }
2394
2395 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2396 for released in released_routes {
2397 let frame = match Frame::build_with_version(
2398 released.negotiated_ver,
2399 FrameType::Goodbye,
2400 control_flags(),
2401 released.channel,
2402 released.epoch,
2403 0,
2404 Vec::new(),
2405 ) {
2406 Ok(frame) => frame,
2407 Err(err) => {
2408 warn!(
2409 route_channel = released.channel,
2410 error = %err,
2411 "failed to build route GOODBYE frame"
2412 );
2413 continue;
2414 }
2415 };
2416 if !released.close_on_delivery_failure() {
2417 crate::forwarding::send_module_route_goodbye(
2418 &self.counters,
2419 &released.sink,
2420 frame,
2421 released.module_id.as_deref(),
2422 "client route released",
2423 );
2424 continue;
2425 }
2426 if let Err(err) = released.sink.try_send(frame) {
2427 warn!(
2428 target_connection_id = released.connection_id.get(),
2429 route_channel = released.channel,
2430 error = %err,
2431 "route GOODBYE was not delivered to client; closing target connection"
2432 );
2433 if self
2434 .forwarding
2435 .escalate_client_delivery_failure(
2436 released.connection_id,
2437 released.channel,
2438 released.epoch,
2439 CloseReason::new(
2440 "route_goodbye_delivery_failed",
2441 format!(
2442 "failed to enqueue route GOODBYE for channel {}: {err}",
2443 released.channel
2444 ),
2445 ),
2446 crate::forwarding::UndeliveredFrame {
2447 module_id: released.module_id.as_deref(),
2448 sink: &released.sink,
2449 },
2450 )
2451 .unwrap_or(false)
2452 {
2453 self.counters.increment_goodbye_relay_client_failed();
2454 }
2455 }
2456 }
2457 }
2458
2459 fn send_abandoned_route_bind_goodbye(
2471 &self,
2472 module_sink: &crate::FrameSink,
2473 negotiated_ver: u8,
2474 module_channel: u16,
2475 module_epoch: u32,
2476 ) {
2477 let frame = match Frame::build_with_version(
2478 negotiated_ver,
2479 FrameType::Goodbye,
2480 control_flags(),
2481 module_channel,
2482 module_epoch,
2483 0,
2484 Vec::new(),
2485 ) {
2486 Ok(frame) => frame,
2487 Err(err) => {
2488 warn!(
2489 route_channel = module_channel,
2490 error = %err,
2491 "failed to build GOODBYE for abandoned route.bind"
2492 );
2493 return;
2494 }
2495 };
2496 crate::forwarding::send_module_route_goodbye(
2497 &self.counters,
2498 module_sink,
2499 frame,
2500 None,
2501 "abandoned route.bind",
2502 );
2503 }
2504
2505 fn handle_hello(
2506 &self,
2507 connection_id: ConnectionId,
2508 sink: Option<crate::FrameSink>,
2509 frame: Frame,
2510 ) -> Result<Vec<Frame>, RouterError> {
2511 debug!(
2512 connection_id = connection_id.get(),
2513 corr = frame.header.corr,
2514 "handling HELLO"
2515 );
2516 if self
2520 .registry
2521 .get_module_by_connection(connection_id)
2522 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2523 .is_some()
2524 {
2525 return Ok(vec![control_error_frame(
2526 &frame,
2527 "invalid_hello",
2528 "connection is already registered as a module",
2529 )?]);
2530 }
2531 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2532 Ok(value) => value,
2533 Err(err) => {
2534 return Ok(vec![control_error_frame(
2535 &frame,
2536 "invalid_hello",
2537 format!("malformed HELLO body: {err}"),
2538 )?])
2539 }
2540 };
2541 if let Err(err) = validate_hello_display_name(&hello_value) {
2542 return Ok(vec![control_error_frame(
2543 &frame,
2544 "invalid_manifest",
2545 err.to_string(),
2546 )?]);
2547 }
2548 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2549 return Ok(vec![control_error_frame(
2550 &frame,
2551 "invalid_capability_grammar",
2552 err.to_string(),
2553 )?]);
2554 }
2555 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2556 return Ok(vec![control_error_frame(
2557 &frame,
2558 "invalid_manifest",
2559 err.to_string(),
2560 )?]);
2561 }
2562 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2563 return Ok(vec![control_error_frame(
2564 &frame,
2565 "invalid_event_declaration",
2566 err.to_string(),
2567 )?]);
2568 }
2569 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2570 return Ok(vec![control_error_frame(
2571 &frame,
2572 "invalid_manifest",
2573 err.to_string(),
2574 )?]);
2575 }
2576 if let Some(provenance) = hello_value
2577 .get("manifest")
2578 .and_then(|manifest| manifest.get("provenance"))
2579 {
2580 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2581 return Ok(vec![control_error_frame(
2582 &frame,
2583 "invalid_manifest",
2584 format!("malformed manifest provenance: {err}"),
2585 )?]);
2586 }
2587 }
2588 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2589 let module_id = hello_value["manifest"]["module_id"]
2590 .as_str()
2591 .unwrap_or("<unknown>");
2592 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2593 }
2594 if let Err(reason) = validate_hello_flow_operations(&hello_value) {
2595 let module_id = hello_value["manifest"]["module_id"]
2596 .as_str()
2597 .unwrap_or("<unknown>");
2598 warn!(module_id, %reason, "dropping flow_operations from HELLO");
2599 }
2600 if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
2601 let module_id = hello_value["manifest"]["module_id"]
2602 .as_str()
2603 .unwrap_or("<unknown>");
2604 warn!(module_id, %reason, "dropping confirm_operations from HELLO");
2605 }
2606 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2607 Ok(hello) => hello,
2608 Err(err) => {
2609 return Ok(vec![control_error_frame(
2610 &frame,
2611 "invalid_hello",
2612 format!("malformed HELLO body: {err}"),
2613 )?])
2614 }
2615 };
2616
2617 if hello.protocol_ver != hello.manifest.protocol_ver {
2618 return Ok(vec![control_error_frame(
2619 &frame,
2620 "invalid_manifest",
2621 format!(
2622 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2623 hello.protocol_ver, hello.manifest.protocol_ver
2624 ),
2625 )?]);
2626 }
2627
2628 if hello.manifest.module_id.trim().is_empty() {
2629 return Ok(vec![control_error_frame(
2630 &frame,
2631 "invalid_manifest",
2632 "manifest module_id must not be empty",
2633 )?]);
2634 }
2635
2636 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2637 Ok(negotiated_ver) => negotiated_ver,
2638 Err(message) => {
2639 return Ok(vec![control_error_frame(
2640 &frame,
2641 "version_unsupported",
2642 message,
2643 )?])
2644 }
2645 };
2646
2647 let swap_admission = self
2655 .supervisor
2656 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2657 if swap_admission == SwapHelloAdmission::Refused {
2658 warn!(
2659 module_id = %hello.manifest.module_id,
2660 connection_id = connection_id.get(),
2661 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2662 );
2663 return Ok(vec![control_error_frame(
2664 &frame,
2665 "swap_token_invalid",
2666 format!(
2667 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2668 hello.manifest.module_id
2669 ),
2670 )?]);
2671 }
2672 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2673
2674 let nonce_verified = if swap_candidate {
2679 true
2680 } else {
2681 match self.supervisor.reserved_hello_verification(
2682 &hello.manifest.module_id,
2683 hello.launch_nonce.as_deref(),
2684 ) {
2685 Ok(verified) => verified,
2686 Err(rejection) => {
2687 let message = match rejection {
2688 ReservedHelloRejection::Exact { module_id } => format!(
2689 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2690 ),
2691 ReservedHelloRejection::Prefix {
2692 prefix,
2693 owner_module_id,
2694 } => format!(
2695 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2696 hello.manifest.module_id
2697 ),
2698 };
2699 return Ok(vec![control_error_frame(
2700 &frame,
2701 "reserved_module",
2702 message,
2703 )?]);
2704 }
2705 }
2706 };
2707
2708 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2709 &hello.manifest.module_id,
2710 hello.manifest.capabilities.as_ref(),
2711 );
2712 if let Some(refusal) = reserved_capability_refusals.first() {
2713 let capability = refusal.capability.clone();
2714 let bound_module = refusal.claimants[0].clone();
2715 log_duplicate_claim_events(reserved_capability_refusals);
2716 return Ok(vec![control_error_frame(
2717 &frame,
2718 "reserved_capability",
2719 format!(
2720 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2721 capability, bound_module, hello.manifest.module_id
2722 ),
2723 )?]);
2724 }
2725
2726 if self
2729 .forwarding
2730 .connection_has_client_routes(connection_id)
2731 .map_err(RouterError::Forwarding)?
2732 {
2733 return Ok(vec![control_error_frame(
2734 &frame,
2735 "invalid_hello",
2736 "connection has open client routes and cannot also register as a module",
2737 )?]);
2738 }
2739
2740 self.hello_launch_nonces
2745 .lock()
2746 .unwrap_or_else(|poisoned| poisoned.into_inner())
2747 .record(connection_id, hello.launch_nonce.as_deref());
2748 let control_ops = effective_module_control_ops(hello.control_ops);
2749 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2752 if swap_candidate {
2753 return self.register_swap_candidate(
2754 connection_id,
2755 sink,
2756 &frame,
2757 hello.manifest,
2758 negotiated_ver,
2759 control_ops,
2760 hello_ack,
2761 );
2762 }
2763 let registration = match self
2770 .registry
2771 .register_unpublished_verified_with_control_ops(
2772 hello.manifest,
2773 negotiated_ver,
2774 connection_id,
2775 control_ops,
2776 nonce_verified,
2777 ) {
2778 Ok(registration) => registration,
2779 Err(RegistryError::DuplicateModuleId { module_id }) => {
2780 return Ok(vec![control_error_frame(
2781 &frame,
2782 "duplicate_module_id",
2783 format!(
2784 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2785 ),
2786 )?])
2787 }
2788 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2789 return Ok(vec![control_error_frame(
2790 &frame,
2791 "invalid_module_id",
2792 err.to_string(),
2793 )?])
2794 }
2795 Err(err) => {
2796 return Ok(vec![control_error_frame(
2797 &frame,
2798 "registry_error",
2799 err.to_string(),
2800 )?])
2801 }
2802 };
2803 #[cfg(test)]
2804 self.hold_between_registry_and_forwarding();
2805
2806 let reply = if let Some(sink) = sink {
2807 let concurrency = manifest_concurrency(®istration.manifest);
2823 if let Err(err) = self.forwarding.register_module_connection_acked(
2824 connection_id,
2825 registration.manifest.module_id.clone(),
2826 negotiated_ver,
2827 concurrency,
2828 sink,
2829 hello_ack,
2830 ) {
2831 if matches!(
2836 self.deregister_connection(
2837 connection_id,
2838 RegistrationEndReason::RegistrationFailed,
2839 ),
2840 Ok(r) if !r.is_empty()
2841 ) {
2842 crate::supervise::notify_registration_release();
2843 }
2844 return Ok(vec![control_error_frame(
2845 &frame,
2846 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2847 "invalid_hello"
2848 } else {
2849 forwarding_error_code(&err)
2850 },
2851 err.to_string(),
2852 )?]);
2853 }
2854 Vec::new()
2855 } else {
2856 vec![hello_ack]
2859 };
2860
2861 if !self
2866 .registry
2867 .publish_registration(connection_id)
2868 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2869 {
2870 debug!(
2871 module_id = %registration.manifest.module_id,
2872 connection_id = connection_id.get(),
2873 "registration was removed before it could be published"
2874 );
2875 }
2876
2877 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2886 info!(
2887 module_id = %registration.manifest.module_id,
2888 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2889 );
2890 }
2891
2892 self.apply_registration_capabilities(®istration);
2893
2894 self.supervisor
2895 .record_activity_registration(®istration.manifest.module_id);
2896
2897 info!(
2898 module_id = %registration.manifest.module_id,
2899 module_version = %registration.manifest.module_version,
2900 negotiated_ver,
2901 routable_provider = manifest_provides_routable_role(®istration.manifest),
2902 connection_id = connection_id.get(),
2903 "module registered"
2904 );
2905
2906 Ok(reply)
2907 }
2908
2909 #[allow(clippy::too_many_arguments)]
2926 fn register_swap_candidate(
2927 &self,
2928 connection_id: ConnectionId,
2929 sink: Option<crate::FrameSink>,
2930 frame: &Frame,
2931 manifest: ModuleManifest,
2932 negotiated_ver: u8,
2933 control_ops: Vec<String>,
2934 hello_ack: Frame,
2935 ) -> Result<Vec<Frame>, RouterError> {
2936 let module_id = manifest.module_id.clone();
2937 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2938 manifest,
2939 negotiated_ver,
2940 connection_id,
2941 control_ops,
2942 ) {
2943 Ok(registration) => registration,
2944 Err(RegistryError::DuplicateModuleId { module_id }) => {
2945 return Ok(vec![control_error_frame(
2946 frame,
2947 "duplicate_module_id",
2948 format!(
2949 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2950 ),
2951 )?])
2952 }
2953 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2954 return Ok(vec![control_error_frame(
2955 frame,
2956 "invalid_module_id",
2957 err.to_string(),
2958 )?])
2959 }
2960 Err(err) => {
2961 return Ok(vec![control_error_frame(
2962 frame,
2963 "registry_error",
2964 err.to_string(),
2965 )?])
2966 }
2967 };
2968 #[cfg(test)]
2969 self.hold_between_registry_and_forwarding();
2970 let reply = if let Some(sink) = sink {
2971 let concurrency = manifest_concurrency(®istration.manifest);
2975 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2976 connection_id,
2977 module_id.clone(),
2978 negotiated_ver,
2979 concurrency,
2980 sink,
2981 hello_ack,
2982 ) {
2983 if matches!(
2984 self.deregister_connection(
2985 connection_id,
2986 RegistrationEndReason::RegistrationFailed,
2987 ),
2988 Ok(r) if !r.is_empty()
2989 ) {
2990 crate::supervise::notify_registration_release();
2991 }
2992 return Ok(vec![control_error_frame(
2993 frame,
2994 forwarding_error_code(&err),
2995 err.to_string(),
2996 )?]);
2997 }
2998 Vec::new()
2999 } else {
3000 vec![hello_ack]
3001 };
3002 if !self
3003 .registry
3004 .publish_registration(connection_id)
3005 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3006 {
3007 debug!(
3008 module_id = %module_id,
3009 connection_id = connection_id.get(),
3010 "swap candidate registration was removed before it could be published"
3011 );
3012 }
3013 self.supervisor.mark_swap_candidate_admitted(&module_id);
3014 info!(
3015 module_id = %module_id,
3016 module_version = %registration.manifest.module_version,
3017 negotiated_ver,
3018 ready = registration.ready,
3019 connection_id = connection_id.get(),
3020 "swap candidate registered; not routable until cutover"
3021 );
3022 Ok(reply)
3023 }
3024
3025 fn build_hello_ack(
3026 &self,
3027 frame: &Frame,
3028 negotiated_ver: u8,
3029 module_id: &str,
3030 ) -> Result<Frame, RouterError> {
3031 let ack = ModuleHelloAckBody {
3032 negotiated_ver,
3033 subc_ops: module_subc_ops(),
3034 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
3035 storage: self
3036 .storage_config
3037 .as_ref()
3038 .map(|cfg| cfg.descriptor_for(module_id)),
3039 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3040 };
3041 let body = serde_json::to_vec(&ack).map_err(|err| {
3042 RouterError::backend(
3043 0,
3044 frame.header.corr,
3045 format!("failed to encode HELLO_ACK: {err}"),
3046 )
3047 })?;
3048
3049 Frame::build_with_version(
3050 negotiated_ver,
3051 FrameType::HelloAck,
3052 control_flags(),
3053 0,
3054 0,
3055 frame.header.corr,
3056 body,
3057 )
3058 .map_err(RouterError::FrameBuild)
3059 }
3060
3061 async fn handle_client_control_request(
3062 &self,
3063 ctx: &RouteCtx,
3064 frame: Frame,
3065 request: ClientControlRequest,
3066 route_open_timings: &mut Option<RouteOpenTimings>,
3067 ) -> Result<Vec<Frame>, RouterError> {
3068 if let Some(operation) = privileged_operation(&request) {
3069 let caller = self.resolve_caller(ctx.connection_id).await;
3070 let op = client_control_request_op(&request);
3071 let text = prompt_text(operation, &caller);
3072 match self.privileged_operator_prompt {
3073 PrivilegedOperatorPrompt::Log => {
3074 info!(target: "subc_daemon::privileged_prompt", operation = op,
3075 target = ?operation, caller = %caller.description(), summary = %text,
3076 "privileged request would prompt the operator");
3077 }
3078 PrivilegedOperatorPrompt::Enforce => {
3079 let key =
3080 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
3081 let stamp = self
3082 .operator_apps
3083 .lock()
3084 .unwrap_or_else(|p| p.into_inner())
3085 .get(&ctx.connection_id)
3086 .cloned();
3087 let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
3088 &self.operator_app_policy,
3089 &self.launch_table,
3090 stamp.as_ref(),
3091 )
3092 .then(|| self.shortcut_context(stamp, None, None));
3093 let outcome = self
3094 .forwarding
3095 .operator_confirms()
3096 .approve_privileged_with_shortcut(
3097 ctx,
3098 &frame,
3099 op,
3100 text,
3101 caller.description(),
3102 key,
3103 shortcut,
3104 )
3105 .await;
3106 if outcome.code != "confirmed" {
3107 return Ok(vec![outcome.frame(&frame)]);
3108 }
3109 }
3110 }
3111 }
3112 match request {
3113 ClientControlRequest::DaemonShutdownIntent { intent } => {
3114 if intent.as_str() != Some("restart") {
3115 return Ok(vec![control_error_frame(
3116 &frame,
3117 "invalid_params",
3118 "intent must be 'restart'",
3119 )?]);
3120 }
3121 let response = ClientControlResponse::DaemonShutdownIntent {
3122 intent: "restart".to_owned(),
3123 expires_at_ms: self.supervisor.announce_daemon_restart(),
3124 };
3125 Ok(vec![control_response_body_frame(
3126 &frame,
3127 &response,
3128 "ClientControlResponse::DaemonShutdownIntent",
3129 )?])
3130 }
3131 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
3132 ClientControlRequest::CatalogList { module_id } => {
3133 self.handle_catalog_list(frame, module_id)
3134 }
3135 ClientControlRequest::RouteOpen {
3136 target,
3137 identity,
3138 consumer_identity,
3139 consumer_capabilities,
3140 role_versions,
3141 busy_refusals,
3142 admission_facts,
3143 scope,
3144 ..
3145 } => {
3146 self.handle_route_open(
3147 ctx,
3148 frame,
3149 RouteOpenRequest {
3150 target,
3151 identity,
3152 consumer_identity,
3153 consumer_capabilities,
3154 role_versions,
3155 busy_refusals,
3156 admission_facts,
3157 scope,
3158 },
3159 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
3160 )
3161 .await
3162 }
3163 ClientControlRequest::RoutePoll {
3164 route_channel,
3165 route_epoch,
3166 kind,
3167 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
3168 ClientControlRequest::SupervisorList {} => {
3169 let handler = self.clone();
3171 let sink = ctx.egress.clone();
3172 tokio::spawn(
3173 async move {
3174 let result = handler.handle_supervisor_list(frame.clone()).await;
3175 let replies = match result {
3176 Ok(replies) => replies,
3177 Err(error) => {
3178 control_error_frame(&frame, "backend_error", error.to_string())
3179 .into_iter()
3180 .collect()
3181 }
3182 };
3183 for reply in replies {
3184 if sink.send(reply).await.is_err() {
3185 break;
3186 }
3187 }
3188 }
3189 .with_current_subscriber(),
3190 );
3191 Ok(Vec::new())
3192 }
3193 ClientControlRequest::SupervisorSpawnSnapshot {} => {
3194 self.handle_supervisor_spawn_snapshot(frame)
3195 }
3196 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
3197 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
3198 }
3199 ClientControlRequest::SupervisorRestart {
3200 module_id,
3201 drain_timeout_ms,
3202 } => {
3203 self.log_supervisor_request_received(
3204 ctx,
3205 frame.header.corr,
3206 ops::SUPERVISOR_RESTART,
3207 Some(&module_id),
3208 None,
3209 )?;
3210 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3211 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
3212 .await
3213 }
3214 ClientControlRequest::SupervisorSwap {
3215 module_id,
3216 ready_timeout_ms,
3217 } => {
3218 self.log_supervisor_request_received(
3219 ctx,
3220 frame.header.corr,
3221 ops::SUPERVISOR_SWAP,
3222 Some(&module_id),
3223 None,
3224 )?;
3225 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3226 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
3227 .await
3228 }
3229 ClientControlRequest::SupervisorReload { module_id } => {
3230 self.log_supervisor_request_received(
3231 ctx,
3232 frame.header.corr,
3233 ops::SUPERVISOR_RELOAD,
3234 Some(&module_id),
3235 None,
3236 )?;
3237 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3238 self.handle_supervisor_reload(frame, module_id, requester)
3239 .await
3240 }
3241 ClientControlRequest::SupervisorRescan { preview } => {
3242 if !preview {
3243 self.log_supervisor_request_received(
3244 ctx,
3245 frame.header.corr,
3246 ops::SUPERVISOR_RESCAN,
3247 None,
3248 None,
3249 )?;
3250 }
3251 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3252 let operation = self.handle_supervisor_rescan(
3253 frame,
3254 if preview {
3255 RescanMode::Preview
3256 } else {
3257 RescanMode::Execute
3258 },
3259 requester,
3260 );
3261 self.observe_control_wait(ctx, None, ops::SUPERVISOR_RESCAN, operation)
3262 .await
3263 }
3264 ClientControlRequest::SupervisorRescanChecked {
3265 preview,
3266 candidate,
3267 expect,
3268 } => {
3269 self.observe_control_wait(
3270 ctx,
3271 None,
3272 ops::SUPERVISOR_RESCAN,
3273 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect),
3274 )
3275 .await
3276 }
3277 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
3278 self.log_supervisor_request_received(
3279 ctx,
3280 frame.header.corr,
3281 ops::SUPERVISOR_RELEASE_RESERVED,
3282 Some(&module_id),
3283 None,
3284 )?;
3285 self.handle_supervisor_release_reserved(frame, module_id)
3286 .await
3287 }
3288 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
3289 self.log_supervisor_request_received(
3290 ctx,
3291 frame.header.corr,
3292 ops::SUPERVISOR_SET_ENABLED,
3293 Some(&module_id),
3294 Some(enabled),
3295 )?;
3296 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3297 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
3298 .await
3299 }
3300 ClientControlRequest::SupervisorHealthProbe { module_id } => {
3301 self.handle_supervisor_health_probe(ctx, frame, module_id)
3302 .await
3303 }
3304 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
3305 ClientControlRequest::SupervisorRoutes { module_id } => {
3306 self.handle_supervisor_routes(frame, module_id)
3307 }
3308 ClientControlRequest::SupervisorProvenance { module_id } => {
3309 self.handle_supervisor_provenance(ctx, frame, module_id)
3310 .await
3311 }
3312 ClientControlRequest::SupervisorStderrTail {
3313 module_id,
3314 max_lines,
3315 max_bytes,
3316 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
3317 ClientControlRequest::SupervisorTerminals { module_id } => {
3318 self.handle_supervisor_terminals(ctx, frame, module_id)
3319 .await
3320 }
3321 }
3322 }
3323
3324 fn log_supervisor_request_received(
3325 &self,
3326 ctx: &RouteCtx,
3327 corr: u64,
3328 op: &'static str,
3329 module_id: Option<&str>,
3330 enabled: Option<bool>,
3331 ) -> Result<(), RouterError> {
3332 let caller = self.supervisor_requester(ctx, corr)?;
3333 let caller = principal_label(&caller);
3334 info!(
3337 target: "control",
3338 op,
3339 module_id,
3340 enabled,
3341 connection_id = ctx.connection_id.get(),
3342 caller = %caller,
3343 "supervisor request received"
3344 );
3345 Ok(())
3346 }
3347
3348 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3349 Ok(self
3350 .registry
3351 .get_module_by_connection(ctx.connection_id)
3352 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3353 .map(|registration| Principal::Reserved {
3354 module_id: registration.manifest.module_id,
3355 })
3356 .unwrap_or(Principal::Direct))
3357 }
3358
3359 fn handle_module_control_request(
3360 &self,
3361 connection_id: ConnectionId,
3362 frame: Frame,
3363 request: ModuleControlRequestFromModule,
3364 ) -> Result<Vec<Frame>, RouterError> {
3365 match request {
3366 ModuleControlRequestFromModule::CatalogUpdate {
3367 provides,
3368 capabilities,
3369 ready,
3370 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3371 ModuleControlRequestFromModule::LiveRoots {} => {
3372 let registered = self
3373 .registry
3374 .get_module_by_connection(connection_id)
3375 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3376 let Some(registration) = registered else {
3377 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3378 };
3379 let response = self
3380 .forwarding
3381 .live_roots(®istration.manifest.module_id)
3382 .map_err(RouterError::Forwarding)?;
3383 Ok(vec![control_response_body_frame(
3384 &frame,
3385 &response,
3386 "ModuleControlResponseToModule::LiveRoots",
3387 )?])
3388 }
3389 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3390 self.handle_scope_sync(connection_id, frame, generation, scopes)
3391 }
3392 ModuleControlRequestFromModule::ScopeApply {
3393 generation,
3394 upsert,
3395 end,
3396 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3397 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3398 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3399 }
3400 }
3401 }
3402
3403 fn handle_scope_served_by(
3404 &self,
3405 connection_id: ConnectionId,
3406 frame: Frame,
3407 ) -> Result<Vec<Frame>, RouterError> {
3408 let registered = self
3409 .registry
3410 .get_module_by_connection(connection_id)
3411 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3412 let Some(registered) = registered else {
3413 return Ok(vec![control_error_frame(
3414 &frame,
3415 "not_registered",
3416 "scope.served_by requires an active module registration owned by this connection",
3417 )?]);
3418 };
3419 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3420 {
3421 Ok(request) => request,
3422 Err(err) => {
3423 return Ok(vec![control_error_frame(
3424 &frame,
3425 "invalid_control_body",
3426 format!("malformed scope.served_by body: {err}"),
3427 )?])
3428 }
3429 };
3430 let answer = self
3431 .scopes
3432 .read()
3433 .unwrap_or_else(|poisoned| poisoned.into_inner())
3434 .served_by(
3435 ®istered.manifest.module_id,
3436 &request.scope_ref,
3437 request.scope_epoch,
3438 &request.module_id,
3439 );
3440 let reply = subc_control::ScopeServedByReply::new(
3441 answer,
3442 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3443 );
3444 Ok(vec![control_response_body_frame(
3445 &frame,
3446 &reply,
3447 "ScopeServedByReply",
3448 )?])
3449 }
3450
3451 fn handle_operator_confirm(
3452 &self,
3453 ctx: &RouteCtx,
3454 frame: Frame,
3455 ) -> Result<Vec<Frame>, RouterError> {
3456 use crate::operator_confirm::{audit, Outcome};
3457 let registration = self
3458 .registry
3459 .get_module_by_connection(ctx.connection_id)
3460 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3461 let Some(registration) = registration else {
3462 let outcome = Outcome::refusal("not_registered");
3463 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3464 return Ok(vec![outcome.frame(&frame)]);
3465 };
3466 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3467 Ok(request) => request,
3468 Err(_) => {
3469 let outcome = Outcome::refusal("invalid_control_body");
3470 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3471 return Ok(vec![outcome.frame(&frame)]);
3472 }
3473 };
3474 let declared = matches!(
3475 request.kind,
3476 Some(subc_protocol::session::OperatorConfirmKind::Operation)
3477 ) && request.operation.as_ref().is_some_and(|operation| {
3478 registration
3479 .manifest
3480 .confirm_operations
3481 .as_ref()
3482 .is_some_and(|operations| operations.contains(operation))
3483 });
3484 let module_id = registration.manifest.module_id;
3485 let nonce = self
3489 .hello_launch_nonces
3490 .lock()
3491 .unwrap_or_else(|p| p.into_inner())
3492 .nonce(ctx.connection_id)
3493 .map(str::to_owned);
3494 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3495 self.supervisor
3496 .spawned_consumer_authorized(&module_id, nonce)
3497 });
3498 let confirms = self.forwarding.operator_confirms();
3499 self.forwarding
3500 .with_operator_route(
3501 ctx.connection_id,
3502 request.route_channel,
3503 request.route_epoch,
3504 |binding| {
3505 let shortcut = binding
3506 .filter(|binding| {
3507 nonce_proven
3508 && crate::operator_confirm::valid_summary(&request.summary)
3509 && declared
3510 && crate::operator_confirm::ShortcutContext::route_eligible(
3511 &self.operator_app_policy,
3512 &self.launch_table,
3513 binding,
3514 )
3515 })
3516 .map(|binding| {
3517 let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
3518 &self.operator_app_policy,
3519 binding,
3520 );
3521 let stamp = if phone.is_some() {
3522 None
3523 } else {
3524 binding.operator_app.clone()
3525 };
3526 self.shortcut_context(
3527 stamp,
3528 phone,
3529 Some((request.route_channel, request.route_epoch)),
3530 )
3531 });
3532 confirms.admit_with_shortcut(
3533 ctx,
3534 frame,
3535 module_id,
3536 nonce_proven,
3537 request,
3538 binding,
3539 shortcut,
3540 )
3541 },
3542 )
3543 .map_err(RouterError::Forwarding)
3544 }
3545
3546 fn shortcut_context(
3547 &self,
3548 stamp: Option<crate::operator_apps::OperatorAppStamp>,
3549 phone_fingerprint: Option<String>,
3550 route: Option<(u16, u32)>,
3551 ) -> crate::operator_confirm::ShortcutContext {
3552 crate::operator_confirm::ShortcutContext {
3553 ledger: self.activity.clone(),
3554 forwarding: self.forwarding.clone(),
3555 launches: self.launch_table.clone(),
3556 stamp,
3557 phone_fingerprint,
3558 route,
3559 connections: self.operator_apps.clone(),
3560 #[cfg(test)]
3561 hooks: Default::default(),
3562 }
3563 }
3564
3565 #[cfg(any(test, feature = "test-support"))]
3567 pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
3568 self.operator_app_policy.phone_enabled = enabled;
3569 self
3570 }
3571
3572 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3575 let mut scopes = ScopeRecords::from(scopes);
3576 for (index, record) in scopes.records.iter_mut().enumerate() {
3577 if let Some(refusal) = Self::canonicalize_workdir(record) {
3578 scopes.refusals.insert(index, refusal);
3579 }
3580 }
3581 scopes
3582 }
3583
3584 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3585 let workdir = record.workdir.as_mut()?;
3586 let path = std::path::Path::new(workdir);
3587 if !path.is_absolute() {
3588 return Some((
3589 error_codes::INVALID_WORKDIR,
3590 "workdir must be absolute".to_string(),
3591 ));
3592 }
3593 match ProjectRootId::from_path_allowing_missing(path) {
3594 Ok(root) => match root.as_path().to_str() {
3595 Some(canonical) => {
3596 *workdir = canonical.to_string();
3597 None
3598 }
3599 None => Some((
3600 error_codes::INVALID_WORKDIR,
3601 "canonical workdir is not UTF-8".to_string(),
3602 )),
3603 },
3604 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3605 }
3606 }
3607
3608 fn handle_scope_sync(
3612 &self,
3613 connection_id: ConnectionId,
3614 frame: Frame,
3615 generation: u64,
3616 scopes: Vec<ScopeRecord>,
3617 ) -> Result<Vec<Frame>, RouterError> {
3618 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3619 }
3620
3621 fn handle_scope_change(
3622 &self,
3623 connection_id: ConnectionId,
3624 frame: Frame,
3625 generation: u64,
3626 scopes: Vec<ScopeRecord>,
3627 end: Option<Vec<ScopeEnd>>,
3628 ) -> Result<Vec<Frame>, RouterError> {
3629 let is_apply = end.is_some();
3630 let Some(registration) = self
3631 .registry
3632 .get_module_by_connection(connection_id)
3633 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3634 else {
3635 return Ok(vec![control_error_frame(
3636 &frame,
3637 "not_registered",
3638 "scope.sync and scope.apply require an active module registration owned by this connection",
3639 )?]);
3640 };
3641 let owner = registration.manifest.module_id;
3642 let scopes = Self::prepare_scope_records(scopes);
3643 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3644 let is_current_launch = |connection: ConnectionId| {
3645 self.hello_launch_nonces
3646 .lock()
3647 .unwrap_or_else(|poisoned| poisoned.into_inner())
3648 .presented(connection, current_nonce.as_deref())
3649 };
3650 let mut table = self
3655 .scopes
3656 .write()
3657 .unwrap_or_else(|poisoned| poisoned.into_inner());
3658 let outcome = match end {
3659 Some(end) => table.apply(
3660 &owner,
3661 connection_id,
3662 is_current_launch,
3663 generation,
3664 scopes,
3665 end,
3666 ),
3667 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3668 };
3669 let drained = match &outcome {
3670 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3671 Err(_) => Vec::new(),
3672 };
3673 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3674 drop(table);
3675 if has_deadlines {
3676 self.scope_deadline_added.notify_one();
3677 }
3678 match outcome {
3679 Ok(applied) => {
3680 let counts = ScopeOutcomeCounts::of(&applied.results);
3681 info!(
3682 owner = %owner,
3683 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3684 generation,
3685 records = applied.results.len(),
3686 created = counts.created,
3687 replaced = counts.replaced,
3688 updated = counts.updated,
3689 unchanged = counts.unchanged,
3690 refused = counts.refused,
3691 ended = applied.ended.len(),
3692 tag_changes = applied.tag_changes.len(),
3693 routes_closed = drained.len(),
3694 "scope change accepted"
3695 );
3696 for refused in applied
3702 .results
3703 .iter()
3704 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3705 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3706 {
3707 warn!(
3708 owner = %owner,
3709 generation,
3710 scope_ref = %refused.scope_ref,
3711 scope_epoch = refused.scope_epoch,
3712 code = refused.code.as_deref().unwrap_or(""),
3713 "scope record refused"
3714 );
3715 }
3716 self.close_scope_drained_routes(drained);
3717 let response = if is_apply {
3718 ModuleControlResponseToModule::ScopeApply {
3719 generation,
3720 results: applied.results,
3721 end_results: applied.end_results,
3722 ended: applied.ended,
3723 }
3724 } else {
3725 ModuleControlResponseToModule::ScopeSync {
3726 generation,
3727 results: applied.results,
3728 ended: applied.ended,
3729 }
3730 };
3731 Ok(vec![control_response_body_frame(
3732 &frame,
3733 &response,
3734 "scope change response",
3735 )?])
3736 }
3737 Err(refusal) => {
3738 info!(
3739 owner = %owner,
3740 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3741 generation,
3742 code = refusal.code,
3743 "scope change refused"
3744 );
3745 Ok(vec![control_error_frame(
3746 &frame,
3747 refusal.code,
3748 refusal.message,
3749 )?])
3750 }
3751 }
3752 }
3753
3754 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3760 if drained.is_empty() {
3761 return;
3762 }
3763 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3764 BTreeMap::new();
3765 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3766 for route in drained {
3767 warn!(
3768 module_id = %route.module_id,
3769 reason = ?route.reason,
3770 client_connection_id = route.client.connection_id.get(),
3771 route_channel = route.client.channel,
3772 "closing route because its scope changed"
3773 );
3774 pushes
3775 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3776 .or_insert_with(|| (route.reason, Vec::new()))
3777 .1
3778 .push(EndpointRoute {
3779 goodbye_target: route.client.clone(),
3780 principal: Principal::Unverified,
3781 bound_at: Instant::now(),
3782 draining: false,
3783 drain_reason: None,
3784 });
3785 goodbyes.push(route.module);
3786 goodbyes.push(route.client);
3787 }
3788 for ((module_id, _), (reason, routes)) in pushes {
3789 send_route_control_pushes(
3790 &self.forwarding,
3791 routes,
3792 ClientControlPush::RouteClosed {
3793 module_id,
3794 channels: Vec::new(),
3795 reason,
3796 drained: false,
3797 abandoned: 0,
3798 excluded_subscriptions: 0,
3799 terminal: Some(false),
3800 },
3801 );
3802 }
3803 self.emit_route_goodbyes(goodbyes);
3804 }
3805
3806 fn handle_scope_describe(
3809 &self,
3810 connection_id: ConnectionId,
3811 frame: Frame,
3812 owner: Principal,
3813 scope_ref: String,
3814 ) -> Result<Vec<Frame>, RouterError> {
3815 let registered = self
3816 .registry
3817 .get_module_by_connection(connection_id)
3818 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3819 let Some(registered) = registered else {
3820 return Ok(vec![control_error_frame(
3821 &frame,
3822 "not_registered",
3823 "scope.describe requires an active module registration owned by this connection",
3824 )?]);
3825 };
3826 let mut description = self
3827 .scopes
3828 .read()
3829 .unwrap_or_else(|poisoned| poisoned.into_inner())
3830 .describe(&owner, &scope_ref);
3831 if !registered
3834 .manifest
3835 .capabilities
3836 .as_ref()
3837 .is_some_and(|caps| {
3838 caps.provides
3839 .iter()
3840 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3841 })
3842 {
3843 if let Some(stamp) = description.stamp.as_mut() {
3844 stamp.attributes.on_behalf_of_agent = None;
3845 }
3846 }
3847 if !registered
3848 .manifest
3849 .capabilities
3850 .as_ref()
3851 .is_some_and(|caps| {
3852 caps.provides
3853 .iter()
3854 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
3855 })
3856 {
3857 if let Some(stamp) = description.stamp.as_mut() {
3858 stamp.attributes.flow_version = None;
3859 stamp.attributes.flow_code_hash = None;
3860 stamp.attributes.flow_install_id = None;
3861 }
3862 }
3863 let owner_configured = match &owner {
3864 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3869 _ => false,
3870 };
3871 let response = ModuleControlResponseToModule::ScopeDescribe {
3872 status: description.status,
3873 scope_epoch: description.scope_epoch,
3874 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3875 owner_synced: description.owner_synced,
3876 owner_configured,
3877 scope: description.stamp,
3878 };
3879 Ok(vec![control_response_body_frame(
3880 &frame,
3881 &response,
3882 "ModuleControlResponseToModule::ScopeDescribe",
3883 )?])
3884 }
3885
3886 fn handle_catalog_update(
3887 &self,
3888 connection_id: ConnectionId,
3889 frame: Frame,
3890 provides: Vec<ProviderRole>,
3891 capabilities: Option<CapabilityDeclarations>,
3892 ready: Option<bool>,
3893 ) -> Result<Vec<Frame>, RouterError> {
3894 self.refresh_capability_requirements();
3895 let Some(registration) = self
3896 .registry
3897 .get_module_by_connection(connection_id)
3898 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3899 else {
3900 return Ok(vec![control_error_frame(
3901 &frame,
3902 "not_registered",
3903 "catalog.update requires an active module registration owned by this connection",
3904 )?]);
3905 };
3906
3907 if let Some(message) =
3908 catalog_update_frozen_field_message(®istration.manifest, &provides)
3909 {
3910 return Ok(vec![control_error_frame(
3911 &frame,
3912 "catalog_update_frozen_field",
3913 message,
3914 )?]);
3915 }
3916
3917 let mut candidate = registration.manifest.clone();
3918 candidate.provides = provides.clone();
3919 candidate.capabilities = capabilities
3920 .clone()
3921 .or_else(|| registration.manifest.capabilities.clone());
3922 if let Some(rich_ui) = &candidate.rich_ui {
3923 if let Err(err) = rich_ui.validate() {
3924 return Ok(vec![control_error_frame(
3925 &frame,
3926 "invalid_manifest",
3927 err.to_string(),
3928 )?]);
3929 }
3930 }
3931 if let Err(err) = candidate.validate_capability_grammar() {
3932 return Ok(vec![control_error_frame(
3933 &frame,
3934 "invalid_capability_grammar",
3935 err.to_string(),
3936 )?]);
3937 }
3938
3939 let mut conflicts = self
3942 .capability_evaluator
3943 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3944 if let Some(conflict) = conflicts.first() {
3945 let message = format!(
3946 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3947 conflict.capability, conflict.claimants[0], candidate.module_id
3948 );
3949 for conflict in &mut conflicts {
3950 conflict.source = DuplicateClaimSource::CatalogUpdate;
3951 }
3952 log_duplicate_claim_events(conflicts);
3953 return Ok(vec![control_error_frame(
3954 &frame,
3955 "reserved_capability",
3956 message,
3957 )?]);
3958 }
3959
3960 let updated = self
3961 .registry
3962 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3963 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3964 if updated.is_none() {
3965 return Ok(vec![control_error_frame(
3966 &frame,
3967 "not_registered",
3968 "catalog.update requires an active module registration owned by this connection",
3969 )?]);
3970 }
3971 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3972 log_duplicate_claim_events(
3973 self.capability_evaluator
3974 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3975 );
3976 }
3977 if capability_census_trigger(
3978 registration.manifest.capabilities.as_ref(),
3979 updated
3980 .as_ref()
3981 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3982 ) {
3983 self.enforce_capability_denies();
3984 }
3985 self.refresh_capability_requirements();
3986
3987 let response = ModuleControlResponseToModule::CatalogUpdate {};
3988 control_response_body_frame(
3989 &frame,
3990 &response,
3991 "ModuleControlResponseToModule::CatalogUpdate",
3992 )
3993 .map(|frame| vec![frame])
3994 }
3995
3996 pub(crate) fn with_activity_ledger_config(
3997 mut self,
3998 run_dir: &Path,
3999 incarnation: String,
4000 config: crate::activity::ActivityConfig,
4001 ) -> Self {
4002 self.activity = Some(crate::activity::ActivityLedger::open_config(
4003 run_dir.join("activity.jsonl"),
4004 incarnation,
4005 config,
4006 ));
4007 self.supervisor.set_activity(self.activity.clone());
4008 self
4009 }
4010
4011 pub(crate) async fn configure_activity(
4012 &self,
4013 config: crate::activity::ActivityConfig,
4014 ) -> Result<(), &'static str> {
4015 match &self.activity {
4016 Some(ledger) => ledger.configure(config).await,
4017 None => Ok(()),
4018 }
4019 }
4020
4021 fn cancel_activity_subscription(
4022 &self,
4023 ctx: &RouteCtx,
4024 frame: &Frame,
4025 ) -> Result<bool, RouterError> {
4026 let task = self
4027 .activity_streams
4028 .lock()
4029 .unwrap_or_else(|p| p.into_inner())
4030 .remove(&(ctx.connection_id, frame.header.corr));
4031 let Some(task) = task else {
4032 return Ok(false);
4033 };
4034 task.abort();
4035 let end = Frame::build_with_version(
4036 response_version(frame),
4037 FrameType::StreamEnd,
4038 control_flags(),
4039 0,
4040 0,
4041 frame.header.corr,
4042 Vec::new(),
4043 )
4044 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
4045 let sink = ctx.egress.clone();
4046 tokio::spawn(async move {
4047 let _ = sink.send(end).await;
4048 });
4049 Ok(true)
4050 }
4051
4052 async fn handle_activity_read(
4053 &self,
4054 ctx: &RouteCtx,
4055 frame: Frame,
4056 ) -> Result<Vec<Frame>, RouterError> {
4057 use subc_control::{
4058 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
4059 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
4060 };
4061 #[derive(Deserialize)]
4062 struct Identity {
4063 #[serde(default)]
4064 consumer_identity: Option<ConsumerIdentity>,
4065 }
4066 let principal = match self
4070 .forwarding
4071 .module_id_for_connection(ctx.connection_id)
4072 .map_err(RouterError::Forwarding)?
4073 {
4074 Some(module_id) => Ok(Principal::Reserved { module_id }),
4075 None => {
4076 let identity: Identity = match serde_json::from_slice(&frame.body) {
4077 Ok(identity) => identity,
4078 Err(_) => {
4079 return Ok(vec![control_error_frame(
4080 &frame,
4081 ACTIVITY_MALFORMED,
4082 "malformed activity read",
4083 )?])
4084 }
4085 };
4086 self.route_open_principal(&frame, identity.consumer_identity)?
4087 }
4088 };
4089 let admitted = matches!(principal, Ok(Principal::Direct))
4090 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
4091 if !admitted {
4092 return Ok(vec![control_error_frame(
4093 &frame,
4094 ACTIVITY_READ_FORBIDDEN,
4095 "activity reads require an operator or trusted mirror principal",
4096 )?]);
4097 }
4098 let Some(ledger) = &self.activity else {
4099 return Ok(vec![control_error_frame(
4100 &frame,
4101 ACTIVITY_UNAVAILABLE,
4102 "activity ledger unavailable",
4103 )?]);
4104 };
4105 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
4106 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
4107 {
4108 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
4109 Ok(request) => ledger.query(request).await,
4110 Err(_) => Err(ACTIVITY_MALFORMED),
4111 };
4112 return match result {
4113 Ok(reply) => Ok(vec![control_response_body_frame(
4114 &frame,
4115 &reply,
4116 ops::ACTIVITY_QUERY,
4117 )?]),
4118 Err(code) => Ok(vec![control_error_frame(
4119 &frame,
4120 code,
4121 "activity query refused",
4122 )?]),
4123 };
4124 }
4125 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
4126 Ok(request) => request,
4127 Err(_) => {
4128 return Ok(vec![control_error_frame(
4129 &frame,
4130 ACTIVITY_MALFORMED,
4131 "malformed activity subscription",
4132 )?])
4133 }
4134 };
4135 let mut subscription = match ledger.subscribe(request.since).await {
4136 Ok(subscription) => subscription,
4137 Err(code) => {
4138 return Ok(vec![control_error_frame(
4139 &frame,
4140 code,
4141 "activity subscription refused",
4142 )?])
4143 }
4144 };
4145 let key = (ctx.connection_id, frame.header.corr);
4146 let sink = ctx.egress.clone();
4147 let mut streams = self
4148 .activity_streams
4149 .lock()
4150 .unwrap_or_else(|p| p.into_inner());
4151 streams.retain(|_, task| !task.is_finished());
4152 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
4153 return Ok(vec![control_error_frame(
4154 &frame,
4155 "duplicate_subscription",
4156 "activity subscription correlation id already in use",
4157 )?]);
4158 }
4159 let task = tokio::spawn(async move {
4160 loop {
4161 let entry = if let Some(entry) = subscription.replay.pop_front() {
4162 Some(entry)
4163 } else {
4164 subscription.receiver.recv().await
4165 };
4166 let Some(entry) = entry else {
4167 break;
4168 };
4169 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
4170 return;
4171 };
4172 let Ok(next) = Frame::build_with_version(
4173 response_version(&frame),
4174 FrameType::StreamData,
4175 control_flags(),
4176 0,
4177 0,
4178 frame.header.corr,
4179 body,
4180 ) else {
4181 return;
4182 };
4183 if sink.send(next).await.is_err() {
4184 return;
4185 }
4186 }
4187 if let Some(code) = subscription.terminal_code() {
4190 if let Ok(end) =
4191 control_error_frame(&frame, code, "activity subscriber fell behind")
4192 {
4193 let _ = sink.send(end).await;
4194 }
4195 }
4196 });
4197 streams.insert(key, task.abort_handle());
4198 Ok(Vec::new())
4199 }
4200
4201 async fn handle_activity_record(
4202 &self,
4203 ctx: &RouteCtx,
4204 frame: Frame,
4205 ) -> Result<Vec<Frame>, RouterError> {
4206 use subc_control::{
4207 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
4208 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
4209 };
4210 let result = async {
4211 let module = self
4212 .forwarding
4213 .module_id_for_connection(ctx.connection_id)
4214 .ok()
4215 .flatten()
4216 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
4217 let request: ActivityRecordRequest =
4218 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
4219 crate::activity::validate(&module, &request)?;
4220 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
4221 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
4222 let at_ms = std::time::SystemTime::now()
4223 .duration_since(std::time::UNIX_EPOCH)
4224 .unwrap_or_default()
4225 .as_millis() as u64;
4226 ledger.record(module, request, actor, at_ms).await
4227 }
4228 .await;
4229 match result {
4230 Ok(entry) => Ok(vec![control_response_body_frame(
4231 &frame,
4232 &ActivityRecordResponse { seq: entry.seq },
4233 "activity.record",
4234 )?]),
4235 Err(code) => Ok(vec![control_error_frame(
4236 &frame,
4237 code,
4238 "activity recording refused",
4239 )?]),
4240 }
4241 }
4242
4243 fn activity_actor(
4244 &self,
4245 connection: ConnectionId,
4246 module: &str,
4247 on: Option<&subc_control::ActivityOn>,
4248 ) -> Result<subc_control::ActivityActor, &'static str> {
4249 use crate::forwarding::{DataRoute, DataRouteState};
4250 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
4251 let principal = |id: &str| Principal::Reserved {
4252 module_id: id.to_owned(),
4253 };
4254 let mut actor = ActivityActor {
4255 principal: principal(module),
4256 scope_owner: None,
4257 agent_id: None,
4258 run_id: None,
4259 flow_id: None,
4260 project_root: None,
4261 workdir: None,
4262 project_id: None,
4263 };
4264 let scope = match on {
4265 None => return Ok(actor),
4266 Some(ActivityOn::Scope(scope)) => {
4267 let selector = subc_protocol::scope::ScopeSelector::new(
4268 principal(module),
4269 scope.scope_ref.clone(),
4270 Some(scope.epoch),
4271 );
4272 self.scopes
4273 .read()
4274 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4275 .attribute(&principal(module), module, &selector, true)
4276 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4277 }
4278 Some(ActivityOn::Route(route)) => {
4279 let DataRoute::Module(DataRouteState::Bound(binding)) = self
4280 .forwarding
4281 .lookup_data_route(connection, route.channel, route.epoch)
4282 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4283 else {
4284 return Err(ACTIVITY_ACTOR_UNKNOWN);
4285 };
4286 if binding.module_id != module {
4287 return Err(ACTIVITY_ACTOR_UNKNOWN);
4288 }
4289 actor.principal = binding.principal.clone();
4290 actor.project_root = binding
4291 .project_root
4292 .as_ref()
4293 .map(|root| root.as_path().to_path_buf());
4294 let Some(bound) = &binding.scope else {
4295 return Ok(actor);
4296 };
4297 let owner = principal(&bound.owner);
4298 let selector = subc_protocol::scope::ScopeSelector::new(
4299 owner.clone(),
4300 bound.scope_ref.clone(),
4301 Some(bound.tag.scope_epoch),
4302 );
4303 let admitted = self
4304 .scopes
4305 .read()
4306 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4307 .attribute(&owner, module, &selector, true)
4308 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
4309 if admitted.tag != bound.tag {
4310 return Err(ACTIVITY_ACTOR_UNKNOWN);
4311 }
4312 admitted
4313 }
4314 };
4315 actor.scope_owner = Some(scope.stamp.owner);
4316 actor.agent_id = scope.stamp.attributes.agent_id;
4317 actor.run_id = scope.stamp.attributes.run_id;
4318 actor.flow_id = scope.stamp.attributes.flow_id;
4319 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
4320 actor.project_id = scope.stamp.project_id;
4321 Ok(actor)
4322 }
4323
4324 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4325 self.refresh_capability_requirements();
4326 let mut counters = self.counters.snapshot();
4332 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
4333 obj.insert(
4334 "activity_unparseable_lines".into(),
4335 ledger.read_errors().into(),
4336 );
4337 obj.insert(
4338 "activity_lifecycle_append_failures".into(),
4339 ledger.lifecycle_failures().into(),
4340 );
4341 obj.insert(
4342 "activity_started_digest_skips".into(),
4343 ledger.lifecycle_digest_skips().into(),
4344 );
4345 }
4346 if let (Ok((connections_with_routes, max)), Some(obj)) = (
4347 self.forwarding.client_route_concentration(),
4348 counters.as_object_mut(),
4349 ) {
4350 obj.insert(
4351 "client_connections_with_routes".into(),
4352 connections_with_routes.into(),
4353 );
4354 obj.insert("max_routes_on_one_connection".into(), max.into());
4355 }
4356 if let (Some(open_breakers), Some(obj)) = (
4362 self.route_bind_breakers.open_snapshot(),
4363 counters.as_object_mut(),
4364 ) {
4365 obj.insert("route_bind_breakers_open".into(), open_breakers);
4366 }
4367 if let (Some(reason), Some(obj)) = (
4372 self.local_endpoint_unavailable.clone(),
4373 counters.as_object_mut(),
4374 ) {
4375 obj.insert("local_endpoint_unavailable".into(), reason.into());
4376 }
4377 let response = ClientControlResponse::ServerDescribe {
4378 protocol_ver: PROTOCOL_VERSION,
4379 subc_ops: subc_ops(),
4380 capabilities: self.describe_capabilities(),
4381 connected_clients: self.connected_clients.count(),
4382 counters: Some(counters),
4383 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
4384 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
4385 capability_requirements: self.capability_requirement_statuses(),
4386 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4387 config_path: self
4388 .rescan
4389 .as_ref()
4390 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
4391 };
4392 Ok(vec![control_response_body_frame(
4393 &frame,
4394 &response,
4395 "ClientControlResponse::ServerDescribe",
4396 )?])
4397 }
4398
4399 fn handle_catalog_list(
4400 &self,
4401 frame: Frame,
4402 module_id: Option<String>,
4403 ) -> Result<Vec<Frame>, RouterError> {
4404 let (generation, modules) = self
4405 .registry
4406 .list_modules_with_nonce_verification()
4407 .map_err(|err| {
4408 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4409 })?;
4410 let entries = modules
4411 .into_iter()
4412 .filter(|(registration, _)| {
4413 module_id
4414 .as_deref()
4415 .map(|wanted| registration.manifest.module_id == wanted)
4416 .unwrap_or(true)
4417 })
4418 .map(|(registration, nonce_verified)| {
4419 let not_ready = self.not_ready_reason(®istration);
4420 let roles = registration.manifest.provides;
4421 CatalogEntry::new(
4422 registration.manifest.module_id,
4423 roles,
4424 registration.control_ops,
4425 )
4426 .with_ready(not_ready.is_none())
4427 .with_display_name(registration.manifest.display_name)
4428 .with_not_ready(not_ready)
4429 .with_module_version(Some(registration.manifest.module_version))
4430 .with_provenance(registration.manifest.provenance)
4431 .with_capabilities(registration.manifest.capabilities)
4432 .with_self_signals(registration.manifest.self_signals)
4433 .with_events(registration.manifest.events)
4434 .with_operation_delivery(registration.manifest.operation_delivery)
4435 .with_operation_payload(registration.manifest.operation_payload)
4436 .with_flow_operations(registration.manifest.flow_operations)
4437 .with_confirm_operations(registration.manifest.confirm_operations)
4438 .with_rich_ui(registration.manifest.rich_ui)
4439 .with_role_declarations(registration.manifest.role_declarations)
4440 .with_nonce_verified(nonce_verified)
4441 })
4442 .collect();
4443 let response = ClientControlResponse::CatalogList {
4444 generation,
4445 modules: entries,
4446 subc_ops: subc_ops(),
4447 };
4448 Ok(vec![control_response_body_frame(
4449 &frame,
4450 &response,
4451 "ClientControlResponse::CatalogList",
4452 )?])
4453 }
4454
4455 fn attest_route_open_consumer(
4460 &self,
4461 consumer_identity: Option<&ConsumerIdentity>,
4462 ) -> Option<Principal> {
4463 let Some(consumer_identity) = consumer_identity else {
4464 return Some(Principal::Direct);
4465 };
4466 self.supervisor
4467 .spawned_consumer_authorized(
4468 &consumer_identity.module_id,
4469 &consumer_identity.launch_nonce,
4470 )
4471 .then(|| Principal::Reserved {
4472 module_id: consumer_identity.module_id.clone(),
4473 })
4474 }
4475
4476 fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4480 let Ok(ClientControlRequest::RouteOpen {
4481 identity,
4482 consumer_identity,
4483 scope,
4484 ..
4485 }) = parse_client_control_request(&frame.body)
4486 else {
4487 return RouteOpenCaller::unknown();
4488 };
4489 let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4490 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4491 }
4492
4493 fn route_open_principal(
4494 &self,
4495 frame: &Frame,
4496 consumer_identity: Option<ConsumerIdentity>,
4497 ) -> Result<Result<Principal, Frame>, RouterError> {
4498 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4499 self.route_open_principal_attested(frame, consumer_identity, attested)
4500 }
4501
4502 fn route_open_principal_attested(
4506 &self,
4507 frame: &Frame,
4508 consumer_identity: Option<ConsumerIdentity>,
4509 attested: Option<Principal>,
4510 ) -> Result<Result<Principal, Frame>, RouterError> {
4511 if let Some(principal) = attested {
4512 return Ok(Ok(principal));
4513 }
4514 let Some(consumer_identity) = consumer_identity else {
4515 return Ok(Ok(Principal::Direct));
4516 };
4517
4518 Ok(Err(control_error_frame(
4519 frame,
4520 "bad_consumer_identity",
4521 format!(
4522 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4523 consumer_identity.module_id
4524 ),
4525 )?))
4526 }
4527
4528 #[allow(clippy::too_many_arguments)]
4535 fn route_open_refusal_frame(
4536 &self,
4537 ctx: &RouteCtx,
4538 caller: &RouteOpenCaller,
4539 frame: &Frame,
4540 module_id: &str,
4541 reason: &'static str,
4542 code: &'static str,
4543 message: impl Into<String>,
4544 ) -> Result<Frame, RouterError> {
4545 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4546 control_error_frame(frame, code, message.into())
4547 }
4548
4549 #[allow(clippy::too_many_arguments)]
4558 fn route_bind_rejection(
4559 &self,
4560 ctx: &RouteCtx,
4561 caller: &RouteOpenCaller,
4562 frame: &Frame,
4563 target_module_id: &str,
4564 body: ErrorBody,
4565 daemon_authored: bool,
4566 ) -> Result<Vec<Frame>, RouterError> {
4567 if daemon_authored {
4573 let scope_code = match body.code.as_str() {
4574 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4575 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4576 _ => None,
4577 };
4578 if let Some(code) = scope_code {
4579 self.observe_route_open_refusal(
4580 ctx,
4581 caller,
4582 target_module_id,
4583 "scope_changed_before_commit",
4584 code,
4585 );
4586 return Ok(vec![control_error_body_frame(frame, body)?]);
4587 }
4588 }
4589 self.counters
4590 .increment_route_open_refused("module_rejected");
4591 info!(
4592 target: "control",
4593 code = "module_rejected",
4594 module_code = ?body.code,
4595 module_id = ?target_module_id,
4596 connection_id = ctx.connection_id.get(),
4597 principal = %caller.principal,
4598 scope = %caller.scope,
4599 harness = %caller.harness,
4600 "route.open refused"
4601 );
4602 let refusal = if daemon_authored {
4603 control_error_body_frame(frame, body)?
4604 } else {
4605 control_relayed_error_body_frame(frame, body)?
4606 };
4607 Ok(vec![refusal])
4608 }
4609
4610 #[allow(clippy::too_many_arguments)]
4628 fn route_open_breaker_refusal_frame(
4629 &self,
4630 ctx: &RouteCtx,
4631 caller: &RouteOpenCaller,
4632 frame: &Frame,
4633 module_id: &str,
4634 consecutive_timeouts: u32,
4635 retry_in: Duration,
4636 probe_in_flight: bool,
4637 ) -> Result<Frame, RouterError> {
4638 self.counters
4639 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4640 debug!(
4641 target: "control",
4642 code = "module_timeout",
4643 module_id = ?module_id,
4644 connection_id = ctx.connection_id.get(),
4645 principal = %caller.principal,
4646 scope = %caller.scope,
4647 harness = %caller.harness,
4648 consecutive_timeouts,
4649 retry_in_ms = retry_in.as_millis() as u64,
4650 probe_in_flight,
4651 "route.open refused by open bind-relay breaker"
4652 );
4653 let detail = if probe_in_flight {
4660 "one test route is already being tried; retry once it settles".to_string()
4661 } else {
4662 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4663 };
4664 control_error_frame(
4665 frame,
4666 "module_timeout",
4667 format!(
4668 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4669 timed out in a row); {detail}; its established routes are unaffected"
4670 ),
4671 )
4672 }
4673
4674 fn observe_route_open_refusal(
4687 &self,
4688 ctx: &RouteCtx,
4689 caller: &RouteOpenCaller,
4690 module_id: &str,
4691 reason: &'static str,
4692 code: &'static str,
4693 ) {
4694 self.counters.increment_route_open_refused(code);
4695 info!(
4696 target: "control",
4697 code,
4698 reason,
4699 module_id = ?module_id,
4700 connection_id = ctx.connection_id.get(),
4701 principal = %caller.principal,
4702 scope = %caller.scope,
4703 harness = %caller.harness,
4704 "route.open refused"
4705 );
4706 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4707 self.route_outages.record_not_serving(module_id, reason);
4708 }
4709 }
4710
4711 fn observe_route_open_accept(
4756 &self,
4757 ctx: &RouteCtx,
4758 module_id: &str,
4759 principal: &str,
4760 scope: &str,
4761 ) {
4762 self.route_outages.record_accepted(module_id);
4763 self.counters.increment_route_open_accepted(principal);
4764 info!(
4765 target: "control",
4766 principal,
4767 module_id,
4768 connection_id = ctx.connection_id.get(),
4769 scope = %scope,
4770 "route.open accepted"
4771 );
4772 }
4773
4774 fn supervised_absent_route_open_refusal_frame(
4775 &self,
4776 ctx: &RouteCtx,
4777 caller: &RouteOpenCaller,
4778 frame: &Frame,
4779 module_id: &str,
4780 code: &'static str,
4781 status: &crate::supervise::ModuleStatus,
4782 ) -> Result<Frame, RouterError> {
4783 self.counters.increment_route_open_refused(code);
4784 info!(
4785 target: "control",
4786 code,
4787 reason = "supervised_not_registered",
4788 module_id = ?module_id,
4789 connection_id = ctx.connection_id.get(),
4790 principal = %caller.principal,
4791 scope = %caller.scope,
4792 harness = %caller.harness,
4793 state = %status.state,
4794 enabled = status.enabled,
4795 live = status.live,
4796 "route.open refused"
4797 );
4798 self.route_outages
4802 .record_not_serving(module_id, "supervised_not_registered");
4803 let message = if let Some(pid) = status.waiting_for_previous_pid {
4804 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4805 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4806 })
4807 } else {
4808 format!(
4809 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4810 status.state, status.enabled, status.live
4811 )
4812 };
4813 control_error_frame(frame, code, message)
4814 }
4815
4816 async fn handle_route_open(
4817 &self,
4818 ctx: &RouteCtx,
4819 frame: Frame,
4820 request: RouteOpenRequest,
4821 timings: &mut RouteOpenTimings,
4822 ) -> Result<Vec<Frame>, RouterError> {
4823 let RouteOpenRequest {
4824 target,
4825 mut identity,
4826 consumer_identity,
4827 consumer_capabilities,
4828 role_versions,
4829 busy_refusals,
4830 admission_facts,
4831 scope,
4832 } = request;
4833 let target_module_id = target_module_id(&target).to_string();
4834 timings.module_id.clone_from(&target_module_id);
4835 if self
4836 .registry
4837 .get_module_by_connection(ctx.connection_id)
4838 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4839 .is_some()
4840 {
4841 return Ok(vec![control_error_frame(
4842 &frame,
4843 "invalid_request",
4844 "module connections cannot open client routes",
4845 )?]);
4846 }
4847 debug!(
4848 connection_id = ctx.connection_id.get(),
4849 corr = frame.header.corr,
4850 module_id = %target_module_id,
4851 "handling route.open"
4852 );
4853
4854 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4859 let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4860
4861 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4867 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4868 self.observe_route_open_refusal(
4869 ctx,
4870 &caller,
4871 &target_module_id,
4872 "invalid_role_versions",
4873 error_codes::INVALID_REQUEST,
4874 );
4875 return Ok(vec![control_error_body_frame(
4876 &frame,
4877 ErrorBody {
4878 code: error_codes::INVALID_REQUEST.to_string(),
4879 message: error.to_string(),
4880 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4881 },
4882 )?]);
4883 }
4884
4885 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4886 if status.waiting_for_previous_pid.is_some() {
4887 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4888 ctx,
4889 &caller,
4890 &frame,
4891 &target_module_id,
4892 "module_warming",
4893 &status,
4894 )?]);
4895 }
4896 }
4897
4898 let Some(registration) = self
4917 .registry
4918 .get_module(&target_module_id)
4919 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4920 else {
4921 if let Some((status, warming)) =
4922 self.supervisor_status(&target_module_id, frame.header.corr)?
4923 {
4924 if status.protocol == ModuleProtocol::None {
4931 return Ok(vec![self.route_open_refusal_frame(
4932 ctx,
4933 &caller,
4934 &frame,
4935 &target_module_id,
4936 "protocol_none",
4937 error_codes::MODULE_NO_PROTOCOL,
4938 format!(
4939 "module_id '{target_module_id}' is declared protocol: none; \
4940 it speaks no subc wire and serves no routes"
4941 ),
4942 )?]);
4943 }
4944 let code = if warming {
4945 "module_warming"
4946 } else {
4947 "target_unavailable"
4948 };
4949 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4950 ctx,
4951 &caller,
4952 &frame,
4953 &target_module_id,
4954 code,
4955 &status,
4956 )?]);
4957 }
4958 if let Some(removed_ago_ms) =
4959 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4960 {
4961 return Ok(vec![self.route_open_refusal_frame(
4962 ctx,
4963 &caller,
4964 &frame,
4965 &target_module_id,
4966 "removed",
4967 error_codes::MODULE_REMOVED,
4968 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4969 )?]);
4970 }
4971 return Ok(vec![self.route_open_refusal_frame(
4972 ctx,
4973 &caller,
4974 &frame,
4975 &target_module_id,
4976 "not_registered",
4977 error_codes::UNKNOWN_MODULE,
4978 format!("module_id '{target_module_id}' is not registered"),
4979 )?]);
4980 };
4981
4982 if !registration.ready {
4986 self.counters
4987 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4988 info!(
4989 target: "control",
4990 code = error_codes::MODULE_WARMING,
4991 module_id = ?target_module_id,
4992 connection_id = ctx.connection_id.get(),
4993 principal = %caller.principal,
4994 scope = %caller.scope,
4995 harness = %caller.harness,
4996 reason = "declared_not_ready",
4997 "route.open refused"
4998 );
4999 self.route_outages
5002 .record_not_serving(&target_module_id, "declared_not_ready");
5003 return Ok(vec![control_error_body_frame(
5004 &frame,
5005 ErrorBody {
5006 code: error_codes::MODULE_WARMING.to_string(),
5007 message: format!(
5008 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
5009 ),
5010 detail: Some(serde_json::json!({
5011 "reason": "declared_not_ready"
5012 })),
5013 },
5014 )?]);
5015 }
5016
5017 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
5036 self.counters
5037 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
5038 info!(
5039 target: "control",
5040 code = error_codes::MODULE_WARMING,
5041 module_id = ?target_module_id,
5042 connection_id = ctx.connection_id.get(),
5043 principal = %caller.principal,
5044 scope = %caller.scope,
5045 harness = %caller.harness,
5046 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5047 capability = %capability,
5048 "route.open refused"
5049 );
5050 return Ok(vec![control_error_body_frame(
5051 &frame,
5052 ErrorBody {
5053 code: error_codes::MODULE_WARMING.to_string(),
5054 message: format!(
5055 "module_id '{target_module_id}' requires capability '{capability}', \
5056 which no registered module provides; retry"
5057 ),
5058 detail: Some(serde_json::json!({
5059 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5060 "capability": capability,
5061 })),
5062 },
5063 )?]);
5064 }
5065
5066 if !target_has_required_role(&target, ®istration.manifest.provides) {
5067 return Ok(vec![self.route_open_refusal_frame(
5068 ctx,
5069 &caller,
5070 &frame,
5071 &target_module_id,
5072 "role_not_provided",
5073 "target_unavailable",
5074 format!("module_id '{target_module_id}' does not provide the requested target"),
5075 )?]);
5076 }
5077
5078 if registration.state != ChannelState::Active {
5079 return Ok(vec![self.route_open_refusal_frame(
5080 ctx,
5081 &caller,
5082 &frame,
5083 &target_module_id,
5084 "registration_not_active",
5085 "target_unavailable",
5086 format!("module_id '{target_module_id}' is not active"),
5087 )?]);
5088 }
5089
5090 if self
5091 .forwarding
5092 .module_is_draining(&target_module_id)
5093 .map_err(RouterError::Forwarding)?
5094 {
5095 return Ok(vec![self.route_open_refusal_frame(
5096 ctx,
5097 &caller,
5098 &frame,
5099 &target_module_id,
5100 "reloading",
5101 "module_reloading",
5102 format!("module_id '{target_module_id}' is reloading"),
5103 )?]);
5104 }
5105
5106 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
5107 process_liveness.process_live(&target_module_id) == Some(false)
5108 }) {
5109 if process_liveness.process_replacing(&target_module_id) {
5116 return Ok(vec![self.route_open_refusal_frame(
5117 ctx,
5118 &caller,
5119 &frame,
5120 &target_module_id,
5121 "reloading",
5122 "module_reloading",
5123 format!("module_id '{target_module_id}' is reloading"),
5124 )?]);
5125 }
5126 return Ok(vec![self.route_open_refusal_frame(
5127 ctx,
5128 &caller,
5129 &frame,
5130 &target_module_id,
5131 "supervisor_not_live",
5132 "target_unavailable",
5133 format!("module_id '{target_module_id}' is not live"),
5134 )?]);
5135 }
5136
5137 if !self
5138 .forwarding
5139 .has_live_module_connection(&target_module_id)
5140 .map_err(RouterError::Forwarding)?
5141 {
5142 return Ok(vec![self.route_open_refusal_frame(
5143 ctx,
5144 &caller,
5145 &frame,
5146 &target_module_id,
5147 "no_forwarding_connection",
5148 "target_unavailable",
5149 format!("module_id '{target_module_id}' has no live forwarding connection"),
5150 )?]);
5151 }
5152
5153 if let Some(error) =
5154 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
5155 {
5156 self.observe_route_open_refusal(
5157 ctx,
5158 &caller,
5159 &target_module_id,
5160 "op_not_allowed",
5161 "op_not_allowed",
5162 );
5163 return Ok(vec![error]);
5164 }
5165
5166 let principal =
5167 match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
5168 Ok(principal) => principal,
5169 Err(error) => {
5170 self.observe_route_open_refusal(
5171 ctx,
5172 &caller,
5173 &target_module_id,
5174 "bad_consumer_identity",
5175 "bad_consumer_identity",
5176 );
5177 return Ok(vec![error]);
5178 }
5179 };
5180 timings.principal = principal_label(&principal);
5181
5182 let build_class = identity.build_class.take().filter(|build_class| {
5185 if matches!(&principal, Principal::Reserved { module_id } if module_id == "callosum") {
5186 build_class == subc_protocol::BUILD_CLASS_PRODUCTION
5187 } else {
5188 if self
5189 .build_class_warned
5190 .lock()
5191 .unwrap_or_else(|p| p.into_inner())
5192 .insert(ctx.connection_id)
5193 {
5194 warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
5195 "ignoring build_class from a non-Callosum opener");
5196 }
5197 false
5198 }
5199 });
5200
5201 if let Principal::Reserved {
5205 module_id: opening_module_id,
5206 } = &principal
5207 {
5208 if let Some(opening_registration) = self
5209 .registry
5210 .get_module(opening_module_id)
5211 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5212 {
5213 if let Some(capability) =
5214 denied_capability(&opening_registration.manifest, ®istration.manifest)
5215 {
5216 warn!(
5217 opening_module_id,
5218 target_module_id,
5219 capability,
5220 "refusing route.open because an attested capability deny edge matches"
5221 );
5222 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5223 &frame,
5224 &target_module_id,
5225 "capability_deny_edge",
5226 "capability_forbidden",
5227 format!(
5228 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
5229 ),
5230 )?]);
5231 }
5232 }
5233 }
5234
5235 if admission_facts.is_some() {
5236 let carrier_matches = matches!(
5237 &principal,
5238 Principal::Reserved { module_id }
5239 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
5240 );
5241 if !carrier_matches {
5242 return Ok(vec![self.route_open_refusal_frame(
5243 ctx,
5244 &caller,
5245 &frame,
5246 &target_module_id,
5247 "admission_facts_carrier_not_permitted",
5248 "admission_facts_not_permitted",
5249 "admission facts may only be carried by the configured reserved module",
5250 )?]);
5251 }
5252
5253 let target_allowed = self
5254 .admission_facts_targets
5255 .as_ref()
5256 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
5257 if !target_allowed {
5258 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5259 &frame,
5260 &target_module_id,
5261 "admission_facts_target_not_listed",
5262 "admission_facts_target_not_allowed",
5263 format!(
5264 "admission facts are not permitted for target module_id '{target_module_id}'"
5265 ),
5266 )?]);
5267 }
5268
5269 }
5273
5274 let (bound_scope, mut scope_stamp) = match &scope {
5280 None => (None, None),
5281 Some(selector) => {
5282 let owner_configured = match &selector.owner {
5283 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5289 _ => false,
5290 };
5291 let mut scopes = self
5292 .scopes
5293 .write()
5294 .unwrap_or_else(|poisoned| poisoned.into_inner());
5295 let admitted = scopes
5296 .admit(&principal, &target_module_id, selector, owner_configured)
5297 .and_then(|admission| {
5298 crate::scopes::check_target_flow_support(
5299 &admission.stamp,
5300 &target_module_id,
5301 registration.manifest.capabilities.as_ref(),
5302 )?;
5303 crate::scopes::check_target_agent_run_support(
5304 &admission.stamp,
5305 &target_module_id,
5306 registration.manifest.capabilities.as_ref(),
5307 )?;
5308 let bound = scopes.reserve_bind(
5309 &admission,
5310 &principal,
5311 ctx.connection_id,
5312 selector.hold_presence,
5313 );
5314 Ok((admission, bound))
5315 });
5316 match admitted {
5317 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
5318 Err(refusal) => {
5319 return Ok(vec![self.route_open_refusal_frame(
5320 ctx,
5321 &caller,
5322 &frame,
5323 &target_module_id,
5324 refusal.code,
5325 refusal.code,
5326 refusal.message,
5327 )?]);
5328 }
5329 }
5330 }
5331 };
5332 if let Some(stamp) = &scope_stamp {
5335 caller.scope = route_open_scope_label(stamp);
5336 }
5337
5338 timings.enter(RouteOpenPhase::ProjectRoot);
5346 let legacy_project = match crate::bind_compat::LegacyProject::from_open(&frame.body) {
5347 Ok(project) => project,
5348 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5349 };
5350 timings.enter(RouteOpenPhase::Admission);
5351 let project_root = match legacy_project.effective_root(scope_stamp.as_ref()) {
5352 Ok(root) => root,
5353 Err(err) => {
5354 return Ok(vec![control_error_frame(
5355 &frame,
5356 "invalid_project_root",
5357 err.to_string(),
5358 )?])
5359 }
5360 };
5361 let legacy_bind_project = legacy_project.for_target(scope_stamp.as_ref());
5367
5368 let _concurrency_guard = match self
5379 .route_bind_concurrency
5380 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
5381 {
5382 Ok(guard) => guard,
5383 Err(in_flight) => {
5384 return Ok(vec![self.route_open_target_capacity_refusal(
5385 ctx,
5386 &caller,
5387 &frame,
5388 &target_module_id,
5389 in_flight,
5390 )?]);
5391 }
5392 };
5393
5394 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
5397 RouteBindAdmission::Admitted { guard, probe } => {
5398 if probe {
5399 info!(
5400 module_id = %target_module_id,
5401 connection_id = ctx.connection_id.get(),
5402 "route.bind breaker half-open: admitting one probe"
5403 );
5404 }
5405 guard
5406 }
5407 RouteBindAdmission::Refused {
5408 consecutive_timeouts,
5409 retry_in,
5410 probe_in_flight,
5411 } => {
5412 return Ok(vec![self.route_open_breaker_refusal_frame(
5413 ctx,
5414 &caller,
5415 &frame,
5416 &target_module_id,
5417 consecutive_timeouts,
5418 retry_in,
5419 probe_in_flight,
5420 )?]);
5421 }
5422 };
5423
5424 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
5429 let relay_deadline = Instant::now() + route_bind_relay_timeout;
5430 let operator_app = self
5431 .operator_apps
5432 .lock()
5433 .unwrap_or_else(|p| p.into_inner())
5434 .get(&ctx.connection_id)
5435 .cloned();
5436 timings.enter(RouteOpenPhase::EgressReservation);
5437 let pending = match self
5438 .forwarding
5439 .begin_route_bind_relay_for(
5440 ctx.connection_id,
5441 ctx.egress.clone(),
5442 response_version(&frame),
5443 frame.header.corr,
5444 &target_module_id,
5445 principal.clone(),
5446 bound_scope,
5447 project_root,
5448 identity.harness.clone(),
5449 build_class,
5450 operator_app.clone(),
5451 relay_deadline,
5452 busy_refusals == Some(true),
5453 )
5454 .await
5455 {
5456 Ok(pending) => pending,
5457 Err(err) => {
5458 return Ok(vec![self.route_open_refusal_frame(
5459 ctx,
5460 &caller,
5461 &frame,
5462 &target_module_id,
5463 "relay_reservation_failed",
5464 forwarding_error_code(&err),
5465 err.to_string(),
5466 )?])
5467 }
5468 };
5469 timings.enter(RouteOpenPhase::Admission);
5470 let crate::forwarding::PendingRouteBindRelay {
5471 endpoint,
5472 module_sink,
5473 negotiated_ver,
5474 client_channel,
5475 client_epoch,
5476 module_channel,
5477 module_epoch,
5478 corr: relay_corr,
5479 receiver,
5480 } = pending;
5481 let mut reservation =
5482 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
5483
5484 let relay_registration = self
5489 .registry
5490 .get_module_by_connection(endpoint.connection_id)
5491 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5492 let relay_capabilities = relay_registration
5493 .as_ref()
5494 .and_then(|registration| registration.manifest.capabilities.as_ref());
5495 if let Some(selector) = &scope {
5496 if let Err(refusal) = self
5499 .scopes
5500 .read()
5501 .unwrap_or_else(|poisoned| poisoned.into_inner())
5502 .check_bind_proof(&principal, selector)
5503 {
5504 reservation.release_and_disarm();
5505 return Ok(vec![self.route_open_refusal_frame(
5506 ctx,
5507 &caller,
5508 &frame,
5509 &target_module_id,
5510 refusal.code,
5511 refusal.code,
5512 refusal.message,
5513 )?]);
5514 }
5515 }
5516 if let Some(stamp) = scope_stamp
5517 .as_ref()
5518 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5519 {
5520 if let Err(refusal) = crate::scopes::check_target_flow_support(
5521 stamp,
5522 &target_module_id,
5523 relay_capabilities,
5524 )
5525 .and_then(|()| {
5526 crate::scopes::check_target_agent_run_support(
5527 stamp,
5528 &target_module_id,
5529 relay_capabilities,
5530 )
5531 }) {
5532 reservation.release_and_disarm();
5533 return Ok(vec![self.route_open_refusal_frame(
5534 ctx,
5535 &caller,
5536 &frame,
5537 &target_module_id,
5538 refusal.code,
5539 refusal.code,
5540 refusal.message,
5541 )?]);
5542 }
5543 }
5544
5545 if !relay_capabilities.is_some_and(|caps| {
5548 caps.provides
5549 .iter()
5550 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5551 }) {
5552 if let Some(stamp) = scope_stamp.as_mut() {
5553 stamp.workdir = None;
5554 stamp.project_id = None;
5555 }
5556 }
5557 let worker_run = scope_stamp
5562 .as_ref()
5563 .is_some_and(crate::scopes::is_worker_run)
5564 && relay_capabilities.is_some_and(|caps| {
5565 caps.provides
5566 .iter()
5567 .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5568 });
5569 if !worker_run
5570 && !relay_capabilities.is_some_and(|caps| {
5571 caps.provides
5572 .iter()
5573 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5574 })
5575 {
5576 if let Some(stamp) = scope_stamp.as_mut() {
5577 stamp.attributes.on_behalf_of_agent = None;
5578 }
5579 }
5580 if !relay_capabilities.is_some_and(|caps| {
5581 caps.provides
5582 .iter()
5583 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5584 }) {
5585 if let Some(stamp) = scope_stamp.as_mut() {
5586 stamp.attributes.flow_version = None;
5587 stamp.attributes.flow_code_hash = None;
5588 stamp.attributes.flow_install_id = None;
5589 }
5590 }
5591 let call_mode = if worker_run
5592 || relay_capabilities
5593 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5594 {
5595 Some(derive_call_mode(scope_stamp.as_ref()))
5596 } else {
5597 if let Some(stamp) = scope_stamp.as_mut() {
5598 stamp.expires_at_ms = None;
5599 }
5600 None
5601 };
5602
5603 debug!(
5604 connection_id = ctx.connection_id.get(),
5605 client_channel,
5606 client_epoch,
5607 module_channel,
5608 module_epoch,
5609 "reserved route handle pair"
5610 );
5611 let route_principal_label = principal_label(&principal);
5614 let relay = ModuleControlRequest::RouteBind {
5615 route_channel: module_channel,
5616 epoch: module_epoch,
5617 target,
5618 identity,
5619 principal: Some(principal),
5620 consumer_capabilities,
5621 role_versions,
5622 admission_facts,
5623 scope: scope_stamp,
5624 call_mode,
5625 };
5626 let peer = RouteBindPeer::new()
5627 .with_observed_peer(
5628 relay_capabilities
5629 .is_some_and(|caps| {
5630 caps.provides
5631 .iter()
5632 .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
5633 })
5634 .then(|| {
5635 self.observed_peers
5636 .lock()
5637 .unwrap_or_else(|p| p.into_inner())
5638 .get(&ctx.connection_id)
5639 .cloned()
5640 .unwrap_or_else(ObservedPeer::unobserved)
5641 }),
5642 )
5643 .with_operator_app(
5644 self.connection_operator_app(operator_app.as_ref())
5645 .filter(|_| {
5646 relay_capabilities.is_some_and(|caps| {
5647 caps.provides
5648 .iter()
5649 .any(|cap| cap == OPERATOR_APP_CAPABILITY)
5650 })
5651 }),
5652 );
5653 let relay_body = crate::bind_compat::encode_bind(
5654 &relay,
5655 &peer,
5656 &legacy_bind_project,
5657 relay_capabilities,
5658 )
5659 .map_err(|err| {
5660 RouterError::backend(
5661 0,
5662 frame.header.corr,
5663 format!("failed to encode route.bind request: {err}"),
5664 )
5665 })?;
5666 let relay_frame = Frame::build_with_version(
5667 negotiated_ver,
5668 FrameType::Request,
5669 control_flags(),
5670 0,
5671 0,
5672 relay_corr,
5673 relay_body,
5674 )
5675 .map_err(RouterError::FrameBuild)?;
5676
5677 timings.enter(RouteOpenPhase::BindSend);
5678 let sent = module_sink.send(relay_frame).await;
5679 timings.enter(RouteOpenPhase::Admission);
5680 if let Err(err) = sent {
5681 reservation.release_and_disarm();
5682 return Ok(vec![self.route_open_refusal_frame(
5683 ctx,
5684 &caller,
5685 &frame,
5686 &target_module_id,
5687 "relay_send_failed",
5688 "target_unavailable",
5689 err.to_string(),
5690 )?]);
5691 }
5692
5693 #[cfg(test)]
5694 if let Some(hold) = &self.route_bind_relay_sent_hold {
5695 hold.wait().await;
5699 hold.wait().await;
5700 }
5701
5702 if self
5707 .forwarding
5708 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5709 .map_err(RouterError::Forwarding)?
5710 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5711 {
5712 self.send_abandoned_route_bind_goodbye(
5713 &module_sink,
5714 negotiated_ver,
5715 module_channel,
5716 module_epoch,
5717 );
5718 }
5719
5720 timings.enter(RouteOpenPhase::BindAckWait);
5721 let outcome = timeout_at(relay_deadline, receiver).await;
5722 timings.enter(RouteOpenPhase::Admission);
5723 match outcome {
5724 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5725 reservation.disarm();
5726 if breaker.record_accepted() {
5727 info!(
5728 module_id = %target_module_id,
5729 "route.bind breaker closed: the probe was accepted"
5730 );
5731 }
5732 self.observe_route_open_accept(
5733 ctx,
5734 &target_module_id,
5735 &route_principal_label,
5736 &caller.scope,
5737 );
5738 Ok(Vec::new())
5739 }
5740 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5741 reservation.release_and_disarm();
5742 breaker.record_inconclusive();
5746 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
5747 }
5748 Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
5749 reservation.release_and_disarm();
5750 breaker.record_inconclusive();
5751 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
5752 }
5753 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5754 reservation.release_and_disarm();
5755 breaker.record_inconclusive();
5756 tracing::warn!(
5763 module_id = %target_module_id,
5764 "route.bind relay abandoned: {message}"
5765 );
5766 Ok(vec![self.route_open_refusal_frame(
5767 ctx,
5768 &caller,
5769 &frame,
5770 &target_module_id,
5771 "relay_abandoned",
5772 "target_unavailable",
5773 message,
5774 )?])
5775 }
5776 Ok(Err(_)) => {
5777 reservation.release_and_disarm();
5778 breaker.record_inconclusive();
5779 Ok(vec![self.route_open_refusal_frame(
5780 ctx,
5781 &caller,
5782 &frame,
5783 &target_module_id,
5784 "relay_waiter_canceled",
5785 "target_unavailable",
5786 "route.bind relay waiter was canceled before the module responded",
5787 )?])
5788 }
5789 Err(_) => {
5790 reservation.release_and_disarm();
5791 if let Some(opened) = breaker.record_timeout(
5795 self.route_bind_breaker_threshold,
5796 self.route_bind_breaker_cooldown,
5797 ) {
5798 warn!(
5799 module_id = %target_module_id,
5800 consecutive_timeouts = opened.consecutive_timeouts,
5801 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5802 reopened_after_probe = opened.reopened_after_probe,
5803 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5804 );
5805 }
5806 tracing::warn!(
5814 module_id = %target_module_id,
5815 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5816 "route.bind relay timed out: module did not ack within budget"
5817 );
5818 Ok(vec![self.route_open_refusal_frame(
5819 ctx,
5820 &caller,
5821 &frame,
5822 &target_module_id,
5823 "relay_timed_out",
5824 "module_timeout",
5825 format!(
5826 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5827 route_bind_relay_timeout
5828 ),
5829 )?])
5830 }
5831 }
5832 }
5833
5834 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5835 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5836 snapshot: self.supervisor.spawn_snapshot(),
5837 };
5838 Ok(vec![control_response_body_frame(
5839 &frame,
5840 &response,
5841 "ClientControlResponse::SupervisorSpawnSnapshot",
5842 )?])
5843 }
5844
5845 fn handle_supervisor_spawn_subscribe(
5846 &self,
5847 ctx: &RouteCtx,
5848 frame: Frame,
5849 since: Option<SpawnCursor>,
5850 ) -> Result<Vec<Frame>, RouterError> {
5851 match self.supervisor.subscribe_spawns(
5852 ctx.connection_id,
5853 frame.header.corr,
5854 response_version(&frame),
5855 since,
5856 ctx.egress.clone(),
5857 ) {
5858 Ok(()) => Ok(Vec::new()),
5859 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5860 Ok(vec![control_error_body_frame(
5861 &frame,
5862 ErrorBody {
5863 code: "spawn_cursor_incarnation_mismatch".to_string(),
5864 message: "spawn cursor belongs to a different daemon incarnation"
5865 .to_string(),
5866 detail: Some(serde_json::json!({
5867 "current_daemon_incarnation": current
5868 })),
5869 },
5870 )?])
5871 }
5872 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5873 &frame,
5874 ErrorBody {
5875 code: "spawn_cursor_too_old".to_string(),
5876 message: "spawn cursor predates the retained event ring".to_string(),
5877 detail: Some(serde_json::json!({
5878 "oldest_retained_cursor": oldest
5879 })),
5880 },
5881 )?]),
5882 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5883 0,
5884 frame.header.corr,
5885 format!("failed to open supervisor spawn subscription: {error}"),
5886 )),
5887 }
5888 }
5889
5890 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5891 let generation = self
5892 .registry
5893 .generation()
5894 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5895 let supervised = self.supervisor.list();
5896 let targets = supervised
5897 .iter()
5898 .map(|module| module.subtree_target())
5899 .collect();
5900 let subtree = tokio::spawn(crate::subtree_resources::read(targets));
5901 let mut modules = Vec::new();
5902 for module in supervised {
5903 let status = module.status_for_control("list").map_err(|err| {
5904 RouterError::backend(
5905 0,
5906 frame.header.corr,
5907 format!("failed to read supervisor status: {err}"),
5908 )
5909 })?;
5910 let (configured, _) = module.configuration().map_err(|err| {
5911 RouterError::backend(
5912 0,
5913 frame.header.corr,
5914 format!("failed to read module configuration: {err}"),
5915 )
5916 })?;
5917 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
5918 RouterError::backend(
5919 0,
5920 frame.header.corr,
5921 format!("failed to read retry time: {err}"),
5922 )
5923 })?;
5924 let image = module.running_image_agreement().await;
5926 let resources = Some(crate::subtree_resources::read_process(module.clone()).await);
5929 let mut verdict =
5930 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5931 verdict.spec = Some(launch_spec_agreement(
5932 &configured,
5933 status.process_alive,
5934 status.launch_spec_digest.as_ref(),
5935 ));
5936 let pending_reload = Some(verdict);
5937 let configured_enabled = self
5938 .capability_evaluator
5939 .configured_enabled(&status.module_id);
5940 modules.push(
5943 SupervisorEntry::new(
5944 status.module_id,
5945 status.state.to_string(),
5946 status.enabled,
5947 status.live,
5948 status.health.status,
5949 )
5950 .with_configured_enabled(configured_enabled)
5951 .with_next_retry_at_ms(next_retry_at_ms)
5952 .with_launch_nonce_env(Some(!cfg!(unix)))
5953 .with_launch_nonce_source(status.launch_nonce_source)
5954 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5955 .with_protocol(status.protocol)
5956 .with_pending_reload(pending_reload)
5957 .with_last_probe_ms(status.health.last_probe_ms)
5958 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5959 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5960 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5961 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5962 .with_restart_count(Some(status.restart_count))
5963 .with_max_restarts(Some(status.max_restarts))
5964 .with_lifetime_restarts(Some(status.lifetime_restarts))
5965 .with_spawn_generation(Some(status.spawn_generation))
5966 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5967 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5968 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5969 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5970 .with_resources(resources),
5971 );
5972 }
5973 let readings = subtree
5974 .await
5975 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5976 for (module, reading) in modules.iter_mut().zip(readings) {
5977 if reading.reason.as_deref() == Some("timeout") {
5978 crate::subtree_resources::log_timeout(&module.module_id, "subtree");
5979 }
5980 module.subtree = Some(reading);
5981 }
5982 let response = ClientControlResponse::SupervisorList {
5983 generation,
5984 modules,
5985 };
5986 Ok(vec![control_response_body_frame(
5987 &frame,
5988 &response,
5989 "ClientControlResponse::SupervisorList",
5990 )?])
5991 }
5992
5993 fn handle_supervisor_stderr_tail(
5994 &self,
5995 frame: Frame,
5996 module_id: String,
5997 max_lines: Option<u32>,
5998 max_bytes: Option<u32>,
5999 ) -> Result<Vec<Frame>, RouterError> {
6000 let Some(module) = self.supervisor.get(&module_id) else {
6001 return Ok(vec![control_error_frame(
6002 &frame,
6003 "unknown_module",
6004 format!("module_id '{module_id}' is not supervised"),
6005 )?]);
6006 };
6007
6008 let snapshot = module.stderr_tail(
6009 max_lines.map(|value| value as usize),
6010 max_bytes.map(|value| value as usize),
6011 );
6012
6013 let response = ClientControlResponse::SupervisorStderrTail {
6014 module_id,
6015 tail: StderrTail {
6016 capture: match snapshot.capture {
6017 CaptureState::Captured => StderrCaptureState::Captured,
6018 CaptureState::Incomplete { reason } => {
6019 StderrCaptureState::Incomplete { reason }
6020 }
6021 CaptureState::NotCaptured { reason } => {
6022 StderrCaptureState::NotCaptured { reason }
6023 }
6024 },
6025 entries: snapshot
6026 .entries
6027 .into_iter()
6028 .map(|entry| match entry {
6029 TailEntry::Line {
6030 text,
6031 truncated,
6032 at_ms,
6033 } => StderrTailEntry::Line {
6034 text,
6035 truncated,
6036 at_ms,
6037 },
6038 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
6039 })
6040 .collect(),
6041 dropped_lines: snapshot.dropped_lines,
6042 },
6043 };
6044 Ok(vec![control_response_body_frame(
6045 &frame,
6046 &response,
6047 "ClientControlResponse::SupervisorStderrTail",
6048 )?])
6049 }
6050
6051 async fn observe_control_wait<F: std::future::Future>(
6052 &self,
6053 ctx: &RouteCtx,
6054 target: Option<&str>,
6055 kind: &'static str,
6056 future: F,
6057 ) -> F::Output {
6058 self.counters
6059 .observe_reader_wait(
6060 || {
6061 let module = self
6062 .forwarding
6063 .module_id_for_connection(ctx.connection_id)
6064 .ok()
6065 .flatten();
6066 let principal = if module.is_some() { "module" } else { "direct" };
6067 (
6068 self.resolve_reader_wait_target(kind, target)
6069 .or(module)
6070 .unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
6071 principal.into(),
6072 )
6073 },
6074 kind,
6075 future,
6076 )
6077 .await
6078 }
6079
6080 pub(crate) fn resolve_reader_wait_target(
6083 &self,
6084 op: &str,
6085 selector: Option<&str>,
6086 ) -> Option<String> {
6087 let selector = selector?;
6088 match op {
6089 ops::SUPERVISOR_TERMINALS | ops::SUPERVISOR_PROVENANCE => self
6090 .supervisor
6091 .get(selector)
6092 .map(|module| module.module_id().to_owned()),
6093 ops::SUPERVISOR_HEALTH_PROBE => self
6094 .registry
6095 .get_module(selector)
6096 .ok()
6097 .flatten()
6098 .map(|registration| registration.manifest.module_id),
6099 _ => None,
6100 }
6101 }
6102
6103 async fn handle_supervisor_terminals(
6104 &self,
6105 ctx: &RouteCtx,
6106 frame: Frame,
6107 module_id: String,
6108 ) -> Result<Vec<Frame>, RouterError> {
6109 let Some(module) = self.supervisor.get(&module_id) else {
6110 return Ok(vec![control_error_frame(
6111 &frame,
6112 "unknown_module",
6113 format!("module_id '{module_id}' is not supervised"),
6114 )?]);
6115 };
6116
6117 let terminals = self
6120 .observe_control_wait(
6121 ctx,
6122 Some(module.module_id()),
6123 ops::SUPERVISOR_TERMINALS,
6124 module.read_durable_terminal_history(),
6125 )
6126 .await
6127 .map_err(|error| {
6128 RouterError::backend(
6129 0,
6130 frame.header.corr,
6131 format!("failed to read terminal history: {error}"),
6132 )
6133 })?;
6134 let response = ClientControlResponse::SupervisorTerminals {
6135 module_id,
6136 terminals,
6137 };
6138 Ok(vec![control_response_body_frame(
6139 &frame,
6140 &response,
6141 "ClientControlResponse::SupervisorTerminals",
6142 )?])
6143 }
6144
6145 fn handle_supervisor_routes(
6146 &self,
6147 frame: Frame,
6148 module_id: Option<String>,
6149 ) -> Result<Vec<Frame>, RouterError> {
6150 let modules = self
6151 .forwarding
6152 .route_census(module_id.as_deref())
6153 .map_err(RouterError::Forwarding)?
6154 .into_iter()
6155 .map(|(module_id, routes)| SupervisorRouteModule {
6156 module_id,
6157 routes: routes
6158 .into_iter()
6159 .map(|route| SupervisorRoute {
6160 consumer: match route.principal {
6161 Principal::Reserved { module_id } => {
6162 SupervisorRouteConsumer::Reserved { module_id }
6163 }
6164 Principal::Direct | Principal::Unverified => {
6165 SupervisorRouteConsumer::Direct {
6166 connection_id: route.goodbye_target.connection_id.get(),
6167 }
6168 }
6169 },
6170 age_ms: Instant::now()
6171 .saturating_duration_since(route.bound_at)
6172 .as_millis()
6173 .try_into()
6174 .unwrap_or(u64::MAX),
6175 draining: route.draining,
6176 drain_reason: route.drain_reason,
6177 })
6178 .collect(),
6179 })
6180 .collect();
6181 let response = ClientControlResponse::SupervisorRoutes { modules };
6182 Ok(vec![control_response_body_frame(
6183 &frame,
6184 &response,
6185 "ClientControlResponse::SupervisorRoutes",
6186 )?])
6187 }
6188
6189 async fn handle_supervisor_provenance(
6190 &self,
6191 ctx: &RouteCtx,
6192 frame: Frame,
6193 module_id: Option<String>,
6194 ) -> Result<Vec<Frame>, RouterError> {
6195 let targeted = module_id.is_some();
6196 let selected = if let Some(module_id) = module_id {
6197 let Some(module) = self.supervisor.get(&module_id) else {
6198 return Ok(vec![control_error_frame(
6199 &frame,
6200 "unknown_module",
6201 format!("module_id '{module_id}' is not supervised"),
6202 )?]);
6203 };
6204 vec![module]
6205 } else {
6206 self.supervisor.list()
6207 };
6208
6209 let target = targeted.then(|| selected[0].module_id());
6210 self.observe_control_wait(ctx, target, ops::SUPERVISOR_PROVENANCE, async {
6211 let mut modules = Vec::with_capacity(selected.len());
6212 for module in &selected {
6213 let (status, observed_image) = module
6214 .status_and_running_image_agreement()
6215 .await
6216 .map_err(|err| {
6217 RouterError::backend(
6218 0,
6219 frame.header.corr,
6220 format!("failed to read supervisor status: {err}"),
6221 )
6222 })?;
6223 let module_declared = self
6224 .registry
6225 .get_module(&status.module_id)
6226 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6227 .and_then(|registration| registration.manifest.provenance)
6228 .map(|build| ModuleDeclaredProvenance::Reported { build })
6229 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
6230 #[cfg(test)]
6231 let running_image = match &self.provenance_probe_override {
6232 Some(result) => result.clone(),
6233 None => observed_image,
6234 };
6235 #[cfg(not(test))]
6236 let running_image = observed_image;
6237 modules.push(SupervisorModuleProvenance {
6238 module_id: status.module_id,
6239 module_declared,
6240 daemon_observed: SupervisorObservedProcess {
6241 launch_nonce_source: status.launch_nonce_source,
6242 pid: status.pid,
6243 spawned_at_ms: status.spawned_at_ms,
6244 spawned_from: status.spawned_from,
6245 running_image,
6246 },
6247 });
6248 }
6249 let daemon = SupervisorDaemonProvenance {
6250 daemon_build: self.daemon_provenance.build.clone(),
6251 daemon_observed: DaemonObservedProcess {
6252 pid: self.daemon_provenance.pid,
6253 started_at_ms: self
6254 .daemon_provenance
6255 .start_clock
6256 .map(|clock| clock.started_at_ms())
6257 .or(self.daemon_provenance.started_at_ms),
6258 running_image: self
6259 .daemon_provenance
6260 .probe
6261 .observe(
6262 self.daemon_provenance.pid,
6263 self.daemon_provenance.executable_path.as_deref(),
6264 self.daemon_provenance.executable_identity.clone(),
6265 self.daemon_provenance.process_start_time,
6266 )
6267 .await,
6268 },
6269 };
6270 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
6271 Ok(vec![control_response_body_frame(
6272 &frame,
6273 &response,
6274 "ClientControlResponse::SupervisorProvenance",
6275 )?])
6276 })
6277 .await
6278 }
6279
6280 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6281 self.refresh_capability_requirements();
6282 let generation = self
6283 .registry
6284 .generation()
6285 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6286 let modules = self
6287 .supervisor
6288 .list()
6289 .into_iter()
6290 .map(|module| {
6291 let status = module.status_for_control("health").map_err(|err| {
6292 RouterError::backend(
6293 0,
6294 frame.header.corr,
6295 format!("failed to read supervisor health: {err}"),
6296 )
6297 })?;
6298 let module_id = status.module_id;
6299 let capability_detail = self
6300 .capability_evaluator
6301 .required_problem_detail(&module_id);
6302 Ok(SupervisorHealthEntry {
6303 module_id,
6304 status: status.health.status,
6305 detail: append_capability_problem_detail(
6306 status.health.detail,
6307 capability_detail,
6308 ),
6309 metrics: status.health.metrics,
6310 consecutive_failures: status.health.consecutive_failures,
6311 late_answer_count: status.health.late_answer_count,
6312 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
6313 last_action: status.health.last_action,
6314 last_action_ms: status.health.last_action_ms,
6315 last_probe_ms: status.health.last_probe_ms,
6316 })
6317 })
6318 .collect::<Result<Vec<_>, RouterError>>()?;
6319 let response = ClientControlResponse::SupervisorHealth {
6320 generation,
6321 modules,
6322 };
6323 Ok(vec![control_response_body_frame(
6324 &frame,
6325 &response,
6326 "ClientControlResponse::SupervisorHealth",
6327 )?])
6328 }
6329
6330 async fn handle_supervisor_restart(
6331 &self,
6332 frame: Frame,
6333 module_id: String,
6334 drain_timeout_ms: Option<u64>,
6335 requester: Principal,
6336 ) -> Result<Vec<Frame>, RouterError> {
6337 let operation_lock = self.supervisor.operation_lock();
6338 let _operation_guard = operation_lock.lock().await;
6339 let Some(module) = self.supervisor.get(&module_id) else {
6340 return Ok(vec![control_error_frame(
6341 &frame,
6342 "unknown_module",
6343 format!("module_id '{module_id}' is not supervised"),
6344 )?]);
6345 };
6346
6347 self.route_outages.mark_operator_action(&module_id);
6348 if let Err(err) = module
6349 .restart_requested(drain_timeout_ms, Some(requester))
6350 .await
6351 {
6352 self.route_outages
6353 .operator_action_ended_unrefused(&module_id);
6354 let (code, message) = match err {
6355 crate::supervise::SuperviseError::Disabled { .. } => {
6356 ("module_disabled", err.to_string())
6357 }
6358 crate::supervise::SuperviseError::SwapInProgress { .. } => {
6359 ("swap_in_progress", err.to_string())
6360 }
6361 _ => (
6362 "target_unavailable",
6363 format!("failed to restart module_id '{module_id}': {err}"),
6364 ),
6365 };
6366 return Ok(vec![control_error_frame(&frame, code, message)?]);
6367 }
6368
6369 let response = ClientControlResponse::SupervisorAck {
6370 module_id,
6371 applied: true,
6372 };
6373 Ok(vec![control_response_body_frame(
6374 &frame,
6375 &response,
6376 "ClientControlResponse::SupervisorAck",
6377 )?])
6378 }
6379
6380 async fn handle_supervisor_swap(
6384 &self,
6385 frame: Frame,
6386 module_id: String,
6387 ready_timeout_ms: Option<u64>,
6388 requester: Principal,
6389 ) -> Result<Vec<Frame>, RouterError> {
6390 let module = {
6397 let operation_lock = self.supervisor.operation_lock();
6398 let _operation_guard = operation_lock.lock().await;
6399 self.supervisor.get(&module_id)
6400 };
6401 let Some(module) = module else {
6402 return Ok(vec![control_error_frame(
6403 &frame,
6404 "unknown_module",
6405 format!("module_id '{module_id}' is not supervised"),
6406 )?]);
6407 };
6408
6409 self.route_outages.mark_operator_action(&module_id);
6410 if let Err(err) = module
6411 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
6412 .await
6413 {
6414 self.route_outages
6415 .operator_action_ended_unrefused(&module_id);
6416 use crate::supervise::SuperviseError;
6417 let message = err.to_string();
6418 let error = match err {
6419 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
6420 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
6421 code: "swap_refused".to_string(),
6422 message,
6423 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
6424 },
6425 SuperviseError::SwapFailed {
6426 arm,
6427 candidate_exit,
6428 ..
6429 } => ErrorBody {
6430 code: "swap_failed".to_string(),
6431 message,
6432 detail: Some(serde_json::json!({
6433 "arm": arm.as_str(),
6434 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
6435 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
6436 })),
6437 },
6438 _ => ErrorBody::new(
6439 "target_unavailable",
6440 format!("failed to swap module_id '{module_id}': {message}"),
6441 ),
6442 };
6443 return Ok(vec![control_error_body_frame(&frame, error)?]);
6444 }
6445 self.route_outages
6449 .operator_action_ended_unrefused(&module_id);
6450
6451 let response = ClientControlResponse::SupervisorAck {
6452 module_id,
6453 applied: true,
6454 };
6455 Ok(vec![control_response_body_frame(
6456 &frame,
6457 &response,
6458 "ClientControlResponse::SupervisorAck",
6459 )?])
6460 }
6461
6462 async fn handle_supervisor_reload(
6463 &self,
6464 frame: Frame,
6465 module_id: String,
6466 requester: Principal,
6467 ) -> Result<Vec<Frame>, RouterError> {
6468 let operation_lock = self.supervisor.operation_lock();
6469 let _operation_guard = operation_lock.lock().await;
6470 let Some(module) = self.supervisor.get(&module_id) else {
6471 return Ok(vec![control_error_frame(
6472 &frame,
6473 "unknown_module",
6474 format!("module_id '{module_id}' is not supervised"),
6475 )?]);
6476 };
6477
6478 self.route_outages.mark_operator_action(&module_id);
6479 if let Err(err) = module.reload_requested(Some(requester)).await {
6480 self.route_outages
6481 .operator_action_ended_unrefused(&module_id);
6482 let (code, message) = match err {
6483 crate::supervise::SuperviseError::Disabled { .. } => {
6484 ("module_disabled", err.to_string())
6485 }
6486 crate::supervise::SuperviseError::SwapInProgress { .. } => {
6487 ("swap_in_progress", err.to_string())
6488 }
6489 _ => (
6490 "reload_failed",
6491 format!("failed to reload module_id '{module_id}': {err}"),
6492 ),
6493 };
6494 return Ok(vec![control_error_frame(&frame, code, message)?]);
6495 }
6496
6497 let response = ClientControlResponse::SupervisorAck {
6498 module_id,
6499 applied: true,
6500 };
6501 Ok(vec![control_response_body_frame(
6502 &frame,
6503 &response,
6504 "ClientControlResponse::SupervisorAck",
6505 )?])
6506 }
6507
6508 async fn handle_supervisor_rescan(
6512 &self,
6513 frame: Frame,
6514 mode: RescanMode,
6515 requester: Principal,
6516 ) -> Result<Vec<Frame>, RouterError> {
6517 let preview = mode.previews();
6518 let Some(context) = self.rescan.clone() else {
6519 return Ok(vec![control_error_frame(
6520 &frame,
6521 "rescan_unavailable",
6522 "the daemon was not started with a reloadable config path".to_string(),
6523 )?]);
6524 };
6525
6526 let operation_lock = self.supervisor.operation_lock();
6527 let _operation_guard = operation_lock.lock().await;
6528 let loaded = match self.load_rescan_config(&mode, &context) {
6532 Ok(config) => config,
6533 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
6534 };
6535 let Some((config, activity_config)) = loaded else {
6543 return Ok(vec![control_error_frame(
6544 &frame,
6545 "invalid_daemon_config",
6546 format!(
6547 "daemon config not found at {}; refusing to rescan (an absent config would \
6548 retire every supervised module)",
6549 context.config_path.display()
6550 ),
6551 )?]);
6552 };
6553 let (
6554 configured_port,
6555 storage_config,
6556 admission_facts_carrier_module_id,
6557 admission_facts_targets,
6558 scope_authority_owners,
6559 privileged_operator_prompt,
6560 modules,
6561 reserved_capabilities,
6562 ) = (
6563 config.port,
6564 config.storage,
6565 config.admission_facts_carrier_module_id,
6566 config.admission_facts_targets,
6567 config.scope_authority_owners,
6568 config.privileged_operator_prompt,
6569 config.modules,
6570 config.reserved_capabilities,
6571 );
6572
6573 let mut restart_required = Vec::new();
6582 for section in RestartRequiredSection::ALL {
6583 let changed = match section {
6584 RestartRequiredSection::Port => configured_port != context.configured_port,
6585 RestartRequiredSection::Storage => storage_config != context.storage_config,
6586 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6587 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6588 }
6589 RestartRequiredSection::AdmissionFactsTargets => {
6590 admission_facts_targets != context.admission_facts_targets
6591 }
6592 RestartRequiredSection::ScopeAuthorityOwners => {
6593 scope_authority_owners != context.scope_authority_owners
6594 }
6595 RestartRequiredSection::PrivilegedOperatorPrompt => {
6596 privileged_operator_prompt != context.privileged_operator_prompt
6597 }
6598 };
6599 if changed {
6600 restart_required.push(section.label().to_string());
6601 }
6602 }
6603 if !restart_required.is_empty() {
6604 warn!(
6605 config_path = %context.config_path.display(),
6606 sections = %restart_required.join(", "),
6607 "daemon config changed outside the modules section; restart the daemon to apply those changes"
6608 );
6609 }
6610
6611 for configured in &modules {
6612 if let Err(err) = validate_spec(&configured.module_spec()) {
6613 return Ok(vec![control_error_frame(
6614 &frame,
6615 "invalid_daemon_config",
6616 format!("supervisor rescan rejected daemon config: {err}"),
6617 )?]);
6618 }
6619 }
6620
6621 let configured_capabilities = modules
6622 .iter()
6623 .map(|module| (module.module_id.clone(), module.enabled))
6624 .collect::<Vec<_>>();
6625 let preview_capability_warnings = if preview {
6626 let (_, registrations) = self.runtime_capability_snapshot()?;
6627 let current_modules = self
6628 .supervisor
6629 .list()
6630 .into_iter()
6631 .map(|module| module.module_id().to_string())
6632 .collect::<BTreeSet<_>>();
6633 let resulting_modules = configured_capabilities.clone();
6634 let removed = current_modules
6635 .into_iter()
6636 .filter(|module_id| {
6637 !resulting_modules
6638 .iter()
6639 .any(|(configured_id, _)| configured_id == module_id)
6640 })
6641 .collect::<Vec<_>>();
6642 let disabled = match &mode {
6645 RescanMode::Candidate(_) => resulting_modules
6646 .iter()
6647 .filter(|(_, enabled)| !enabled)
6648 .map(|(module_id, _)| module_id.clone())
6649 .collect::<Vec<_>>(),
6650 _ => Vec::new(),
6651 };
6652 self.capability_evaluator.preview_warnings(
6653 resulting_modules,
6654 &removed,
6655 &disabled,
6656 ®istrations,
6657 )
6658 } else {
6659 Vec::new()
6660 };
6661 let reconciled = match &mode {
6662 RescanMode::Candidate(candidate) => self
6663 .preview_candidate(&context, modules, candidate)
6664 .map(|result| (result, Vec::new())),
6665 _ => {
6666 self.reconcile_supervised_modules(
6667 &context.supervisor,
6668 modules,
6669 preview,
6670 requester.clone(),
6671 )
6672 .await
6673 }
6674 };
6675 let (result, changed) = match reconciled {
6676 Ok(result) => result,
6677 Err(message) => {
6678 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6679 }
6680 };
6681 if !preview {
6682 if let Some(ledger) = &self.activity {
6683 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6684 ledger.config_changed(&result.added, &result.removed, &changed, requester);
6685 }
6686 if let Err(code) = self.configure_activity(activity_config).await {
6687 warn!(
6688 code,
6689 "activity retention could not be applied during rescan"
6690 );
6691 }
6692 self.capability_evaluator
6693 .configure(configured_capabilities, reserved_capabilities);
6694 self.capability_evaluator.wake_deadline_loop();
6695 self.refresh_capability_requirements();
6696 }
6697 let mut result = result;
6698 result.restart_required = restart_required;
6699 result.capability_warnings = preview_capability_warnings;
6700 let response = ClientControlResponse::SupervisorRescan { result };
6701 Ok(vec![control_response_body_frame(
6702 &frame,
6703 &response,
6704 "ClientControlResponse::SupervisorRescan",
6705 )?])
6706 }
6707
6708 async fn handle_supervisor_release_reserved(
6709 &self,
6710 frame: Frame,
6711 module_id: String,
6712 ) -> Result<Vec<Frame>, RouterError> {
6713 let Some(context) = self.rescan.clone() else {
6714 return Ok(vec![control_error_frame(
6715 &frame,
6716 "release_unavailable",
6717 "reserved-id release requires a daemon started with a reloadable config path",
6718 )?]);
6719 };
6720 let operation_lock = self.supervisor.operation_lock();
6721 let _operation_guard = operation_lock.lock().await;
6722 let loaded = match crate::daemon_config::load(&context.config_path) {
6723 Ok(Some(config)) => config,
6724 Ok(None) => {
6725 return Ok(vec![control_error_frame(
6726 &frame,
6727 "invalid_daemon_config",
6728 format!(
6729 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6730 context.config_path.display()
6731 ),
6732 )?])
6733 }
6734 Err(err) => {
6735 return Ok(vec![control_error_frame(
6736 &frame,
6737 "invalid_daemon_config",
6738 format!("unable to verify reserved-id release against daemon config: {err}"),
6739 )?])
6740 }
6741 };
6742 if loaded
6743 .modules
6744 .iter()
6745 .any(|configured| configured.module_id == module_id)
6746 {
6747 return Ok(vec![control_error_frame(
6748 &frame,
6749 "reserved_module_configured",
6750 format!(
6751 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6752 ),
6753 )?]);
6754 }
6755 if !self.supervisor.release_retained_reserved_gate(&module_id) {
6756 return Ok(vec![control_error_frame(
6757 &frame,
6758 "reserved_gate_not_retained",
6759 format!(
6760 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6761 ),
6762 )?]);
6763 }
6764
6765 let response = ClientControlResponse::SupervisorAck {
6766 module_id,
6767 applied: true,
6768 };
6769 Ok(vec![control_response_body_frame(
6770 &frame,
6771 &response,
6772 "ClientControlResponse::SupervisorAck",
6773 )?])
6774 }
6775
6776 fn plan_rescan(
6780 &self,
6781 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6782 ) -> Result<RescanPlan, String> {
6783 let mut current = BTreeMap::new();
6784 for module in self.supervisor.list() {
6785 let (spec, health) = module.configuration().map_err(|err| {
6786 format!(
6787 "failed to read configuration for module_id '{}': {err}",
6788 module.module_id()
6789 )
6790 })?;
6791 let enabled = module
6792 .status()
6793 .map_err(|err| {
6794 format!(
6795 "failed to read status for module_id '{}': {err}",
6796 module.module_id()
6797 )
6798 })?
6799 .enabled;
6800 current.insert(
6801 module.module_id().to_string(),
6802 (module, spec, health, enabled),
6803 );
6804 }
6805 let configured = configured_modules
6806 .into_iter()
6807 .map(|module| (module.module_id.clone(), module))
6808 .collect::<BTreeMap<_, _>>();
6809
6810 let added = configured
6811 .keys()
6812 .filter(|module_id| !current.contains_key(*module_id))
6813 .cloned()
6814 .collect::<Vec<_>>();
6815 let removed = current
6816 .keys()
6817 .filter(|module_id| !configured.contains_key(*module_id))
6818 .cloned()
6819 .collect::<Vec<_>>();
6820 let mut changed_pending_reload = Vec::new();
6821 let mut configuration_changes = BTreeSet::new();
6822 let mut enabled_changes = BTreeSet::new();
6823 let mut unchanged = 0_u32;
6824
6825 for (module_id, configured_module) in &configured {
6826 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6827 else {
6828 continue;
6829 };
6830 let launch_changed = *current_spec != configured_module.module_spec();
6834 let configuration_changed =
6835 launch_changed || *current_health != configured_module.health;
6836 let enabled_changed = *current_enabled != configured_module.enabled
6839 || self
6840 .capability_evaluator
6841 .configured_enabled(module_id)
6842 .is_some_and(|enabled| enabled != configured_module.enabled);
6843 if configuration_changed {
6844 configuration_changes.insert(module_id.clone());
6845 }
6846 if launch_changed {
6847 changed_pending_reload.push(module_id.clone());
6848 }
6849 if enabled_changed {
6850 enabled_changes.insert(module_id.clone());
6851 }
6852 if !configuration_changed && !enabled_changed {
6853 unchanged = unchanged.saturating_add(1);
6854 }
6855 }
6856 Ok(RescanPlan {
6857 current,
6858 configured,
6859 added,
6860 removed,
6861 changed_pending_reload,
6862 configuration_changes,
6863 enabled_changes,
6864 unchanged,
6865 })
6866 }
6867
6868 async fn reconcile_supervised_modules(
6877 &self,
6878 supervisor: &Supervisor,
6879 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6880 preview: bool,
6881 requester: Principal,
6882 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6883 let plan = self.plan_rescan(configured_modules)?;
6884
6885 if preview {
6891 return Ok((plan.preview_result(), Vec::new()));
6892 }
6893 let changed = plan
6894 .configuration_changes
6895 .union(&plan.enabled_changes)
6896 .cloned()
6897 .collect::<Vec<_>>();
6898 let RescanPlan {
6899 current,
6900 configured,
6901 added,
6902 removed,
6903 changed_pending_reload,
6904 configuration_changes,
6905 enabled_changes,
6906 unchanged,
6907 } = plan;
6908 if !(added.is_empty()
6913 && removed.is_empty()
6914 && configuration_changes.is_empty()
6915 && enabled_changes.is_empty())
6916 {
6917 if let Some(context) = &self.rescan {
6918 context
6919 .generation
6920 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6921 }
6922 }
6923
6924 for module_id in &removed {
6925 let module = ¤t
6926 .get(module_id)
6927 .expect("removed module came from current supervisor state")
6928 .0;
6929 module
6930 .retire_requested(Some(requester.clone()))
6931 .await
6932 .map_err(|err| {
6933 format!("failed to retire module_id '{module_id}' during rescan: {err}")
6934 })?;
6935 self.supervisor.record_rescan_removal(module_id);
6969 self.supervisor.retire(module_id);
6970 self.route_outages.forget(module_id);
6971 }
6972
6973 for module_id in configured.keys() {
6974 let Some((module, _, _, current_enabled)) = current.get(module_id) else {
6975 continue;
6976 };
6977 let configured_module = configured
6978 .get(module_id)
6979 .expect("configured module id came from configured map");
6980 if configuration_changes.contains(module_id) {
6981 module
6982 .update_configuration(
6983 configured_module.module_spec(),
6984 configured_module.health.clone(),
6985 configured_module.drain_timeout_ms,
6986 )
6987 .await
6988 .map_err(|err| {
6989 format!(
6990 "failed to update module_id '{module_id}' configuration during rescan: {err}"
6991 )
6992 })?;
6993 }
6994 if *current_enabled != configured_module.enabled {
6997 self.route_outages.mark_operator_action(module_id);
7000 module
7001 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
7002 .await
7003 .map_err(|err| {
7004 self.route_outages.operator_action_ended_unrefused(module_id);
7005 format!(
7006 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
7007 configured_module.enabled
7008 )
7009 })?;
7010 }
7011 }
7012
7013 for module_id in &added {
7014 let configured_module = configured
7015 .get(module_id)
7016 .expect("added module id came from configured map");
7017 supervisor
7018 .supervise_configured_requested(
7019 configured_module.module_spec(),
7020 configured_module.enabled,
7021 configured_module.health.clone(),
7022 configured_module.drain_timeout_ms,
7023 configured_module.restart,
7024 configured_module.register_timeout,
7025 Some(requester.clone()),
7026 )
7027 .map_err(|err| {
7028 format!("failed to add module_id '{module_id}' during rescan: {err}")
7029 })?;
7030 }
7031
7032 Ok((
7033 SupervisorRescanResult {
7034 added,
7035 removed,
7036 changed_pending_reload,
7037 enabled_changes: enabled_changes.iter().cloned().collect(),
7038 unchanged,
7039 preview: false,
7040 restart_required: Vec::new(),
7043 capability_warnings: Vec::new(),
7044 module_actions: Vec::new(),
7045 earlier_pending: Vec::new(),
7046 expect: None,
7047 },
7048 changed,
7049 ))
7050 }
7051
7052 async fn handle_supervisor_set_enabled(
7053 &self,
7054 frame: Frame,
7055 module_id: String,
7056 enabled: bool,
7057 requester: Principal,
7058 ) -> Result<Vec<Frame>, RouterError> {
7059 let operation_lock = self.supervisor.operation_lock();
7060 let _operation_guard = operation_lock.lock().await;
7061 let Some(module) = self.supervisor.get(&module_id) else {
7062 return Ok(vec![control_error_frame(
7063 &frame,
7064 "unknown_module",
7065 format!("module_id '{module_id}' is not supervised"),
7066 )?]);
7067 };
7068
7069 self.route_outages.mark_operator_action(&module_id);
7072 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
7073 Ok(applied) => applied,
7074 Err(err) => {
7075 self.route_outages
7076 .operator_action_ended_unrefused(&module_id);
7077 return Ok(vec![control_error_frame(
7078 &frame,
7079 "target_unavailable",
7080 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
7081 )?]);
7082 }
7083 };
7084 if !applied {
7085 self.route_outages
7088 .operator_action_ended_unrefused(&module_id);
7089 }
7090
7091 self.capability_evaluator.wake_deadline_loop();
7092 self.refresh_capability_requirements();
7093 let response = ClientControlResponse::SupervisorAck { module_id, applied };
7094 Ok(vec![control_response_body_frame(
7095 &frame,
7096 &response,
7097 "ClientControlResponse::SupervisorAck",
7098 )?])
7099 }
7100
7101 async fn handle_supervisor_health_probe(
7102 &self,
7103 ctx: &RouteCtx,
7104 frame: Frame,
7105 module_id: String,
7106 ) -> Result<Vec<Frame>, RouterError> {
7107 self.refresh_capability_requirements();
7108 let Some(registration) = self
7109 .registry
7110 .get_module(&module_id)
7111 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
7112 else {
7113 return Ok(vec![control_error_frame(
7114 &frame,
7115 "unknown_module",
7116 format!("module_id '{module_id}' is not registered"),
7117 )?]);
7118 };
7119
7120 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
7133 {
7134 return Ok(vec![control_error_frame(
7135 &frame,
7136 "health_not_advertised",
7137 format!("module_id '{module_id}' did not advertise health.check"),
7138 )?]);
7139 }
7140
7141 self.observe_control_wait(
7142 ctx,
7143 Some(®istration.manifest.module_id),
7144 ops::SUPERVISOR_HEALTH_PROBE,
7145 async {
7146 let deadline = Instant::now() + self.health_probe_timeout;
7147 let pending = match self.forwarding.begin_module_control_rpc_for(
7148 &module_id,
7149 MODULE_CONTROL_OP_HEALTH_CHECK,
7150 deadline,
7151 ) {
7152 Ok(pending) => pending,
7153 Err(err) => {
7154 return Ok(vec![control_error_frame(
7155 &frame,
7156 forwarding_error_code(&err),
7157 err.to_string(),
7158 )?])
7159 }
7160 };
7161
7162 let PendingModuleControlRpc {
7163 endpoint,
7164 module_sink,
7165 negotiated_ver,
7166 corr: probe_corr,
7167 receiver,
7168 } = pending;
7169 let mut guard =
7170 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
7171 let probe_body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {})
7172 .map_err(|err| {
7173 RouterError::backend(
7174 0,
7175 frame.header.corr,
7176 format!("failed to encode health.check request: {err}"),
7177 )
7178 })?;
7179 let probe_frame = Frame::build_with_version(
7180 negotiated_ver,
7181 FrameType::Request,
7182 control_flags(),
7183 0,
7184 0,
7185 probe_corr,
7186 probe_body,
7187 )
7188 .map_err(RouterError::FrameBuild)?;
7189
7190 match timeout_at(deadline, module_sink.send(probe_frame)).await {
7193 Ok(Ok(())) => {}
7194 Ok(Err(err)) => {
7195 return Ok(vec![control_error_frame(
7196 &frame,
7197 "target_unavailable",
7198 err.to_string(),
7199 )?]);
7200 }
7201 Err(_) => {
7202 return Ok(vec![control_error_frame(
7203 &frame,
7204 "module_timeout",
7205 format!(
7206 "module_id '{module_id}' did not answer health.check within {:?}",
7207 self.health_probe_timeout
7208 ),
7209 )?]);
7210 }
7211 }
7212
7213 match timeout_at(deadline, receiver).await {
7214 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
7215 guard.disarm();
7216 let Some(report) = response.health_report() else {
7217 return Ok(vec![control_error_frame(
7218 &frame,
7219 "invalid_control_body",
7220 "health.check RPC returned a non-health response",
7221 )?]);
7222 };
7223 let HealthReport {
7228 status,
7229 detail,
7230 metrics,
7231 } = report;
7232 let capability_detail = self
7233 .capability_evaluator
7234 .required_problem_detail(&module_id);
7235 let response = ClientControlResponse::SupervisorHealthProbe {
7236 module_id,
7237 status,
7238 detail: append_capability_problem_detail(detail, capability_detail),
7239 metrics,
7240 };
7241 Ok(vec![control_response_body_frame(
7242 &frame,
7243 &response,
7244 "ClientControlResponse::SupervisorHealthProbe",
7245 )?])
7246 }
7247 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
7248 guard.disarm();
7249 Ok(vec![control_relayed_error_body_frame(&frame, body)?])
7254 }
7255 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
7256 guard.disarm();
7257 Ok(vec![control_error_frame(
7258 &frame,
7259 "target_unavailable",
7260 message,
7261 )?])
7262 }
7263 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
7264 guard.disarm();
7265 Ok(vec![control_error_frame(
7266 &frame,
7267 "invalid_control_body",
7268 message,
7269 )?])
7270 }
7271 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
7272 guard.disarm();
7273 Ok(vec![control_error_frame(
7274 &frame,
7275 "invalid_control_body",
7276 format!("expected module-control op '{expected}', got '{actual}'"),
7277 )?])
7278 }
7279 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
7280 guard.disarm();
7281 Ok(vec![control_error_frame(
7282 &frame,
7283 "module_timeout",
7284 format!(
7285 "module_id '{module_id}' answered health.check after {:?}",
7286 self.health_probe_timeout
7287 ),
7288 )?])
7289 }
7290 Ok(Err(_)) => Ok(vec![control_error_frame(
7291 &frame,
7292 "target_unavailable",
7293 "health.check waiter was canceled before the module responded",
7294 )?]),
7295 Err(_) => Ok(vec![control_error_frame(
7296 &frame,
7297 "module_timeout",
7298 format!(
7299 "module_id '{module_id}' did not answer health.check within {:?}",
7300 self.health_probe_timeout
7301 ),
7302 )?]),
7303 }
7304 },
7305 )
7306 .await
7307 }
7308
7309 fn supervisor_status(
7310 &self,
7311 module_id: &str,
7312 corr: u64,
7313 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
7314 self.supervisor
7315 .get(module_id)
7316 .map(|module| {
7317 let warming = module.is_warming_for_control("status").map_err(|err| {
7318 RouterError::backend(
7319 0,
7320 corr,
7321 format!(
7322 "failed to read supervisor warming state for module_id '{module_id}': {err}"
7323 ),
7324 )
7325 })?;
7326 module.status_for_control("status").map_err(|err| {
7327 RouterError::backend(
7328 0,
7329 corr,
7330 format!(
7331 "failed to read supervisor status for module_id '{module_id}': {err}"
7332 ),
7333 )
7334 }).map(|status| (status, warming))
7335 })
7336 .transpose()
7337 }
7338
7339 fn guard_module_control_op(
7340 &self,
7341 frame: &Frame,
7342 module_id: &str,
7343 op: &str,
7344 ) -> Result<Option<Frame>, RouterError> {
7345 if self.module_grants_op(module_id, op, frame.header.corr)? {
7346 return Ok(None);
7347 }
7348
7349 Ok(Some(control_error_frame(
7350 frame,
7351 "op_not_allowed",
7352 format!("module_id '{module_id}' did not grant control op '{op}'"),
7353 )?))
7354 }
7355
7356 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
7357 let Some(registration) = self
7358 .registry
7359 .get_module(module_id)
7360 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
7361 else {
7362 return Ok(false);
7363 };
7364 Ok(module_registration_grants_op(®istration.control_ops, op))
7365 }
7366
7367 fn handle_status_update(
7368 &self,
7369 endpoint: ModuleEndpointId,
7370 frame: Frame,
7371 ) -> Result<Vec<Frame>, RouterError> {
7372 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
7373 Ok(update) => update,
7374 Err(err) => {
7375 if is_known_module_push_op(&frame.body) {
7380 return Ok(vec![control_error_frame(
7381 &frame,
7382 "invalid_control_body",
7383 format!("malformed module control push body: {err}"),
7384 )?]);
7385 }
7386 return Ok(Vec::new());
7387 }
7388 };
7389
7390 match update {
7391 ModuleControlPush::RouteStatus {
7392 route_channel,
7393 route_epoch,
7394 status,
7395 } => {
7396 self.forwarding
7397 .cache_status(endpoint, route_channel, route_epoch, status)
7398 .map_err(RouterError::Forwarding)?;
7399 }
7400 }
7401 Ok(Vec::new())
7402 }
7403
7404 fn handle_route_poll(
7405 &self,
7406 ctx: &RouteCtx,
7407 frame: Frame,
7408 route_channel: u16,
7409 route_epoch: u32,
7410 kind: PollKind,
7411 ) -> Result<Vec<Frame>, RouterError> {
7412 let snapshot = self
7413 .forwarding
7414 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
7415 .map_err(RouterError::Forwarding)?;
7416 let response = match (kind, snapshot) {
7417 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
7418 ClientControlResponse::RoutePoll {
7419 route_channel,
7420 route_epoch,
7421 status,
7422 live: None,
7423 }
7424 }
7425 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7426 route_channel,
7427 route_epoch,
7428 status: None,
7429 live: None,
7430 },
7431 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
7432 let live = self
7458 .process_liveness
7459 .as_ref()
7460 .and_then(|source| source.process_live(&module_id))
7461 .unwrap_or(true);
7462 ClientControlResponse::RoutePoll {
7463 route_channel,
7464 route_epoch,
7465 status: None,
7466 live: Some(live),
7467 }
7468 }
7469 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7470 route_channel,
7471 route_epoch,
7472 status: None,
7473 live: Some(false),
7474 },
7475 };
7476
7477 Ok(vec![control_response_body_frame(
7478 &frame,
7479 &response,
7480 "ClientControlResponse::RoutePoll",
7481 )?])
7482 }
7483
7484 pub(crate) fn observe_module_control_completion(
7485 &self,
7486 completion: ModuleControlRpcCompletion,
7487 ) -> bool {
7488 match completion {
7489 ModuleControlRpcCompletion::Unknown => false,
7490 ModuleControlRpcCompletion::Settled => true,
7491 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
7492 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
7493 info!(
7494 module_id = %module_id,
7495 latency_ms,
7496 "late health.check answer proves the module is alive"
7497 );
7498 match self
7499 .supervisor
7500 .record_late_health_answer(&module_id, latency_ms)
7501 {
7502 Ok(true) => {}
7503 Ok(false) => debug!(
7504 module_id = %module_id,
7505 latency_ms,
7506 "late health.check answer has no active supervisor snapshot"
7507 ),
7508 Err(err) => warn!(
7509 module_id = %module_id,
7510 latency_ms,
7511 error = %err,
7512 "failed to record late health.check answer"
7513 ),
7514 }
7515 true
7516 }
7517 }
7518 }
7519
7520 fn refuse_to_end_module_connection_for_a_client(
7541 &self,
7542 module_connection_id: ConnectionId,
7543 corr: u64,
7544 err: ForwardingError,
7545 ) -> Result<(), RouterError> {
7546 if let ForwardingError::ConnectionClosing { connection_id } = err {
7547 if connection_id != module_connection_id {
7548 warn!(
7549 module_connection_id = module_connection_id.get(),
7550 client_connection_id = connection_id.get(),
7551 corr,
7552 "dropping a route.bind response for a closing client; the module connection keeps serving"
7553 );
7554 return Ok(());
7555 }
7556 }
7557 Err(RouterError::Forwarding(err))
7558 }
7559
7560 fn handle_module_relay_response(
7561 &self,
7562 connection_id: ConnectionId,
7563 frame: Frame,
7564 ) -> Result<Vec<Frame>, RouterError> {
7565 let mut secondary_error = None;
7566 let outcome = match frame.header.ty {
7567 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
7568 Ok(probe) if probe.op == "route.bind" => {
7569 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
7570 Ok(ModuleControlResponse::RouteBindAck {}) => {
7571 RouteBindRelayOutcome::Accepted
7572 }
7573 Ok(other) => {
7574 let message =
7575 format!("route.bind response carried unexpected body: {other:?}");
7576 secondary_error = Some(control_error_frame(
7577 &frame,
7578 "invalid_control_body",
7579 message.clone(),
7580 )?);
7581 RouteBindRelayOutcome::ModuleGone(message)
7582 }
7583 Err(err) => {
7584 let message = format!("malformed route.bind response body: {err}");
7585 secondary_error = Some(control_error_frame(
7586 &frame,
7587 "invalid_control_body",
7588 message.clone(),
7589 )?);
7590 RouteBindRelayOutcome::ModuleGone(message)
7591 }
7592 }
7593 }
7594 Ok(probe) => {
7595 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
7596 {
7597 Ok(response) => ModuleControlRpcOutcome::Response(response),
7598 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7599 "malformed {} response body: {err}",
7600 probe.op
7601 )),
7602 };
7603 let completion = self
7604 .forwarding
7605 .complete_module_control_rpc(
7606 connection_id,
7607 frame.header.corr,
7608 Some(&probe.op),
7609 outcome,
7610 )
7611 .map_err(RouterError::Forwarding)?;
7612 if !self.observe_module_control_completion(completion) {
7613 debug!(
7614 connection_id = connection_id.get(),
7615 corr = frame.header.corr,
7616 op = %probe.op,
7617 "dropping late or unknown module-control RPC response"
7618 );
7619 }
7620 return Ok(Vec::new());
7621 }
7622 Err(err) => {
7623 if let Some(expected_op) = self
7624 .forwarding
7625 .pending_module_control_op(connection_id, frame.header.corr)
7626 .map_err(RouterError::Forwarding)?
7627 {
7628 let completion = self
7629 .forwarding
7630 .complete_module_control_rpc(
7631 connection_id,
7632 frame.header.corr,
7633 None,
7634 ModuleControlRpcOutcome::MalformedResponse(format!(
7635 "malformed {expected_op} response body: {err}"
7636 )),
7637 )
7638 .map_err(RouterError::Forwarding)?;
7639 if !self.observe_module_control_completion(completion) {
7640 debug!(
7641 connection_id = connection_id.get(),
7642 corr = frame.header.corr,
7643 "dropping late malformed module-control RPC response"
7644 );
7645 }
7646 return Ok(Vec::new());
7647 }
7648 let message = format!("malformed route.bind response body: {err}");
7649 secondary_error = Some(control_error_frame(
7650 &frame,
7651 "invalid_control_body",
7652 message.clone(),
7653 )?);
7654 RouteBindRelayOutcome::ModuleGone(message)
7655 }
7656 },
7657 FrameType::Error => {
7658 if self
7659 .forwarding
7660 .pending_module_control_op(connection_id, frame.header.corr)
7661 .map_err(RouterError::Forwarding)?
7662 .is_some()
7663 {
7664 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7665 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7666 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7667 "malformed module-control ERROR body: {err}"
7668 )),
7669 };
7670 let completion = self
7671 .forwarding
7672 .complete_module_control_rpc(
7673 connection_id,
7674 frame.header.corr,
7675 None,
7676 outcome,
7677 )
7678 .map_err(RouterError::Forwarding)?;
7679 if !self.observe_module_control_completion(completion) {
7680 debug!(
7681 connection_id = connection_id.get(),
7682 corr = frame.header.corr,
7683 "dropping late or unknown module-control RPC error"
7684 );
7685 }
7686 return Ok(Vec::new());
7687 }
7688 match serde_json::from_slice::<ErrorBody>(&frame.body) {
7689 Ok(body) => RouteBindRelayOutcome::Rejected(body),
7690 Err(err) => {
7691 let message = format!("malformed route.bind ERROR body: {err}");
7692 secondary_error = Some(control_error_frame(
7693 &frame,
7694 "invalid_control_body",
7695 message.clone(),
7696 )?);
7697 RouteBindRelayOutcome::ModuleGone(message)
7698 }
7699 }
7700 }
7701 ty => {
7702 return Ok(vec![control_error_frame(
7703 &frame,
7704 "unsupported_control_frame",
7705 format!("unsupported module channel-0 frame {ty:?}"),
7706 )?])
7707 }
7708 };
7709
7710 let settled =
7711 self.forwarding
7712 .complete_pending_relay(connection_id, frame.header.corr, outcome);
7713 let completion = match settled {
7714 Ok(completion) => completion,
7715 Err(err) => {
7716 self.refuse_to_end_module_connection_for_a_client(
7717 connection_id,
7718 frame.header.corr,
7719 err,
7720 )?;
7721 return Ok(secondary_error.into_iter().collect());
7722 }
7723 };
7724 if let Some(target) = completion.abandoned.as_ref() {
7725 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7726 }
7727 if !completion.settled {
7728 debug!(
7729 connection_id = connection_id.get(),
7730 corr = frame.header.corr,
7731 frame_type = ?frame.header.ty,
7732 "dropping late or unknown route.bind relay response"
7733 );
7734 }
7735 Ok(secondary_error.into_iter().collect())
7736 }
7737
7738 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7739 debug!(connection_id = connection_id.get(), "handling GOODBYE");
7740 self.cleanup_connection_with_end_reason(
7744 connection_id,
7745 RegistrationEndReason::ExplicitGoodbye,
7746 )
7747 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7748 Ok(Vec::new())
7749 }
7750}
7751
7752impl Default for ControlHandler {
7753 fn default() -> Self {
7754 Self::new(Arc::new(Registry::default()))
7755 }
7756}
7757
7758impl crate::supervise::SwapPromotionObserver for ControlHandler {
7759 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7760 self.apply_registration_capabilities(registration);
7761 }
7762}
7763
7764fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7765 CapabilityRequirementStatus {
7766 consumer: status.consumer,
7767 capability: status.capability,
7768 need: match status.need {
7769 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7770 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7771 },
7772 verdict: status.verdict.as_str().to_string(),
7773 episode_seq: status.episode_seq,
7774 config_satisfiable: status.config_satisfiable,
7775 runtime_available: status.runtime_available,
7776 detail: status.detail,
7777 }
7778}
7779
7780fn append_capability_problem_detail(
7781 detail: Option<String>,
7782 capability_detail: Option<String>,
7783) -> Option<String> {
7784 match (detail, capability_detail) {
7785 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7786 (Some(detail), None) => Some(detail),
7787 (None, Some(capability_detail)) => Some(capability_detail),
7788 (None, None) => None,
7789 }
7790}
7791
7792fn subc_ops() -> Vec<String> {
7793 SUBC_CONTROL_OPS
7794 .iter()
7795 .map(|op| (*op).to_string())
7796 .collect()
7797}
7798
7799fn module_subc_ops() -> Vec<String> {
7800 SUBC_CONTROL_OPS
7801 .iter()
7802 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7803 .map(|op| (*op).to_string())
7804 .collect()
7805}
7806
7807#[cfg(test)]
7808fn module_baseline_control_ops() -> Vec<String> {
7809 MODULE_BASELINE_CONTROL_OPS
7810 .iter()
7811 .map(|op| (*op).to_string())
7812 .collect()
7813}
7814
7815fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7816 let mut seen = HashSet::new();
7817 let mut effective = Vec::new();
7818 for op in MODULE_BASELINE_CONTROL_OPS {
7819 if seen.insert((*op).to_string()) {
7820 effective.push((*op).to_string());
7821 }
7822 }
7823 for op in declared.unwrap_or_default() {
7824 if seen.insert(op.clone()) {
7825 effective.push(op);
7826 }
7827 }
7828 effective
7829}
7830
7831fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7832 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7833}
7834
7835fn target_module_id(target: &RouteTarget) -> &str {
7836 match target {
7837 RouteTarget::ToolProvider { module_id }
7838 | RouteTarget::ManagementSurface { module_id }
7839 | RouteTarget::InternalService { module_id, .. } => module_id,
7840 }
7841}
7842
7843fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7844 roles.iter().any(|role| match (target, role) {
7845 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7846 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7847 (
7848 RouteTarget::InternalService { service_id, .. },
7849 ProviderRole::InternalService {
7850 service_id: provided,
7851 ..
7852 },
7853 ) => service_id == provided,
7854 _ => false,
7855 })
7856}
7857
7858fn is_routable_role(role: &ProviderRole) -> bool {
7859 matches!(
7860 role,
7861 ProviderRole::ToolProvider { .. }
7862 | ProviderRole::ManagementSurface { .. }
7863 | ProviderRole::InternalService { .. }
7864 )
7865}
7866
7867#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7868enum ControlRequestBodyError {
7869 UnknownOp,
7870 InvalidBody,
7871}
7872
7873#[derive(Debug, Deserialize)]
7874struct ControlOpProbe {
7875 op: String,
7876}
7877
7878const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7881
7882fn is_known_module_push_op(body: &[u8]) -> bool {
7883 serde_json::from_slice::<ControlOpProbe>(body)
7884 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7885 .unwrap_or(false)
7886}
7887
7888fn is_known_module_request_op(body: &[u8]) -> bool {
7889 serde_json::from_slice::<ControlOpProbe>(body)
7890 .map(|probe| is_module_to_subc_op(&probe.op))
7891 .unwrap_or(false)
7892}
7893
7894fn is_module_to_subc_op(op: &str) -> bool {
7895 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7896}
7897
7898fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7899 debug!(
7900 op = %op,
7901 connection_id = connection_id.get(),
7902 corr,
7903 "control dispatch"
7904 );
7905}
7906
7907fn log_slow_control_dispatch(
7908 dispatch_started_at: Option<StdInstant>,
7909 op: &'static str,
7910 connection_id: ConnectionId,
7911 corr: u64,
7912 route_open_timings: Option<&RouteOpenTimings>,
7913) {
7914 let Some(dispatch_started_at) = dispatch_started_at else {
7915 return;
7916 };
7917 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7919 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7920 if let Some(timings) = route_open_timings {
7921 warn!(
7922 op = %op,
7923 connection_id = connection_id.get(),
7924 corr,
7925 elapsed_ms = elapsed.as_millis() as u64,
7926 module_id = ?timings.module_id,
7927 principal = %timings.principal,
7928 admission_ms = timings.admission.as_millis() as u64,
7929 project_root_ms = timings.project_root.as_millis() as u64,
7930 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7931 bind_send_ms = timings.bind_send.as_millis() as u64,
7932 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7933 "slow control dispatch"
7934 );
7935 return;
7936 }
7937 warn!(
7938 op = %op,
7939 connection_id = connection_id.get(),
7940 corr,
7941 elapsed_ms = elapsed.as_millis() as u64,
7942 "slow control dispatch"
7943 );
7944 }
7945}
7946
7947#[cfg(test)]
7949type TestCallerLookup =
7950 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7951
7952fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7954 match request {
7955 ClientControlRequest::SupervisorRestart { module_id, .. } => {
7956 Some(PrivilegedOperation::Restart { module_id })
7957 }
7958 ClientControlRequest::SupervisorSwap { module_id, .. } => {
7959 Some(PrivilegedOperation::Swap { module_id })
7960 }
7961 ClientControlRequest::SupervisorReload { module_id } => {
7962 Some(PrivilegedOperation::Reload { module_id })
7963 }
7964 ClientControlRequest::SupervisorRescan { preview } => {
7965 if *preview {
7966 None
7967 } else {
7968 Some(PrivilegedOperation::Rescan)
7969 }
7970 }
7971 ClientControlRequest::SupervisorRescanChecked {
7976 preview,
7977 candidate,
7978 expect,
7979 } => (!*preview && candidate.is_none() && expect.is_some())
7980 .then_some(PrivilegedOperation::Rescan),
7981 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7982 Some(PrivilegedOperation::SetEnabled {
7983 module_id,
7984 enabled: *enabled,
7985 })
7986 }
7987 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7988 Some(PrivilegedOperation::ReleaseReserved { module_id })
7989 }
7990 ClientControlRequest::ServerDescribe {}
7991 | ClientControlRequest::DaemonShutdownIntent { .. }
7994 | ClientControlRequest::CatalogList { .. }
7995 | ClientControlRequest::RouteOpen { .. }
7996 | ClientControlRequest::RoutePoll { .. }
7997 | ClientControlRequest::SupervisorList {}
7998 | ClientControlRequest::SupervisorHealth {}
7999 | ClientControlRequest::SupervisorHealthProbe { .. }
8000 | ClientControlRequest::SupervisorRoutes { .. }
8001 | ClientControlRequest::SupervisorProvenance { .. }
8002 | ClientControlRequest::SupervisorStderrTail { .. }
8003 | ClientControlRequest::SupervisorTerminals { .. }
8004 | ClientControlRequest::SupervisorSpawnSnapshot {}
8005 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
8006 }
8007}
8008
8009fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
8010 match request {
8011 ClientControlRequest::DaemonShutdownIntent { .. } => ops::DAEMON_SHUTDOWN_INTENT,
8012 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
8013 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
8014 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
8015 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
8016 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
8017 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
8018 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
8019 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
8020 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
8021 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
8022 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
8023 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
8024 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
8025 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
8026 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
8027 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
8028 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
8029 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
8030 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
8031 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
8032 }
8033}
8034
8035struct RouteOpenCaller {
8043 principal: String,
8047 scope: String,
8050 harness: String,
8053}
8054
8055impl RouteOpenCaller {
8056 fn new(
8057 principal: Option<&Principal>,
8058 scope: Option<&ScopeSelector>,
8059 identity: &BindIdentity,
8060 ) -> Self {
8061 Self {
8062 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
8063 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
8064 harness: bounded_log_text(&identity.harness),
8065 }
8066 }
8067
8068 fn unknown() -> Self {
8071 Self {
8072 principal: "unknown".to_string(),
8073 scope: "unknown".to_string(),
8074 harness: "unknown".to_string(),
8075 }
8076 }
8077}
8078
8079const MAX_LOGGED_TEXT_CHARS: usize = 64;
8082
8083fn bounded_log_text(value: &str) -> String {
8087 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
8088 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
8089 bounded.push('…');
8090 }
8091 format!("{bounded:?}")
8092}
8093
8094fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
8100 use subc_protocol::scope::ScopeKind;
8101 let kind = match stamp.kind {
8102 ScopeKind::Head => "head",
8103 ScopeKind::Worker => "worker",
8104 ScopeKind::Ephemeral => "ephemeral",
8105 };
8106 format!(
8107 "{kind}/{}/{}@{}",
8108 principal_label(&stamp.owner),
8109 bounded_log_text(&stamp.scope_ref),
8110 stamp.scope_epoch
8111 )
8112}
8113
8114fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
8121 let epoch = selector
8122 .scope_epoch
8123 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
8124 format!(
8125 "requested/{}/{}@{epoch}",
8126 bounded_log_text(&principal_label(&selector.owner)),
8127 bounded_log_text(&selector.scope_ref),
8128 )
8129}
8130
8131fn principal_label(principal: &Principal) -> String {
8132 match principal {
8133 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
8134 Principal::Direct => "direct".to_string(),
8135 other => format!("{other:?}"),
8136 }
8137}
8138
8139fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
8140 match request {
8141 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
8142 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
8143 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
8144 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
8145 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
8146 }
8147}
8148
8149fn parse_client_control_request(
8150 body: &[u8],
8151) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
8152 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
8153 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8154 Ok(probe)
8155 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
8156 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
8157 {
8158 ControlRequestBodyError::InvalidBody
8159 }
8160 Ok(_) => ControlRequestBodyError::UnknownOp,
8161 Err(_) => ControlRequestBodyError::InvalidBody,
8162 };
8163 (err, classification)
8164 })
8165}
8166
8167fn parse_module_control_request_from_module(
8168 body: &[u8],
8169) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
8170 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
8171 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8172 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
8173 Ok(_) => ControlRequestBodyError::UnknownOp,
8174 Err(_) => ControlRequestBodyError::InvalidBody,
8175 };
8176 (err, classification)
8177 })
8178}
8179
8180#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
8181enum ProviderRoleKind {
8182 ToolProvider,
8183 PipelineStage,
8184 ManagementSurface,
8185 InternalService,
8186}
8187
8188fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
8189 match role {
8190 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
8191 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
8192 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
8193 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
8194 }
8195}
8196
8197fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
8198 roles.iter().map(provider_role_kind).collect()
8199}
8200
8201const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
8204
8205#[derive(Debug, Default, PartialEq, Eq)]
8207struct ScopeOutcomeCounts {
8208 created: usize,
8209 replaced: usize,
8210 updated: usize,
8211 unchanged: usize,
8212 refused: usize,
8213}
8214
8215impl ScopeOutcomeCounts {
8216 fn of(results: &[ScopeRecordResult]) -> Self {
8217 let mut counts = Self::default();
8218 for result in results {
8219 let slot = match result.outcome {
8220 ScopeRecordOutcome::Created => &mut counts.created,
8221 ScopeRecordOutcome::Replaced => &mut counts.replaced,
8222 ScopeRecordOutcome::Updated => &mut counts.updated,
8223 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
8224 ScopeRecordOutcome::Refused => &mut counts.refused,
8225 };
8226 *slot += 1;
8227 }
8228 counts
8229 }
8230}
8231
8232#[cfg(test)]
8233mod scope_outcome_count_tests {
8234 use super::*;
8235
8236 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
8237 ScopeRecordResult {
8238 scope_ref: "r".to_string(),
8239 scope_epoch: 1,
8240 outcome,
8241 code: None,
8242 message: None,
8243 version: None,
8244 parent_state: None,
8245 }
8246 }
8247
8248 #[test]
8251 fn every_outcome_is_counted_in_its_own_field() {
8252 let results = [
8253 result(ScopeRecordOutcome::Created),
8254 result(ScopeRecordOutcome::Created),
8255 result(ScopeRecordOutcome::Replaced),
8256 result(ScopeRecordOutcome::Updated),
8257 result(ScopeRecordOutcome::Unchanged),
8258 result(ScopeRecordOutcome::Refused),
8259 result(ScopeRecordOutcome::Refused),
8260 result(ScopeRecordOutcome::Refused),
8261 ];
8262 assert_eq!(
8263 ScopeOutcomeCounts::of(&results),
8264 ScopeOutcomeCounts {
8265 created: 2,
8266 replaced: 1,
8267 updated: 1,
8268 unchanged: 1,
8269 refused: 3,
8270 }
8271 );
8272 }
8273}
8274
8275fn capability_census_trigger(
8279 old: Option<&CapabilityDeclarations>,
8280 new: Option<&CapabilityDeclarations>,
8281) -> bool {
8282 let old_provides = old
8283 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
8284 .unwrap_or_default();
8285 let old_denies = old
8286 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
8287 .unwrap_or_default();
8288 let new = new.cloned().unwrap_or(CapabilityDeclarations {
8289 provides: Vec::new(),
8290 requires: Vec::new(),
8291 must_never_reach: Vec::new(),
8292 });
8293
8294 new.provides
8295 .iter()
8296 .any(|capability| !old_provides.contains(capability))
8297 || new
8298 .must_never_reach
8299 .iter()
8300 .any(|capability| !old_denies.contains(capability))
8301}
8302
8303fn denied_capability<'a>(
8306 opening_manifest: &'a ModuleManifest,
8307 target_manifest: &ModuleManifest,
8308) -> Option<&'a str> {
8309 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
8310 let target_capabilities = target_manifest.capabilities.as_ref()?;
8311 opening_capabilities
8312 .must_never_reach
8313 .iter()
8314 .find(|denied| {
8315 target_capabilities
8316 .provides
8317 .iter()
8318 .any(|provided| provided == *denied)
8319 })
8320 .map(String::as_str)
8321}
8322
8323fn catalog_update_frozen_field_message(
8324 registered: &ModuleManifest,
8325 provides: &[ProviderRole],
8326) -> Option<String> {
8327 let old_has_provides = !registered.provides.is_empty();
8328 let new_has_provides = !provides.is_empty();
8329 if old_has_provides != new_has_provides {
8330 return Some(format!(
8331 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
8332 registered.module_id
8333 ));
8334 }
8335
8336 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
8337 return Some(format!(
8338 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
8339 registered.module_id
8340 ));
8341 }
8342
8343 let registered_concurrency = manifest_concurrency(registered);
8344 let mut candidate = registered.clone();
8345 candidate.provides = provides.to_vec();
8346 let candidate_concurrency = manifest_concurrency(&candidate);
8347 if candidate_concurrency != registered_concurrency {
8348 return Some(format!(
8349 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
8350 registered.module_id, registered_concurrency, candidate_concurrency
8351 ));
8352 }
8353
8354 None
8357}
8358
8359fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
8360 manifest.provides.iter().any(is_routable_role)
8361}
8362
8363fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
8369 manifest
8370 .provides
8371 .iter()
8372 .find_map(|provider| match provider {
8373 ProviderRole::ToolProvider { concurrency, .. }
8374 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
8375 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
8376 })
8377 .unwrap_or(Concurrency::ModuleManaged)
8378}
8379
8380fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
8387 let has_management_surface = manifest
8388 .provides
8389 .iter()
8390 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
8391 if !has_management_surface {
8392 return false;
8393 }
8394 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
8395 return false;
8396 };
8397 let Some(provides) = raw
8398 .get("manifest")
8399 .and_then(|manifest| manifest.get("provides"))
8400 .and_then(serde_json::Value::as_array)
8401 else {
8402 return false;
8403 };
8404 provides.iter().any(|role| {
8409 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
8410 && role.get("concurrency").is_none()
8411 })
8412}
8413
8414fn negotiate_version(peer_version: u8) -> Result<u8, String> {
8415 if peer_version != PROTOCOL_VERSION {
8416 return Err(format!(
8417 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
8418 ));
8419 }
8420 Ok(PROTOCOL_VERSION)
8421}
8422
8423fn pong(frame: &Frame) -> Result<Frame, RouterError> {
8424 Frame::build_with_version(
8425 response_version(frame),
8426 FrameType::Pong,
8427 frame.header.flags,
8428 0,
8429 0,
8430 frame.header.corr,
8431 Vec::new(),
8432 )
8433 .map_err(RouterError::FrameBuild)
8434}
8435
8436pub(crate) fn control_error_frame(
8437 frame: &Frame,
8438 code: &'static str,
8439 message: impl Into<String>,
8440) -> Result<Frame, RouterError> {
8441 control_error_body_frame(
8442 frame,
8443 ErrorBody {
8444 code: code.to_string(),
8445 message: message.into(),
8446 detail: None,
8447 },
8448 )
8449}
8450
8451fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8459 control_error_frame_with_flags(frame, error, daemon_error_flags())
8460}
8461
8462fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8468 control_error_frame_with_flags(frame, error, control_flags())
8469}
8470
8471fn control_error_frame_with_flags(
8472 frame: &Frame,
8473 error: ErrorBody,
8474 flags: Flags,
8475) -> Result<Frame, RouterError> {
8476 let body = serde_json::to_vec(&error).map_err(|err| {
8477 RouterError::backend(
8478 0,
8479 frame.header.corr,
8480 format!("failed to encode control ERROR: {err}"),
8481 )
8482 })?;
8483
8484 Frame::build_with_version(
8485 response_version(frame),
8486 FrameType::Error,
8487 flags,
8488 0,
8489 0,
8490 frame.header.corr,
8491 body,
8492 )
8493 .map_err(RouterError::FrameBuild)
8494}
8495
8496pub(crate) fn control_response_body_frame<T: Serialize>(
8497 frame: &Frame,
8498 reply: &T,
8499 label: &'static str,
8500) -> Result<Frame, RouterError> {
8501 let body = serde_json::to_vec(reply).map_err(|err| {
8502 RouterError::backend(
8503 0,
8504 frame.header.corr,
8505 format!("failed to encode {label}: {err}"),
8506 )
8507 })?;
8508
8509 Frame::build_with_version(
8510 response_version(frame),
8511 FrameType::Response,
8512 control_flags(),
8513 0,
8514 0,
8515 frame.header.corr,
8516 body,
8517 )
8518 .map_err(RouterError::FrameBuild)
8519}
8520
8521fn forwarding_error_code(err: &ForwardingError) -> &'static str {
8551 match err {
8552 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
8553 ForwardingError::NoModuleConnection => "target_unavailable",
8554 ForwardingError::ModuleReloading { .. } => "module_reloading",
8555 ForwardingError::ClientRouteChannelExhausted { .. }
8556 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
8557 ForwardingError::StaleModuleEndpoint
8558 | ForwardingError::UnknownReservation { .. }
8559 | ForwardingError::ConnectionClosing { .. }
8560 | ForwardingError::ClientEgressClosed { .. }
8561 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
8562 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
8565 ForwardingError::RelayCorrelationExhausted
8566 | ForwardingError::RouteOpenBuild(_)
8567 | ForwardingError::Poisoned => "forwarding_error",
8568 }
8569}
8570
8571fn response_version(frame: &Frame) -> u8 {
8572 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
8573 frame.header.ver
8574 } else {
8575 PROTOCOL_VERSION
8576 }
8577}
8578
8579fn control_flags() -> Flags {
8580 Flags::new(false, Priority::Passive, false)
8581}
8582
8583fn send_goodbye_target_best_effort(
8588 counters: &DaemonCounters,
8589 target: &GoodbyeTarget,
8590 context: &'static str,
8591) {
8592 let Ok(frame) = Frame::build_with_version(
8593 target.negotiated_ver,
8594 FrameType::Goodbye,
8595 control_flags(),
8596 target.channel,
8597 target.epoch,
8598 0,
8599 Vec::new(),
8600 ) else {
8601 return;
8602 };
8603 crate::forwarding::send_module_route_goodbye(
8604 counters,
8605 &target.sink,
8606 frame,
8607 target.module_id.as_deref(),
8608 context,
8609 );
8610}
8611
8612pub(crate) fn send_route_control_pushes(
8613 forwarding: &ForwardingTable,
8614 routes: Vec<EndpointRoute>,
8615 push: ClientControlPush,
8616) {
8617 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
8618 for route in routes {
8619 let target = route.goodbye_target;
8620 if let Some((existing, channels)) = targets
8621 .iter_mut()
8622 .find(|(existing, _)| existing.connection_id == target.connection_id)
8623 {
8624 debug_assert_eq!(
8625 existing.negotiated_ver, target.negotiated_ver,
8626 "one connection cannot negotiate multiple frame versions"
8627 );
8628 if !channels.contains(&target.channel) {
8629 channels.push(target.channel);
8630 }
8631 continue;
8632 }
8633 let channel = target.channel;
8634 targets.push((target, vec![channel]));
8635 }
8636 for (target, mut channels) in targets {
8637 channels.sort_unstable();
8638 let mut push = push.clone();
8639 match &mut push {
8640 ClientControlPush::RouteClosing {
8641 channels: covered, ..
8642 }
8643 | ClientControlPush::RouteClosed {
8644 channels: covered, ..
8645 } => *covered = channels,
8646 }
8647 let body = match serde_json::to_vec(&push) {
8648 Ok(body) => body,
8649 Err(err) => {
8650 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8651 continue;
8652 }
8653 };
8654 let frame = match Frame::build_with_version(
8655 target.negotiated_ver,
8656 FrameType::Push,
8657 control_flags(),
8658 0,
8659 0,
8660 0,
8661 body.clone(),
8662 ) {
8663 Ok(frame) => frame,
8664 Err(err) => {
8665 warn!(
8666 route_channel = target.channel,
8667 error = %err,
8668 "failed to build route lifecycle control PUSH frame"
8669 );
8670 continue;
8671 }
8672 };
8673 if let Err(err) = target.sink.try_send(frame) {
8674 if target.close_on_delivery_failure() {
8675 warn!(
8676 target_connection_id = target.connection_id.get(),
8677 route_channel = target.channel,
8678 error = %err,
8679 "route lifecycle control PUSH was not delivered to client; closing target connection"
8680 );
8681 let _ = forwarding.escalate_client_delivery_failure(
8682 target.connection_id,
8683 target.channel,
8684 target.epoch,
8685 CloseReason::new(
8686 "route_lifecycle_push_delivery_failed",
8687 format!(
8688 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8689 target.channel
8690 ),
8691 ),
8692 crate::forwarding::UndeliveredFrame {
8693 module_id: target.module_id.as_deref(),
8694 sink: &target.sink,
8695 },
8696 );
8697 }
8698 }
8699 }
8700}
8701
8702#[cfg(test)]
8703mod tests {
8704 #[tokio::test]
8705 async fn route_busy_wait_attribution_resolves_targets_instead_of_trusting_selectors() {
8706 let registry = Arc::new(Registry::default());
8707 let handler = ControlHandler::new(registry);
8708 assert_eq!(
8709 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
8710 None
8711 );
8712 let (ctx, mut rx) = route_ctx(ConnectionId::new(901));
8713 hello_via_sink(
8714 &handler,
8715 &ctx,
8716 &mut rx,
8717 hello_frame("daemon", PROTOCOL_VERSION, 1),
8718 )
8719 .await;
8720 assert_eq!(
8721 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
8722 Some("daemon".into())
8723 );
8724 assert_eq!(
8725 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("missing")),
8726 None
8727 );
8728 assert_eq!(
8729 handler.resolve_reader_wait_target(ops::SUPERVISOR_TERMINALS, Some("daemon")),
8730 None,
8731 "a registered module is not necessarily supervised"
8732 );
8733 assert_eq!(
8734 handler.resolve_reader_wait_target(ops::SUPERVISOR_PROVENANCE, None),
8735 None,
8736 "aggregate provenance has no single target"
8737 );
8738 assert_eq!(
8739 handler.resolve_reader_wait_target(ops::APP_OPEN, Some("daemon")),
8740 None
8741 );
8742 }
8743
8744 mod peer_manifest {
8745 include!("peer_manifest_tests.rs");
8746 }
8747 mod activity_read {
8748 include!("activity_read_tests.rs");
8749 }
8750
8751 mod lifecycle_activity {
8752 include!("control_lifecycle_tests.rs");
8753 }
8754 #[cfg(unix)]
8755 #[tokio::test]
8756 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8757 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8758 let path = dir.join("subc.jsonc");
8759 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8760 "program":"/bin/sleep","args":["60"],"protocol":"none",
8761 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8762 }}}).to_string()).unwrap();
8763 let mut configured = crate::daemon_config::load(&path)
8764 .unwrap()
8765 .unwrap()
8766 .modules
8767 .pop()
8768 .unwrap();
8769 let registry = std::sync::Arc::new(crate::Registry::default());
8770 let handle = crate::SupervisorHandle::new();
8771 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8772 .with_handle(handle.clone());
8773 let module = supervisor
8774 .supervise_configured_with_health(
8775 configured.module_spec(),
8776 true,
8777 configured.health.clone(),
8778 None,
8779 configured.restart,
8780 )
8781 .unwrap();
8782 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8783 let before = module.status().unwrap().pid;
8784 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8785 configured.health.cadence = std::time::Duration::from_secs(3600);
8786 let health_only = handler
8787 .reconcile_supervised_modules(
8788 &supervisor,
8789 vec![configured.clone()],
8790 false,
8791 Principal::Direct,
8792 )
8793 .await
8794 .unwrap();
8795 assert!(
8796 health_only.0.changed_pending_reload.is_empty(),
8797 "health policy is already applied live"
8798 );
8799 assert_eq!(module.status().unwrap().pid, before);
8800 assert_eq!(
8801 module.configuration().unwrap().1.http,
8802 configured.health.http
8803 );
8804 configured.args = vec!["61".into()];
8805 let launch = handler
8806 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8807 .await
8808 .unwrap();
8809 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8810 assert_eq!(
8811 module.status().unwrap().pid,
8812 before,
8813 "a launch edit is stored until reload"
8814 );
8815 module.drain().await.unwrap();
8816 }
8817 use cortexkit_test_support::ScratchDir;
8818 use std::{
8819 collections::BTreeMap,
8820 fmt,
8821 path::PathBuf,
8822 sync::{Arc, Mutex},
8823 time::Duration,
8824 };
8825
8826 use serde_json::{json, Value};
8827 use subc_protocol::{
8828 manifest::{
8829 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8830 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8831 },
8832 session::HealthStatus,
8833 FrameType,
8834 };
8835
8836 use super::*;
8837 use crate::{
8838 forwarding::{DataRoute, DataRouteState},
8839 registry::ChannelState,
8840 router::FrameSink,
8841 stderr_tail::DEFAULT_MAX_LINE_BYTES,
8842 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8843 RouteCtx, Router,
8844 };
8845 use tokio::{
8846 sync::mpsc,
8847 time::{sleep, Instant},
8848 };
8849 use tracing::{
8850 field::{Field, Visit},
8851 Event, Subscriber,
8852 };
8853 use tracing_subscriber::{layer::Context, prelude::*, Layer};
8854
8855 fn fake_aft_stub_path() -> PathBuf {
8874 let mut path = std::env::current_exe().expect("current_exe available in tests");
8875 path.pop(); path.pop(); path.push(if cfg!(windows) {
8878 "fake-aft-stub.exe"
8879 } else {
8880 "fake-aft-stub"
8881 });
8882 assert!(
8883 path.exists(),
8884 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8885 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8886 path.display()
8887 );
8888 path
8889 }
8890
8891 fn client_retries(code: &str) -> bool {
8896 subc_protocol::error_codes::is_retryable_route_open(code)
8897 }
8898
8899 #[test]
8910 fn retryability_of_forwarding_codes_matches_the_failure() {
8911 let transient = [
8914 ForwardingError::NoModuleConnection,
8915 ForwardingError::ModuleReloading {
8916 module_id: "m".into(),
8917 },
8918 ForwardingError::StaleModuleEndpoint,
8919 ForwardingError::UnknownReservation {
8920 client_channel: 1,
8921 module_channel: 1,
8922 },
8923 ForwardingError::ConnectionClosing {
8924 connection_id: ConnectionId::new(1),
8925 },
8926 ForwardingError::ClientEgressClosed {
8927 connection_id: ConnectionId::new(1),
8928 },
8929 ForwardingError::ModuleEgressUnavailable {
8930 connection_id: ConnectionId::new(1),
8931 },
8932 ];
8933 for err in transient {
8934 let code = forwarding_error_code(&err);
8935 assert!(
8936 client_retries(code),
8937 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8938 );
8939 }
8940
8941 let permanent = [
8946 ForwardingError::ConnectionRoleConflict {
8947 connection_id: ConnectionId::new(1),
8948 },
8949 ForwardingError::ClientRouteChannelExhausted {
8950 connection_id: ConnectionId::new(1),
8951 },
8952 ForwardingError::ModuleRouteChannelExhausted {
8953 endpoint: ModuleEndpointId {
8954 connection_id: ConnectionId::new(1),
8955 generation: 1,
8956 },
8957 },
8958 ForwardingError::RelayCorrelationExhausted,
8959 ForwardingError::RouteOpenBuild("x".into()),
8960 ForwardingError::Poisoned,
8961 ];
8962 for err in permanent {
8963 let code = forwarding_error_code(&err);
8964 assert!(
8965 !client_retries(code),
8966 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8967 );
8968 }
8969 }
8970
8971 #[tokio::test]
8999 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
9000 let handler = ControlHandler::default();
9001 let frame =
9002 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
9003
9004 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
9008 assert_eq!(
9009 stamped,
9010 Principal::Direct,
9011 "a caller that proved nothing must not be stamped as a supervised module"
9012 );
9013
9014 let forged = handler
9019 .route_open_principal(
9020 &frame,
9021 Some(ConsumerIdentity {
9022 module_id: "aft".to_string(),
9023 launch_nonce: "not-a-real-nonce".to_string(),
9024 }),
9025 )
9026 .unwrap();
9027 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
9028 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
9029 }
9030
9031 #[test]
9048 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
9049 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
9050 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
9051 let ClientControlRequest::RouteOpen {
9052 consumer_identity, ..
9053 } = parsed
9054 else {
9055 panic!("route.open body must parse as RouteOpen");
9056 };
9057 assert_eq!(
9058 consumer_identity,
9059 Some(ConsumerIdentity {
9060 module_id: "aft".to_string(),
9061 launch_nonce: "n".to_string(),
9062 }),
9063 "the wire field name must reach the value route_open_principal reads"
9064 );
9065 }
9066
9067 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
9068 ModuleManifest::builder(module_id, "0.1.0")
9069 .protocol_ver(protocol_ver)
9070 .provides(vec![ProviderRole::ToolProvider {
9071 tools: vec![Tool {
9072 name: "read".to_string(),
9073 description: None,
9074 execution_mode: ExecutionMode::Pure,
9075 schema: json!({"type": "object"}),
9076 }],
9077 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
9078 concurrency: Concurrency::ModuleManaged,
9079 emits_push: true,
9080 sub_supervises: true,
9081 }])
9082 .build()
9083 }
9084
9085 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
9086 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
9087 }
9088
9089 fn hello_frame_with_control_ops(
9090 module_id: &str,
9091 protocol_ver: u8,
9092 corr: u64,
9093 control_ops: Option<Vec<String>>,
9094 ) -> Frame {
9095 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
9096 }
9097
9098 fn hello_frame_with_nonce(
9099 module_id: &str,
9100 protocol_ver: u8,
9101 corr: u64,
9102 launch_nonce: Option<&str>,
9103 ) -> Frame {
9104 hello_frame_full(
9105 module_id,
9106 protocol_ver,
9107 corr,
9108 None,
9109 launch_nonce.map(ToOwned::to_owned),
9110 )
9111 }
9112
9113 fn hello_frame_full(
9114 module_id: &str,
9115 protocol_ver: u8,
9116 corr: u64,
9117 control_ops: Option<Vec<String>>,
9118 launch_nonce: Option<String>,
9119 ) -> Frame {
9120 let body = serde_json::to_vec(&ModuleHelloBody {
9121 manifest: manifest(module_id, protocol_ver),
9122 protocol_ver,
9123 control_ops,
9124 launch_nonce,
9125 })
9126 .unwrap();
9127 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9128 }
9129
9130 fn hello_frame_with_rich_ui(
9131 module_id: &str,
9132 protocol_ver: u8,
9133 corr: u64,
9134 launch_nonce: Option<&str>,
9135 rich_ui: Value,
9136 ) -> Frame {
9137 let mut body = serde_json::to_value(ModuleHelloBody {
9138 manifest: manifest(module_id, protocol_ver),
9139 protocol_ver,
9140 control_ops: None,
9141 launch_nonce: launch_nonce.map(str::to_string),
9142 })
9143 .unwrap();
9144 body["manifest"]["rich_ui"] = rich_ui;
9145 Frame::build(
9146 FrameType::Hello,
9147 control_flags(),
9148 0,
9149 0,
9150 corr,
9151 serde_json::to_vec(&body).unwrap(),
9152 )
9153 .unwrap()
9154 }
9155
9156 fn non_routable_hello_frame_with_control_ops(
9157 module_id: &str,
9158 corr: u64,
9159 control_ops: Option<Vec<String>>,
9160 ) -> Frame {
9161 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9162 manifest.provides.clear();
9163 let body = serde_json::to_vec(&ModuleHelloBody {
9164 manifest,
9165 protocol_ver: PROTOCOL_VERSION,
9166 control_ops,
9167 launch_nonce: None,
9168 })
9169 .unwrap();
9170 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9171 }
9172
9173 fn capability_grammar_hello_frame(
9174 capabilities: Value,
9175 runtime_computed: Option<Value>,
9176 corr: u64,
9177 ) -> Frame {
9178 let mut body = serde_json::to_value(ModuleHelloBody {
9179 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
9180 protocol_ver: PROTOCOL_VERSION,
9181 control_ops: None,
9182 launch_nonce: None,
9183 })
9184 .expect("HELLO body serializes");
9185 body["manifest"]["capabilities"] = capabilities;
9186 if let Some(runtime_computed) = runtime_computed {
9187 body["runtime_computed"] = runtime_computed;
9188 }
9189 Frame::build(
9190 FrameType::Hello,
9191 control_flags(),
9192 0,
9193 0,
9194 corr,
9195 serde_json::to_vec(&body).expect("HELLO body reserializes"),
9196 )
9197 .expect("HELLO frame builds")
9198 }
9199
9200 fn channel_request(channel: u16, corr: u64) -> Frame {
9201 Frame::build(
9202 FrameType::Request,
9203 Flags::new(true, Priority::Interactive, false),
9204 channel,
9205 0,
9206 corr,
9207 b"opaque".to_vec(),
9208 )
9209 .unwrap()
9210 }
9211
9212 fn route_ctx(
9213 connection_id: ConnectionId,
9214 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
9215 let (tx, rx) = mpsc::channel(8);
9216 (
9217 RouteCtx {
9218 connection_id,
9219 egress: FrameSink::new(tx),
9220 },
9221 rx,
9222 )
9223 }
9224
9225 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
9226 serde_json::from_slice(&frame.body).unwrap()
9227 }
9228
9229 async fn hello_via_sink(
9234 handler: &ControlHandler,
9235 ctx: &RouteCtx,
9236 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9237 hello: Frame,
9238 ) -> Frame {
9239 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
9240 assert!(
9241 replies.is_empty(),
9242 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
9243 );
9244 let ack = rx
9245 .try_recv()
9246 .expect("HELLO_ACK is queued on the module sink")
9247 .frame;
9248 assert_eq!(ack.header.ty, FrameType::HelloAck);
9249 ack
9250 }
9251
9252 fn parse_error(frame: &Frame) -> Value {
9253 serde_json::from_slice(&frame.body).unwrap()
9254 }
9255
9256 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
9257 serde_json::from_slice(&frame.body).unwrap()
9258 }
9259
9260 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
9261 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
9262 route_channel,
9263 route_epoch: 0,
9264 kind,
9265 })
9266 .unwrap();
9267 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9268 }
9269
9270 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
9271 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
9272 module_id: module_id.to_string(),
9273 })
9274 .unwrap();
9275 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9276 }
9277
9278 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
9279 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
9280 }
9281
9282 fn legacy_route_open_frame(corr: u64, module_id: &str, root: ScratchDir) -> Frame {
9284 let path = ProjectRootId::from_path_allowing_missing(root.path()).unwrap();
9285 let mut frame = route_open_frame(corr, module_id, root);
9286 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
9287 body["identity"]["project_root"] = serde_json::json!(path.as_path());
9288 frame.body = serde_json::to_vec(&body).unwrap();
9289 frame
9290 }
9291
9292 fn route_open_frame_with_consumer_capabilities(
9293 corr: u64,
9294 module_id: &str,
9295 _project_root: ScratchDir,
9296 consumer_capabilities: Option<Vec<String>>,
9297 ) -> Frame {
9298 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9299 busy_refusals: None,
9300 target: RouteTarget::ToolProvider {
9301 module_id: module_id.to_string(),
9302 },
9303 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
9304 consumer_identity: None,
9305 consumer_capabilities,
9306 role_versions: None,
9307 admission_facts: None,
9308 scope: None,
9309 })
9310 .unwrap();
9311 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9312 }
9313
9314 fn route_open_frame_with_role_versions(
9315 corr: u64,
9316 module_id: &str,
9317 _project_root: ScratchDir,
9318 role_versions: Option<BTreeMap<String, String>>,
9319 ) -> Frame {
9320 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9321 busy_refusals: None,
9322 target: RouteTarget::ToolProvider {
9323 module_id: module_id.to_string(),
9324 },
9325 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9326 consumer_identity: None,
9327 consumer_capabilities: None,
9328 role_versions,
9329 admission_facts: None,
9330 scope: None,
9331 })
9332 .unwrap();
9333 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9334 }
9335
9336 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
9337 entries
9338 .iter()
9339 .map(|(role, version)| (role.to_string(), version.to_string()))
9340 .collect()
9341 }
9342
9343 fn route_open_frame_with_admission_facts(
9344 corr: u64,
9345 module_id: &str,
9346 _project_root: ScratchDir,
9347 consumer_identity: Option<subc_control::ConsumerIdentity>,
9348 facts: Option<Value>,
9349 ) -> Frame {
9350 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9351 busy_refusals: None,
9352 target: RouteTarget::ToolProvider {
9353 module_id: module_id.to_string(),
9354 },
9355 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9356 consumer_identity,
9357 consumer_capabilities: None,
9358 role_versions: None,
9359 admission_facts: facts,
9360 scope: None,
9361 })
9362 .unwrap();
9363 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9364 }
9365
9366 #[derive(Clone, Default)]
9367 struct EventCapture {
9368 events: Arc<Mutex<Vec<CapturedEvent>>>,
9369 }
9370
9371 #[derive(Clone, Debug)]
9372 struct CapturedEvent {
9373 target: String,
9374 level: tracing::Level,
9375 fields: BTreeMap<String, String>,
9376 }
9377
9378 impl EventCapture {
9379 fn events(&self) -> Vec<CapturedEvent> {
9380 self.events.lock().unwrap().clone()
9381 }
9382 }
9383
9384 impl<S> Layer<S> for EventCapture
9385 where
9386 S: Subscriber,
9387 {
9388 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
9389 let mut visitor = EventFieldVisitor::default();
9390 event.record(&mut visitor);
9391 self.events.lock().unwrap().push(CapturedEvent {
9392 target: event.metadata().target().to_string(),
9393 level: *event.metadata().level(),
9394 fields: visitor.fields,
9395 });
9396 }
9397 }
9398
9399 #[derive(Default)]
9400 struct EventFieldVisitor {
9401 fields: BTreeMap<String, String>,
9402 }
9403
9404 impl Visit for EventFieldVisitor {
9405 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
9406 self.fields
9407 .insert(field.name().to_string(), format!("{value:?}"));
9408 }
9409 }
9410
9411 fn health_response(corr: u64, status: HealthStatus) -> Frame {
9412 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
9413 status,
9414 detail: Some("warming".to_string()),
9415 metrics: Some(json!({"queue_depth": 3})),
9416 })
9417 .unwrap();
9418 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9419 }
9420
9421 fn route_bind_ack(corr: u64) -> Frame {
9422 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
9423 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9424 }
9425
9426 fn unique_project_root(label: &str) -> ScratchDir {
9427 ScratchDir::new(label)
9428 }
9429
9430 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
9431 match parse_route_poll(frame) {
9432 ClientControlResponse::RoutePoll {
9433 status: None,
9434 live: Some(live),
9435 ..
9436 } => assert_eq!(live, expected_live),
9437 other => panic!("unexpected route.poll response: {other:?}"),
9438 }
9439 }
9440
9441 fn bind_liveness_route(
9442 registry: &Registry,
9443 forwarding: &ForwardingTable,
9444 module_id: &str,
9445 ) -> (RouteCtx, u16, u32) {
9446 let module_connection = ConnectionId::new(101);
9447 let client_connection = ConnectionId::new(202);
9448 let registration = registry
9449 .register_with_control_ops(
9450 manifest(module_id, PROTOCOL_VERSION),
9451 PROTOCOL_VERSION,
9452 module_connection,
9453 module_baseline_control_ops(),
9454 )
9455 .unwrap();
9456 let (module_tx, _module_rx) = mpsc::channel(8);
9457 let endpoint = forwarding
9458 .register_module_connection(
9459 module_connection,
9460 module_id.to_string(),
9461 PROTOCOL_VERSION,
9462 manifest_concurrency(®istration.manifest),
9463 FrameSink::new(module_tx),
9464 )
9465 .unwrap();
9466 let (client_ctx, _client_rx) = route_ctx(client_connection);
9467 let pending = forwarding
9468 .begin_route_bind_relay_for_test(
9469 client_connection,
9470 client_ctx.egress.clone(),
9471 1,
9472 module_id,
9473 )
9474 .unwrap();
9475 assert_eq!(pending.endpoint, endpoint);
9476 let route_channel = pending.client_channel;
9477 let route_epoch = pending.client_epoch;
9478 forwarding
9479 .complete_pending_relay(
9480 module_connection,
9481 pending.corr,
9482 RouteBindRelayOutcome::Accepted,
9483 )
9484 .unwrap();
9485 (client_ctx, route_channel, route_epoch)
9486 }
9487
9488 struct FakeProcessLiveness {
9489 live: Option<bool>,
9490 }
9491
9492 impl ModuleProcessLiveness for FakeProcessLiveness {
9493 fn process_live(&self, _module_id: &str) -> Option<bool> {
9494 self.live
9495 }
9496 }
9497
9498 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9499 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
9500 {
9501 let registry = Arc::new(Registry::default());
9502 let supervisor_handle = SupervisorHandle::new();
9503 let supervisor = Supervisor::new_for_test(
9504 Arc::clone(®istry),
9505 RestartPolicy::new(1, Duration::from_millis(10)),
9506 )
9507 .with_handle(supervisor_handle.clone());
9508 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
9509 let module = supervisor
9510 .spawn(ModuleSpec {
9511 carry_containment: false,
9512 module_id: "stderr-tail-wire".to_string(),
9513 program: fake_aft_stub_path(),
9514 args: Vec::new(),
9515 env: vec![
9516 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
9517 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
9518 ],
9519 reserved: false,
9520 reserved_prefixes: Vec::new(),
9521 protocol: ModuleProtocol::Subc,
9522 overlap: Default::default(),
9523 })
9524 .unwrap();
9525
9526 let deadline = Instant::now() + Duration::from_secs(5);
9527 loop {
9528 let tail = module.stderr_tail(None, None);
9529 if tail
9530 .entries
9531 .iter()
9532 .any(|entry| matches!(entry, TailEntry::ProcessStart))
9533 && tail.entries.iter().any(|entry| {
9534 matches!(
9535 entry,
9536 TailEntry::Line {
9537 truncated: true,
9538 ..
9539 }
9540 )
9541 })
9542 {
9543 break;
9544 }
9545 assert!(
9546 Instant::now() < deadline,
9547 "module did not produce a truncated line and restart boundary: {tail:?}"
9548 );
9549 sleep(Duration::from_millis(10)).await;
9550 }
9551
9552 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9553 let request = ClientControlRequest::SupervisorStderrTail {
9554 module_id: "stderr-tail-wire".to_string(),
9555 max_lines: None,
9556 max_bytes: None,
9557 };
9558 let frame = Frame::build(
9559 FrameType::Request,
9560 control_flags(),
9561 0,
9562 0,
9563 1,
9564 serde_json::to_vec(&request).unwrap(),
9565 )
9566 .unwrap();
9567 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9568 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9569 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
9570 serde_json::from_slice(&responses[0].body).unwrap()
9571 else {
9572 panic!("expected supervisor.stderr_tail response");
9573 };
9574
9575 assert!(
9576 tail.entries
9577 .iter()
9578 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
9579 "the control response lost the restart boundary"
9580 );
9581 let Some(StderrTailEntry::Line {
9582 text,
9583 truncated,
9584 at_ms,
9585 }) = tail.entries.iter().find(|entry| {
9586 matches!(
9587 entry,
9588 StderrTailEntry::Line {
9589 truncated: true,
9590 ..
9591 }
9592 )
9593 })
9594 else {
9595 panic!("the control response lost the truncated line");
9596 };
9597 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
9598 assert!(*truncated);
9599 assert!(
9600 at_ms.is_some(),
9601 "the control response lost the line's capture time"
9602 );
9603 }
9604
9605 #[tokio::test(flavor = "current_thread")]
9610 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
9611 let dir = ScratchDir::new("terminals-off-worker");
9612 let journal_path = dir.join("terminals.jsonl");
9613 let registry = Arc::new(Registry::default());
9614 let supervisor_handle = SupervisorHandle::new();
9615 let supervisor =
9616 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9617 .with_handle(supervisor_handle.clone())
9618 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
9619 let module = supervisor
9620 .spawn(ModuleSpec {
9621 carry_containment: false,
9622 module_id: "terminal-off-worker".to_string(),
9623 program: fake_aft_stub_path(),
9624 args: Vec::new(),
9625 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9626 reserved: false,
9627 reserved_prefixes: Vec::new(),
9628 protocol: ModuleProtocol::Subc,
9629 overlap: Default::default(),
9630 })
9631 .unwrap();
9632 let deadline = Instant::now() + Duration::from_secs(5);
9633 while module.terminal_history().entries.len() != 2 {
9634 assert!(Instant::now() < deadline, "module did not record two exits");
9635 sleep(Duration::from_millis(10)).await;
9636 }
9637
9638 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
9639 let handler =
9640 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
9641 let frame = Frame::build(
9642 FrameType::Request,
9643 control_flags(),
9644 0,
9645 0,
9646 1,
9647 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
9648 module_id: "terminal-off-worker".to_string(),
9649 })
9650 .unwrap(),
9651 )
9652 .unwrap();
9653 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9654 let spawned_at = std::time::Instant::now();
9655 let read = tokio::spawn({
9656 let handler = Arc::clone(&handler);
9657 async move { handler.handle_control_frame(&ctx, frame).await }
9658 });
9659 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
9662 .await
9663 .unwrap()
9664 .expect("the history read reached its pause");
9665 let elapsed = spawned_at.elapsed();
9666 assert!(
9667 elapsed < Duration::from_secs(2) && !read.is_finished(),
9668 "this task could not run while the history read was paused \
9669 (resumed after {elapsed:?}, read finished: {})",
9670 read.is_finished()
9671 );
9672
9673 drop(release);
9674 let responses = read.await.unwrap().unwrap();
9675 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9676 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
9677 panic!("expected supervisor.terminals response");
9678 };
9679 assert_eq!(terminals.entries.len(), 2);
9680 assert_eq!(terminals.journal_skipped_lines, 0);
9681 assert_eq!(terminals.journal_read_errors, 0);
9682 }
9683
9684 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9685 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
9686 let registry = Arc::new(Registry::default());
9687 let supervisor_handle = SupervisorHandle::new();
9688 let supervisor =
9689 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9690 .with_handle(supervisor_handle.clone());
9691 let module = supervisor
9692 .spawn(ModuleSpec {
9693 carry_containment: false,
9694 module_id: "terminal-golden".to_string(),
9695 program: fake_aft_stub_path(),
9696 args: Vec::new(),
9697 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9698 reserved: false,
9699 reserved_prefixes: Vec::new(),
9700 protocol: ModuleProtocol::Subc,
9701 overlap: Default::default(),
9702 })
9703 .unwrap();
9704
9705 let deadline = Instant::now() + Duration::from_secs(5);
9706 while module.terminal_history().entries.len() != 2 {
9707 assert!(
9708 Instant::now() < deadline,
9709 "module did not retain two terminal exits: {:?}",
9710 module.terminal_history()
9711 );
9712 sleep(Duration::from_millis(10)).await;
9713 }
9714
9715 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9716 let request = ClientControlRequest::SupervisorTerminals {
9717 module_id: "terminal-golden".to_string(),
9718 };
9719 let frame = Frame::build(
9720 FrameType::Request,
9721 control_flags(),
9722 0,
9723 0,
9724 1,
9725 serde_json::to_vec(&request).unwrap(),
9726 )
9727 .unwrap();
9728 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9729 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9730 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9731 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9732 panic!("expected supervisor.terminals response");
9733 };
9734 assert_eq!(terminals.entries.len(), 2);
9735 assert_eq!(terminals.dropped, 0);
9736
9737 let mut rendered = serde_json::to_value(response).unwrap();
9738 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9741 for (index, entry) in rendered["entries"]
9742 .as_array_mut()
9743 .expect("terminal response entries array")
9744 .iter_mut()
9745 .enumerate()
9746 {
9747 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9748 }
9749
9750 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9751 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9752 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9753 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9754 std::fs::write(&golden_path, &serialized).unwrap();
9755 }
9756 let expected: Value =
9757 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9758 assert_eq!(rendered, expected);
9759 }
9760
9761 #[test]
9762 fn hello_registers_manifest_and_returns_ack() {
9763 let registry = Arc::new(Registry::default());
9764 let handler = ControlHandler::new(Arc::clone(®istry));
9765 let conn = ConnectionId::new(1);
9766
9767 let responses = handler
9768 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9769 .unwrap();
9770
9771 assert_eq!(responses.len(), 1);
9772 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9773 assert_eq!(responses[0].header.channel, 0);
9774 assert_eq!(responses[0].header.corr, 7);
9775 let ack = parse_ack(&responses[0]);
9776 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9777 assert!(ack
9778 .subc_capabilities
9779 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9780 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9781 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9782 assert!(ack
9783 .subc_ops
9784 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9785 assert!(ack
9786 .subc_ops
9787 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9788
9789 let registration = registry.get_module("aft").unwrap().unwrap();
9790 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9791 assert_eq!(registration.state, ChannelState::Active);
9792 assert_eq!(registration.connection_id, conn);
9793 assert_eq!(registration.control_ops, module_baseline_control_ops());
9794 assert!(registration.manifest.display_name.is_none());
9795 }
9796
9797 #[test]
9798 fn hello_display_name_refuses_invalid_manifest_by_field() {
9799 let registry = Arc::new(Registry::default());
9800 let handler = ControlHandler::new(Arc::clone(®istry));
9801 for (index, invalid) in [
9802 json!(""),
9803 json!("a".repeat(41)),
9804 json!(" Routing"),
9805 json!("a\nb"),
9806 json!("a\u{202e}b"),
9807 json!(7),
9808 ]
9809 .into_iter()
9810 .enumerate()
9811 {
9812 let module_id = format!("bad-display-name-{index}");
9813 let mut body = serde_json::to_value(ModuleHelloBody {
9814 manifest: manifest(&module_id, PROTOCOL_VERSION),
9815 protocol_ver: PROTOCOL_VERSION,
9816 control_ops: None,
9817 launch_nonce: None,
9818 })
9819 .unwrap();
9820 body["manifest"]["display_name"] = invalid;
9821 let hello = Frame::build(
9822 FrameType::Hello,
9823 control_flags(),
9824 0,
9825 0,
9826 7,
9827 serde_json::to_vec(&body).unwrap(),
9828 )
9829 .unwrap();
9830 let response = handler
9831 .handle_control(ConnectionId::new(100 + index as u64), hello)
9832 .unwrap();
9833 assert_eq!(response[0].header.ty, FrameType::Error);
9834 let error = parse_error(&response[0]);
9835 assert_eq!(error["code"], "invalid_manifest");
9836 assert!(error["message"].as_str().unwrap().contains("display_name"));
9837 assert!(registry.get_module(&module_id).unwrap().is_none());
9838 }
9839 }
9840
9841 #[test]
9842 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9843 let invalid_identifiers = [
9844 ("case_change", "credentials-Provider/v1"),
9845 ("leading_zero", "credentials-provider/v01"),
9846 ("trailing_hyphen", "credentials-provider-/v1"),
9847 ("consecutive_hyphens", "credentials--provider/v1"),
9848 ("uppercase", "Credentials-provider/v1"),
9849 ("missing_v", "credentials-provider/1"),
9850 ("whitespace", "credentials provider/v1"),
9851 ("zero_version", "credentials-provider/v0"),
9852 ("out_of_range_version", "credentials-provider/v4294967296"),
9853 (
9854 "overlength_name",
9855 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9856 ),
9857 ];
9858 let mut cases = invalid_identifiers
9859 .into_iter()
9860 .map(|(name, identifier)| {
9861 (
9862 format!("identifier_{name}"),
9863 "capabilities.provides[0]".to_string(),
9864 identifier.to_string(),
9865 json!({ "provides": [identifier] }),
9866 None,
9867 )
9868 })
9869 .collect::<Vec<_>>();
9870 cases.extend([
9871 (
9872 "unknown_need".to_string(),
9873 "capabilities.requires[0].need".to_string(),
9874 "deferred".to_string(),
9875 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9876 None,
9877 ),
9878 (
9879 "duplicate_provides".to_string(),
9880 "capabilities.provides[1]".to_string(),
9881 "credentials-provider/v1".to_string(),
9882 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9883 None,
9884 ),
9885 (
9886 "duplicate_must_never_reach".to_string(),
9887 "capabilities.must_never_reach[1]".to_string(),
9888 "credentials-provider/v1".to_string(),
9889 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9890 None,
9891 ),
9892 (
9893 "duplicate_requires_same_need".to_string(),
9894 "capabilities.requires[1]".to_string(),
9895 "credentials-provider/v1".to_string(),
9896 json!({ "requires": [
9897 { "capability": "credentials-provider/v1", "need": "required" },
9898 { "capability": "credentials-provider/v1", "need": "required" }
9899 ] }),
9900 None,
9901 ),
9902 (
9903 "duplicate_requires_conflicting_need".to_string(),
9904 "capabilities.requires[1]".to_string(),
9905 "credentials-provider/v1".to_string(),
9906 json!({ "requires": [
9907 { "capability": "credentials-provider/v1", "need": "required" },
9908 { "capability": "credentials-provider/v1", "need": "optional" }
9909 ] }),
9910 None,
9911 ),
9912 (
9913 "capabilities_root_pointer".to_string(),
9914 "runtime_computed[0]".to_string(),
9915 "/capabilities".to_string(),
9916 json!({}),
9917 Some(json!(["/capabilities"])),
9918 ),
9919 (
9920 "capabilities_descendant_pointer".to_string(),
9921 "runtime_computed[0]".to_string(),
9922 "/capabilities/provides".to_string(),
9923 json!({}),
9924 Some(json!(["/capabilities/provides"])),
9925 ),
9926 (
9927 "malformed_pointer_without_leading_slash".to_string(),
9928 "runtime_computed[0]".to_string(),
9929 "capabilities".to_string(),
9930 json!({}),
9931 Some(json!(["capabilities"])),
9932 ),
9933 (
9934 "malformed_pointer_escape".to_string(),
9935 "runtime_computed[0]".to_string(),
9936 "/roles/~2/tools".to_string(),
9937 json!({}),
9938 Some(json!(["/roles/~2/tools"])),
9939 ),
9940 (
9941 "unknown_capabilities_field".to_string(),
9942 "capabilities.future".to_string(),
9943 "<array>".to_string(),
9944 json!({ "future": [] }),
9945 None,
9946 ),
9947 ]);
9948
9949 for (index, (name, field, value, capabilities, runtime_computed)) in
9950 cases.into_iter().enumerate()
9951 {
9952 let registry = Arc::new(Registry::default());
9953 let handler = ControlHandler::new(Arc::clone(®istry));
9954 let response = handler
9955 .handle_control(
9956 ConnectionId::new((index + 1) as u64),
9957 capability_grammar_hello_frame(
9958 capabilities,
9959 runtime_computed,
9960 index as u64 + 1,
9961 ),
9962 )
9963 .expect("invalid HELLO returns a refusal");
9964
9965 assert_eq!(response.len(), 1, "{name} must emit one refusal");
9966 let error = parse_error(&response[0]);
9967 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9968 let message = error["message"]
9969 .as_str()
9970 .expect("error message is a string");
9971 assert!(
9972 message.contains(&field),
9973 "{name}: field missing from {message}"
9974 );
9975 assert!(
9976 message.contains(&value),
9977 "{name}: value missing from {message}"
9978 );
9979 assert_eq!(
9980 registry
9981 .active_registration_count()
9982 .expect("registry reads"),
9983 0,
9984 "{name}: refused HELLO must not create a catalog entry"
9985 );
9986 }
9987 }
9988
9989 #[test]
9990 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9991 let registry = Arc::new(Registry::default());
9992 let handler = ControlHandler::new(Arc::clone(®istry));
9993 let capabilities = json!({
9994 "provides": ["credentials-provider/v1"],
9995 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9996 "must_never_reach": ["federation-transport/v1"]
9997 });
9998 let response = handler
9999 .handle_control(
10000 ConnectionId::new(99),
10001 capability_grammar_hello_frame(
10002 capabilities.clone(),
10003 Some(json!(["/roles/0/tools"])),
10004 99,
10005 ),
10006 )
10007 .expect("valid HELLO registers");
10008 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10009
10010 let request = Frame::build(
10011 FrameType::Request,
10012 control_flags(),
10013 0,
10014 0,
10015 100,
10016 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10017 .expect("catalog request serializes"),
10018 )
10019 .expect("catalog request frame builds");
10020 let response = handler
10021 .handle_catalog_list(request, None)
10022 .expect("catalog list succeeds");
10023 let ClientControlResponse::CatalogList { modules, .. } =
10024 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
10025 else {
10026 panic!("catalog request must return catalog.list");
10027 };
10028 assert_eq!(modules.len(), 1);
10029 assert_eq!(
10030 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
10031 capabilities
10032 );
10033 }
10034
10035 #[test]
10036 fn catalog_list_mirrors_management_operation_description() {
10037 let registry = Arc::new(Registry::default());
10038 let handler = ControlHandler::new(Arc::clone(®istry));
10039 let description = "List managed records and return their identifiers and metadata.";
10040 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
10041 manifest.provides = vec![ProviderRole::ManagementSurface {
10042 operations: vec![ManagementOperation {
10043 name: "records.list".to_string(),
10044 kind: ManagementOperationKind::Query,
10045 description: Some(description.to_string()),
10046 }],
10047 config_schema: json!({"type": "object"}),
10048 observability: vec![ObservabilitySurface {
10049 name: "records.stats".to_string(),
10050 kind: ObservabilityKind::Snapshot,
10051 }],
10052 identity_scope: vec![IdentityScope::Project],
10053 concurrency: Concurrency::ModuleManaged,
10054 }];
10055 registry
10056 .register_with_control_ops(
10057 manifest,
10058 PROTOCOL_VERSION,
10059 ConnectionId::new(99),
10060 Vec::new(),
10061 )
10062 .expect("described management manifest registers");
10063
10064 let request = Frame::build(
10065 FrameType::Request,
10066 control_flags(),
10067 0,
10068 0,
10069 100,
10070 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10071 .expect("catalog request serializes"),
10072 )
10073 .expect("catalog request frame builds");
10074 let response = handler
10075 .handle_catalog_list(request, None)
10076 .expect("catalog list succeeds");
10077 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10078 assert_eq!(
10079 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
10080 "catalog.list must preserve the declared operation description verbatim"
10081 );
10082 }
10083
10084 #[test]
10085 fn catalog_list_mirrors_registered_operation_delivery() {
10086 let registry = Arc::new(Registry::default());
10087 let handler = ControlHandler::new(Arc::clone(®istry));
10088 let delivery =
10089 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
10090 let mut body = serde_json::to_value(ModuleHelloBody {
10091 manifest: manifest("watch-management", PROTOCOL_VERSION),
10092 protocol_ver: PROTOCOL_VERSION,
10093 control_ops: None,
10094 launch_nonce: None,
10095 })
10096 .unwrap();
10097 body["manifest"]["provides"] = json!([{
10098 "role": "management_surface",
10099 "operations": [
10100 {"name": "status", "kind": "query"},
10101 {"name": "future", "kind": "query"},
10102 {"name": "reset", "kind": "mutate"}
10103 ],
10104 "config_schema": {}, "observability": [], "identity_scope": []
10105 }]);
10106 body["manifest"]["operation_delivery"] = delivery.clone();
10107 let hello = Frame::build(
10108 FrameType::Hello,
10109 control_flags(),
10110 0,
10111 0,
10112 99,
10113 serde_json::to_vec(&body).unwrap(),
10114 )
10115 .unwrap();
10116 let response = handler
10117 .handle_control(ConnectionId::new(99), hello)
10118 .unwrap();
10119 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10120
10121 let (_, registrations) = registry.list_modules().unwrap();
10122 let registered = ®istrations[0].manifest;
10123 assert!(registered.is_watch_operation("status"));
10124 assert!(!registered.is_watch_operation("future"));
10125 assert!(!registered.is_watch_operation("reset"));
10126 assert!(!registered.is_watch_operation("missing"));
10127
10128 let request = Frame::build(
10129 FrameType::Request,
10130 control_flags(),
10131 0,
10132 0,
10133 100,
10134 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10135 )
10136 .unwrap();
10137 let response = handler.handle_catalog_list(request, None).unwrap();
10138 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
10139 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
10140 let ClientControlResponse::CatalogList { modules, .. } =
10141 serde_json::from_slice(&response[0].body).unwrap()
10142 else {
10143 panic!("expected catalog.list");
10144 };
10145 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
10146 }
10147
10148 #[test]
10149 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
10150 let registry = Arc::new(Registry::default());
10151 let supervisor = SupervisorHandle::new();
10152 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10153 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10154 let descriptor = json!({
10155 "schema": "ckios.richui.manifest/1",
10156 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10157 "bytes": 4096,
10158 "chunks": 4
10159 });
10160 let connection = ConnectionId::new(201);
10161 let hello = handler
10162 .handle_control(
10163 connection,
10164 hello_frame_with_rich_ui(
10165 "vault",
10166 PROTOCOL_VERSION,
10167 201,
10168 Some("the-real-nonce"),
10169 descriptor.clone(),
10170 ),
10171 )
10172 .unwrap();
10173 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10174
10175 let request =
10176 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
10177 let provides = registry
10178 .get_module_by_connection(connection)
10179 .unwrap()
10180 .unwrap()
10181 .manifest
10182 .provides;
10183 handler
10184 .handle_catalog_update(connection, request, provides, None, Some(false))
10185 .unwrap();
10186
10187 let request = Frame::build(
10188 FrameType::Request,
10189 control_flags(),
10190 0,
10191 0,
10192 203,
10193 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10194 )
10195 .unwrap();
10196 let response = handler.handle_catalog_list(request, None).unwrap();
10197 let ClientControlResponse::CatalogList { modules, .. } =
10198 serde_json::from_slice(&response[0].body).unwrap()
10199 else {
10200 panic!("expected catalog.list");
10201 };
10202 assert_eq!(
10203 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
10204 descriptor
10205 );
10206 assert!(modules[0].nonce_verified);
10207 }
10208
10209 fn hello_with_role_declarations(declarations: Value) -> Frame {
10210 let mut body = serde_json::to_value(ModuleHelloBody {
10211 manifest: manifest("role-test", PROTOCOL_VERSION),
10212 protocol_ver: PROTOCOL_VERSION,
10213 control_ops: None,
10214 launch_nonce: None,
10215 })
10216 .unwrap();
10217 body["manifest"]["role_declarations"] = declarations;
10218 Frame::build(
10219 FrameType::Hello,
10220 control_flags(),
10221 0,
10222 0,
10223 301,
10224 serde_json::to_vec(&body).unwrap(),
10225 )
10226 .unwrap()
10227 }
10228
10229 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
10230 let request = Frame::build(
10231 FrameType::Request,
10232 control_flags(),
10233 0,
10234 0,
10235 302,
10236 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10237 )
10238 .unwrap();
10239 let response = handler.handle_catalog_list(request, None).unwrap();
10240 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
10241 let ClientControlResponse::CatalogList { mut modules, .. } =
10242 serde_json::from_slice(&response[0].body).unwrap()
10243 else {
10244 panic!("expected catalog.list");
10245 };
10246 (raw["modules"][0].clone(), modules.remove(0))
10247 }
10248
10249 #[test]
10250 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
10251 let registry = Arc::new(Registry::default());
10252 let handler = ControlHandler::new(Arc::clone(®istry));
10253 let declarations = json!({
10254 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
10255 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
10256 });
10257 let connection = ConnectionId::new(301);
10258 let response = handler
10259 .handle_control(
10260 connection,
10261 hello_with_role_declarations(declarations.clone()),
10262 )
10263 .unwrap();
10264 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10265 let registered = registry
10266 .get_module_by_connection(connection)
10267 .unwrap()
10268 .unwrap()
10269 .manifest;
10270 assert_eq!(
10274 serde_json::to_value(®istered.role_declarations).unwrap(),
10275 declarations
10276 );
10277 let (raw, entry) = role_declarations_catalog(&handler);
10278 assert_eq!(raw["role_declarations"], declarations);
10279 assert_eq!(entry.role_declarations, registered.role_declarations);
10280
10281 let request =
10282 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
10283 handler
10284 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
10285 .unwrap();
10286 let (raw, entry) = role_declarations_catalog(&handler);
10287 assert_eq!(raw["role_declarations"], declarations);
10288 assert_eq!(entry.role_declarations, registered.role_declarations);
10289 }
10290
10291 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
10292 let registry = Arc::new(Registry::default());
10293 let handler = ControlHandler::new(Arc::clone(®istry));
10294 let capture = EventCapture::default();
10295 let _subscriber =
10296 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10297 let response = handler
10298 .handle_control(
10299 ConnectionId::new(301),
10300 hello_with_role_declarations(declarations),
10301 )
10302 .unwrap();
10303 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10304 assert!(registry
10305 .get_module_by_connection(ConnectionId::new(301))
10306 .unwrap()
10307 .unwrap()
10308 .manifest
10309 .role_declarations
10310 .is_empty());
10311 let (raw, entry) = role_declarations_catalog(&handler);
10312 assert!(raw.get("role_declarations").is_none());
10313 assert!(entry.role_declarations.is_empty());
10314 let warnings: Vec<_> = capture
10315 .events()
10316 .into_iter()
10317 .filter(|event| {
10318 event.fields.get("message").map(String::as_str)
10319 == Some("dropping whole role_declarations map from HELLO")
10320 })
10321 .collect();
10322 assert_eq!(warnings.len(), 1);
10323 assert_eq!(warnings[0].level, tracing::Level::WARN);
10324 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
10325 assert_eq!(warnings[0].fields["reason"], reason);
10326 }
10327
10328 #[test]
10329 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
10330 for value in [
10331 json!(null),
10332 json!(false),
10333 json!(0),
10334 json!("opaque"),
10335 json!([]),
10336 ] {
10337 assert_hello_drops_role_declarations_with_warning(
10338 json!({"routing/v1": {}, "broken/v1": value}),
10339 "role_declarations values must be objects",
10340 );
10341 }
10342 assert_hello_drops_role_declarations_with_warning(
10343 json!([]),
10344 "role_declarations must be an object",
10345 );
10346 }
10347
10348 #[test]
10349 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
10350 assert_hello_drops_role_declarations_with_warning(
10351 json!({"routing/v1": {}, "bad key": {}}),
10352 "role_declarations key must use capability-name grammar",
10353 );
10354 }
10355
10356 #[test]
10357 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
10358 assert_hello_drops_role_declarations_with_warning(
10359 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
10360 "role_declarations exceeds 16384 compact JSON bytes",
10361 );
10362 }
10363
10364 fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
10365 let mut body = serde_json::to_value(ModuleHelloBody {
10366 manifest: manifest("flow-ops", PROTOCOL_VERSION),
10367 protocol_ver: PROTOCOL_VERSION,
10368 control_ops: None,
10369 launch_nonce: None,
10370 })
10371 .unwrap();
10372 body["manifest"]["provides"] = json!([{
10373 "role": "management_surface",
10374 "operations": [
10375 {"name": "diagnostics.findings", "kind": "query"},
10376 {"name": "reset", "kind": "mutate"}
10377 ],
10378 "config_schema": {}, "observability": [], "identity_scope": []
10379 }]);
10380 body["manifest"]["capabilities"] = json!({ "provides": capabilities });
10381 if let Some(flow_operations) = flow_operations {
10382 body["manifest"]["flow_operations"] = flow_operations;
10383 }
10384 Frame::build(
10385 FrameType::Hello,
10386 control_flags(),
10387 0,
10388 0,
10389 401,
10390 serde_json::to_vec(&body).unwrap(),
10391 )
10392 .unwrap()
10393 }
10394
10395 #[test]
10396 fn catalog_list_mirrors_flow_operations_unchanged() {
10397 for (capability, flow_operations) in [
10400 ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
10401 ("call-modes/v1", json!(["diagnostics.findings"])),
10402 ("flow-scopes/v1", json!([])),
10403 ] {
10404 let registry = Arc::new(Registry::default());
10405 let handler = ControlHandler::new(Arc::clone(®istry));
10406 let connection = ConnectionId::new(401);
10407 let response = handler
10408 .handle_control(
10409 connection,
10410 hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
10411 )
10412 .unwrap();
10413 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10414 let registered = registry
10415 .get_module_by_connection(connection)
10416 .unwrap()
10417 .unwrap()
10418 .manifest;
10419 assert_eq!(
10420 serde_json::to_value(®istered.flow_operations).unwrap(),
10421 flow_operations
10422 );
10423 let (raw, entry) = role_declarations_catalog(&handler);
10424 assert_eq!(raw["flow_operations"], flow_operations);
10425 assert_eq!(entry.flow_operations, registered.flow_operations);
10426 }
10427
10428 let registry = Arc::new(Registry::default());
10429 let handler = ControlHandler::new(Arc::clone(®istry));
10430 let response = handler
10431 .handle_control(
10432 ConnectionId::new(401),
10433 hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
10434 )
10435 .unwrap();
10436 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10437 let (raw, entry) = role_declarations_catalog(&handler);
10438 assert!(raw.get("flow_operations").is_none());
10439 assert_eq!(entry.flow_operations, None);
10440 }
10441
10442 fn assert_hello_drops_flow_operations_with_warning(
10443 capabilities: Value,
10444 flow_operations: Value,
10445 reason: &str,
10446 ) {
10447 let registry = Arc::new(Registry::default());
10448 let handler = ControlHandler::new(Arc::clone(®istry));
10449 let capture = EventCapture::default();
10450 let _subscriber =
10451 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10452 let response = handler
10453 .handle_control(
10454 ConnectionId::new(401),
10455 hello_with_flow_operations(capabilities, Some(flow_operations)),
10456 )
10457 .unwrap();
10458 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10459 let registered = registry
10460 .get_module_by_connection(ConnectionId::new(401))
10461 .unwrap()
10462 .expect("the module still registers")
10463 .manifest;
10464 assert_eq!(registered.module_id, "flow-ops");
10465 assert_eq!(registered.flow_operations, None);
10466 let (raw, entry) = role_declarations_catalog(&handler);
10467 assert_eq!(raw["module_id"], "flow-ops");
10468 assert!(raw.get("flow_operations").is_none());
10469 assert_eq!(entry.flow_operations, None);
10470 let warnings: Vec<_> = capture
10471 .events()
10472 .into_iter()
10473 .filter(|event| {
10474 event.fields.get("message").map(String::as_str)
10475 == Some("dropping flow_operations from HELLO")
10476 })
10477 .collect();
10478 assert_eq!(warnings.len(), 1);
10479 assert_eq!(warnings[0].level, tracing::Level::WARN);
10480 assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
10481 assert_eq!(warnings[0].fields["reason"], reason);
10482 }
10483
10484 #[test]
10485 fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
10486 assert_hello_drops_flow_operations_with_warning(
10487 json!(["flow-scopes/v1"]),
10488 json!(["diagnostics.findings", "missing"]),
10489 "flow_operations names 'missing', which is not a declared management operation",
10490 );
10491 }
10492
10493 #[test]
10494 fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
10495 assert_hello_drops_flow_operations_with_warning(
10496 json!([]),
10497 json!(["diagnostics.findings"]),
10498 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10499 );
10500 assert_hello_drops_flow_operations_with_warning(
10502 json!(["agent-run-scopes/v1"]),
10503 json!([]),
10504 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10505 );
10506 }
10507
10508 #[test]
10509 fn hello_malformed_flow_operations_is_dropped_with_warning() {
10510 assert_hello_drops_flow_operations_with_warning(
10511 json!(["flow-scopes/v1"]),
10512 json!(["reset", "reset"]),
10513 "flow_operations names 'reset' more than once",
10514 );
10515 assert_hello_drops_flow_operations_with_warning(
10516 json!(["flow-scopes/v1"]),
10517 json!(vec!["reset"; 257]),
10518 "flow_operations lists 257 names, more than 256",
10519 );
10520 assert_hello_drops_flow_operations_with_warning(
10521 json!(["flow-scopes/v1"]),
10522 json!("reset"),
10523 "flow_operations must be an array of operation names",
10524 );
10525 }
10526
10527 #[test]
10528 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
10529 let registry = Arc::new(Registry::default());
10530 let handler = ControlHandler::new(registry.clone());
10531 let descriptor = json!({
10532 "schema": "ckios.richui.manifest/1",
10533 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10534 "bytes": 128,
10535 "chunks": 2
10536 });
10537 let malformed = [
10538 ("schema", json!("")),
10539 ("schema", json!("a".repeat(65))),
10540 ("schema", json!("Bad schema")),
10541 (
10542 "manifest_digest",
10543 json!(format!("sha256:{}", "A".repeat(64))),
10544 ),
10545 ("manifest_digest", json!("sha256:abc")),
10546 ("bytes", json!(0)),
10547 ("bytes", json!(16_777_217)),
10548 ("bytes", json!("128")),
10549 ("chunks", json!(0)),
10550 ("chunks", json!(1025)),
10551 ("chunks", json!("2")),
10552 ];
10553 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
10554 let mut invalid = descriptor.clone();
10555 invalid[field] = invalid_value;
10556 let response = handler
10557 .handle_control(
10558 ConnectionId::new(210 + index as u64),
10559 hello_frame_with_rich_ui(
10560 &format!("bad-rich-ui-{index}"),
10561 PROTOCOL_VERSION,
10562 210 + index as u64,
10563 None,
10564 invalid,
10565 ),
10566 )
10567 .unwrap();
10568 assert_eq!(response[0].header.ty, FrameType::Error);
10569 let error = parse_error(&response[0]);
10570 assert_eq!(error["code"], "invalid_manifest");
10571 assert!(error["message"]
10572 .as_str()
10573 .unwrap()
10574 .contains(&format!("rich_ui.{field}")));
10575 assert!(registry
10576 .get_module(&format!("bad-rich-ui-{index}"))
10577 .unwrap()
10578 .is_none());
10579 }
10580 }
10581
10582 #[test]
10583 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
10584 let registry = Arc::new(Registry::default());
10585 let handler = ControlHandler::new(Arc::clone(®istry));
10586 let connection = ConnectionId::new(219);
10587 registry
10588 .register_with_control_ops(
10589 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
10590 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
10591 "bad schema",
10592 format!("sha256:{}", "a".repeat(64)),
10593 128,
10594 2,
10595 )))
10596 .build(),
10597 PROTOCOL_VERSION,
10598 connection,
10599 Vec::new(),
10600 )
10601 .unwrap();
10602 let request =
10603 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
10604 let response = handler
10605 .handle_catalog_update(connection, request, Vec::new(), None, None)
10606 .unwrap();
10607 assert_eq!(response[0].header.ty, FrameType::Error);
10608 let error = parse_error(&response[0]);
10609 assert_eq!(error["code"], "invalid_manifest");
10610 assert!(error["message"]
10611 .as_str()
10612 .unwrap()
10613 .contains("rich_ui.schema"));
10614 }
10615
10616 #[test]
10617 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
10618 let registry = Arc::new(Registry::default());
10619 let supervisor = SupervisorHandle::new();
10620 supervisor.open_swap("provider", "candidate-nonce".to_string());
10621 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10622 let connection = ConnectionId::new(218);
10623 let hello = handler
10624 .handle_control(
10625 connection,
10626 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
10627 )
10628 .unwrap();
10629 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10630 assert!(registry.list_modules().unwrap().1.is_empty());
10631
10632 registry.promote_candidate("provider").unwrap().unwrap();
10633 let request = Frame::build(
10634 FrameType::Request,
10635 control_flags(),
10636 0,
10637 0,
10638 219,
10639 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10640 )
10641 .unwrap();
10642 let response = handler.handle_catalog_list(request, None).unwrap();
10643 let ClientControlResponse::CatalogList { modules, .. } =
10644 serde_json::from_slice(&response[0].body).unwrap()
10645 else {
10646 panic!("expected catalog.list");
10647 };
10648 assert_eq!(modules.len(), 1);
10649 assert!(modules[0].nonce_verified);
10650 }
10651
10652 #[test]
10653 fn catalog_list_with_provenance_mirrors_declared_build_bytes() {
10654 for provenance_json in [
10655 r#"{"build_git_sha":"0123456789abcdef0123456789abcdef01234567","build_lock_digest":"abababababababababababababababababababababababababababababababab","wire_crate_version":"0.34.0","store_schema_version":"7","launch_nonce_source":"fd"}"#,
10656 r#"{"build_git_sha_absence_reason":"no_git_dir","build_lock_digest":"abababababababababababababababababababababababababababababababab"}"#,
10657 ] {
10658 let registry = Arc::new(Registry::default());
10659 let handler = ControlHandler::new(registry);
10660 let hello = Frame::build(
10661 FrameType::Hello,
10662 control_flags(),
10663 0,
10664 0,
10665 220,
10666 serde_json::to_vec(&ModuleHelloBody {
10667 manifest: ModuleManifest::builder("built", "0.1.0")
10668 .provenance(Some(serde_json::from_str(provenance_json).unwrap()))
10669 .build(),
10670 protocol_ver: PROTOCOL_VERSION,
10671 control_ops: None,
10672 launch_nonce: None,
10673 })
10674 .unwrap(),
10675 )
10676 .unwrap();
10677 assert_eq!(
10678 handler
10679 .handle_control(ConnectionId::new(220), hello)
10680 .unwrap()[0]
10681 .header
10682 .ty,
10683 FrameType::HelloAck
10684 );
10685 let request = Frame::build(
10686 FrameType::Request,
10687 control_flags(),
10688 0,
10689 0,
10690 221,
10691 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10692 )
10693 .unwrap();
10694 let response = handler.handle_catalog_list(request, None).unwrap();
10695 let ClientControlResponse::CatalogList { modules, .. } =
10696 serde_json::from_slice(&response[0].body).unwrap()
10697 else {
10698 panic!("expected catalog.list");
10699 };
10700 assert_eq!(modules.len(), 1);
10701 assert_eq!(
10702 serde_json::to_vec(&modules[0]).unwrap(),
10703 format!(
10704 r#"{{"module_id":"built","ready":true,"module_version":"0.1.0","provenance":{provenance_json},"roles":[],"control_ops":["route.bind","route.status"]}}"#
10705 )
10706 .as_bytes()
10707 );
10708 }
10709 }
10710
10711 #[test]
10712 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
10713 let registry = Arc::new(Registry::default());
10714 let handler = ControlHandler::new(Arc::clone(®istry));
10715 let hello = Frame::build(
10716 FrameType::Hello,
10717 control_flags(),
10718 0,
10719 0,
10720 220,
10721 serde_json::to_vec(&ModuleHelloBody {
10722 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
10723 protocol_ver: PROTOCOL_VERSION,
10724 control_ops: None,
10725 launch_nonce: Some("unverified".to_string()),
10726 })
10727 .unwrap(),
10728 )
10729 .unwrap();
10730 assert_eq!(
10731 handler
10732 .handle_control(ConnectionId::new(220), hello)
10733 .unwrap()[0]
10734 .header
10735 .ty,
10736 FrameType::HelloAck
10737 );
10738 let request = Frame::build(
10739 FrameType::Request,
10740 control_flags(),
10741 0,
10742 0,
10743 221,
10744 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10745 )
10746 .unwrap();
10747 let response = handler.handle_catalog_list(request, None).unwrap();
10748 let wire: serde_json::Value = serde_json::from_slice(&response[0].body).unwrap();
10749 assert!(wire["modules"][0].get("provenance").is_none());
10750 let ClientControlResponse::CatalogList { modules, .. } =
10751 serde_json::from_slice(&response[0].body).unwrap()
10752 else {
10753 panic!("expected catalog.list");
10754 };
10755 assert_eq!(
10756 serde_json::to_vec(&modules[0]).unwrap(),
10757 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
10758 );
10759 assert!(!modules[0].nonce_verified);
10760 assert!(modules[0].provenance.is_none());
10761 }
10762
10763 #[test]
10764 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
10765 let registry = Arc::new(Registry::default());
10766 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10767 [("vault".to_string(), true), ("squatter".to_string(), true)],
10768 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10769 );
10770 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
10771 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10772 provides: vec!["credentials-provider/v1".to_string()],
10773 requires: Vec::new(),
10774 must_never_reach: Vec::new(),
10775 });
10776 let frame = Frame::build(
10777 FrameType::Hello,
10778 control_flags(),
10779 0,
10780 0,
10781 77,
10782 serde_json::to_vec(&ModuleHelloBody {
10783 manifest: squatter,
10784 protocol_ver: PROTOCOL_VERSION,
10785 control_ops: None,
10786 launch_nonce: None,
10787 })
10788 .expect("HELLO serializes"),
10789 )
10790 .expect("HELLO frame builds");
10791 let response = handler
10792 .handle_control(ConnectionId::new(77), frame)
10793 .expect("reserved claim receives a typed refusal");
10794 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10795 assert_eq!(
10796 registry
10797 .active_registration_count()
10798 .expect("registry reads"),
10799 0,
10800 "a reserved capability refusal must not leave a catalog entry"
10801 );
10802 }
10803
10804 #[test]
10805 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
10806 for settlement in ["timeout", "inconclusive", "drop"] {
10807 let breakers = RouteBindBreakers::default();
10808 let RouteBindAdmission::Admitted {
10809 guard: mut old,
10810 probe: false,
10811 } = breakers.admit("prov")
10812 else {
10813 panic!("ordinary relay admitted")
10814 };
10815 let RouteBindAdmission::Admitted {
10816 guard: mut opener, ..
10817 } = breakers.admit("prov")
10818 else {
10819 panic!("second relay admitted")
10820 };
10821 assert!(
10822 !opener
10823 .record_timeout(1, Duration::ZERO)
10824 .unwrap()
10825 .reopened_after_probe
10826 );
10827 let RouteBindAdmission::Admitted {
10828 guard: mut probe,
10829 probe: true,
10830 } = breakers.admit("prov")
10831 else {
10832 panic!("one cooldown probe admitted")
10833 };
10834 match settlement {
10835 "timeout" => assert!(
10836 !old.record_timeout(1, Duration::ZERO)
10837 .unwrap()
10838 .reopened_after_probe
10839 ),
10840 "inconclusive" => old.record_inconclusive(),
10841 "drop" => drop(old),
10842 _ => unreachable!(),
10843 }
10844 assert!(
10845 matches!(
10846 breakers.admit("prov"),
10847 RouteBindAdmission::Refused {
10848 probe_in_flight: true,
10849 ..
10850 }
10851 ),
10852 "{settlement} of a pre-open relay cannot release the real probe"
10853 );
10854 assert!(
10855 probe
10856 .record_timeout(1, Duration::ZERO)
10857 .unwrap()
10858 .reopened_after_probe
10859 );
10860 assert!(matches!(
10861 breakers.admit("prov"),
10862 RouteBindAdmission::Admitted { probe: true, .. }
10863 ));
10864 }
10865 let breakers = RouteBindBreakers::default();
10866 let admit = || match breakers.admit("prov") {
10867 RouteBindAdmission::Admitted { guard, .. } => guard,
10868 _ => panic!("relay admitted"),
10869 };
10870 admit().record_timeout(1, Duration::ZERO);
10871 let mut old_probe = admit();
10872 breakers.reset_for_new_module_connection("prov");
10873 admit().record_timeout(1, Duration::ZERO);
10874 let _new_probe = admit();
10875 old_probe.record_inconclusive();
10876 assert!(matches!(
10877 breakers.admit("prov"),
10878 RouteBindAdmission::Refused {
10879 probe_in_flight: true,
10880 ..
10881 }
10882 ));
10883 }
10884
10885 #[tokio::test]
10886 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
10887 for candidate in [false, true] {
10888 let registry = Arc::new(Registry::default());
10889 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10890 [("vault".to_string(), true), ("squatter".to_string(), true)],
10891 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10892 );
10893 let conn = ConnectionId::new(77);
10894 let (ctx, mut rx) = route_ctx(conn);
10895 let initial = capability_manifest("squatter", &[], &[]);
10896 if candidate {
10897 registry
10898 .register_candidate_with_control_ops(
10899 initial.clone(),
10900 PROTOCOL_VERSION,
10901 conn,
10902 module_baseline_control_ops(),
10903 )
10904 .unwrap();
10905 handler
10906 .forwarding
10907 .register_candidate_module_connection(
10908 conn,
10909 "squatter".to_string(),
10910 PROTOCOL_VERSION,
10911 manifest_concurrency(&initial),
10912 ctx.egress.clone(),
10913 )
10914 .unwrap();
10915 } else {
10916 hello_via_sink(
10917 &handler,
10918 &ctx,
10919 &mut rx,
10920 hello_frame_with_manifest(initial.clone(), 1),
10921 )
10922 .await;
10923 }
10924 let response = handler
10925 .handle_control_frame(
10926 &ctx,
10927 catalog_update_with_capabilities_frame(
10928 2,
10929 capability_manifest("squatter", &["credentials-provider/v1"], &[])
10930 .capabilities
10931 .unwrap(),
10932 ),
10933 )
10934 .await
10935 .unwrap();
10936 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10937 assert_eq!(
10938 registry
10939 .get_module_by_connection(conn)
10940 .unwrap()
10941 .unwrap()
10942 .manifest,
10943 initial
10944 );
10945 }
10946 }
10947
10948 #[test]
10949 fn server_describe_surfaces_required_capability_verdict_fields() {
10950 let registry = Arc::new(Registry::default());
10951 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10952 [
10953 ("consumer".to_string(), true),
10954 ("provider".to_string(), false),
10955 ],
10956 BTreeMap::new(),
10957 );
10958 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10959 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10960 provides: Vec::new(),
10961 requires: vec![subc_protocol::manifest::CapabilityRequirement {
10962 capability: "credentials-provider/v1".to_string(),
10963 need: subc_protocol::manifest::CapabilityNeed::Required,
10964 }],
10965 must_never_reach: Vec::new(),
10966 });
10967 let hello = Frame::build(
10968 FrameType::Hello,
10969 control_flags(),
10970 0,
10971 0,
10972 78,
10973 serde_json::to_vec(&ModuleHelloBody {
10974 manifest: consumer,
10975 protocol_ver: PROTOCOL_VERSION,
10976 control_ops: None,
10977 launch_nonce: None,
10978 })
10979 .expect("HELLO serializes"),
10980 )
10981 .expect("HELLO frame builds");
10982 handler
10983 .handle_control(ConnectionId::new(78), hello)
10984 .expect("consumer registers");
10985 let describe = Frame::build(
10986 FrameType::Request,
10987 control_flags(),
10988 0,
10989 0,
10990 79,
10991 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
10992 .expect("request serializes"),
10993 )
10994 .expect("describe frame builds");
10995 let response = handler
10996 .handle_server_describe(describe)
10997 .expect("server.describe succeeds");
10998 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
10999 let requirement = &rendered["capability_requirements"][0];
11000 assert_eq!(requirement["consumer"], "consumer");
11001 assert_eq!(requirement["verdict"], "never_provided");
11002 assert_eq!(requirement["episode_seq"], 1);
11003 assert_eq!(requirement["config_satisfiable"], false);
11004 assert_eq!(requirement["runtime_available"], false);
11005 assert!(requirement["detail"]
11006 .as_str()
11007 .expect("detail string")
11008 .contains("credentials-provider/v1"));
11009 }
11010
11011 fn deregister_required_provider(
11017 daemon_shutdown_begun: bool,
11018 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
11019 const CAPABILITY: &str = "project-identity/v1";
11020 let registry = Arc::new(Registry::default());
11021 let daemon_shutdown = DaemonShutdownFlag::default();
11022 let handler = ControlHandler::new(Arc::clone(®istry))
11023 .with_capability_config(
11024 [
11025 ("consumer".to_string(), true),
11026 ("provider".to_string(), true),
11027 ],
11028 BTreeMap::new(),
11029 )
11030 .with_daemon_shutdown_flag(daemon_shutdown.clone());
11031 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
11032 consumer.capabilities = Some(CapabilityDeclarations {
11033 provides: Vec::new(),
11034 requires: vec![subc_protocol::manifest::CapabilityRequirement {
11035 capability: CAPABILITY.to_string(),
11036 need: subc_protocol::manifest::CapabilityNeed::Required,
11037 }],
11038 must_never_reach: Vec::new(),
11039 });
11040 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
11041 let provider_conn = ConnectionId::new(182);
11042 for (conn, manifest, corr) in [
11043 (ConnectionId::new(181), consumer, 181),
11044 (provider_conn, provider, 182),
11045 ] {
11046 let hello = handler
11047 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
11048 .expect("module registers");
11049 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11050 }
11051 assert_eq!(
11052 handler.capability_evaluator.verdict("consumer", CAPABILITY),
11053 Some(CapabilityVerdict::Provided),
11054 "the registered provider must satisfy the requirement first"
11055 );
11056 if daemon_shutdown_begun {
11057 daemon_shutdown.set_for_test();
11058 }
11059
11060 let capture = EventCapture::default();
11061 let _subscriber =
11062 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11063 handler
11064 .cleanup_connection(provider_conn)
11065 .expect("provider deregisters");
11066 let events = capture
11067 .events()
11068 .into_iter()
11069 .filter(|event| {
11070 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
11071 })
11072 .collect();
11073 (
11074 events,
11075 handler.capability_evaluator.verdict("consumer", CAPABILITY),
11076 )
11077 }
11078
11079 #[test]
11080 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
11081 let (events, verdict) = deregister_required_provider(true);
11082 assert_eq!(
11083 verdict,
11084 Some(CapabilityVerdict::NeverProvided),
11085 "shutdown must not change the verdict route.open refusals read"
11086 );
11087 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11088 let event = &events[0];
11089 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
11090 assert_eq!(
11091 event.fields.get("cause").map(String::as_str),
11092 Some("\"daemon_shutdown\""),
11093 "{event:?}"
11094 );
11095 assert_eq!(
11096 event.fields.get("verdict").map(String::as_str),
11097 Some("\"never_provided\""),
11098 "{event:?}"
11099 );
11100 }
11101
11102 #[test]
11103 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
11104 let (events, verdict) = deregister_required_provider(false);
11105 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
11106 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11107 let event = &events[0];
11108 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
11109 assert_eq!(event.fields.get("cause"), None, "{event:?}");
11110 assert_eq!(
11111 event.fields.get("verdict").map(String::as_str),
11112 Some("\"never_provided\""),
11113 "{event:?}"
11114 );
11115 }
11116
11117 #[test]
11118 fn catalog_list_omits_capabilities_for_legacy_manifest() {
11119 let registry = Arc::new(Registry::default());
11120 let handler = ControlHandler::new(Arc::clone(®istry));
11121 let hello = handler
11122 .handle_control(
11123 ConnectionId::new(101),
11124 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
11125 )
11126 .expect("legacy HELLO registers");
11127 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11128
11129 let request = Frame::build(
11130 FrameType::Request,
11131 control_flags(),
11132 0,
11133 0,
11134 102,
11135 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
11136 .expect("catalog request serializes"),
11137 )
11138 .expect("catalog request frame builds");
11139 let response = handler
11140 .handle_catalog_list(request, None)
11141 .expect("catalog list succeeds");
11142 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
11143 assert!(
11144 body["modules"][0].get("capabilities").is_none(),
11145 "legacy manifest must retain an absent capabilities field on catalog.list"
11146 );
11147 }
11148
11149 #[test]
11150 fn hello_ack_omits_storage_when_no_storage_config() {
11151 let registry = Arc::new(Registry::default());
11152 let handler = ControlHandler::new(Arc::clone(®istry));
11153 let responses = handler
11154 .handle_control(
11155 ConnectionId::new(1),
11156 hello_frame("aft", PROTOCOL_VERSION, 7),
11157 )
11158 .unwrap();
11159 let ack = parse_ack(&responses[0]);
11160 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
11161 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
11162 }
11163
11164 #[tokio::test]
11165 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
11166 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
11167 let registry = Arc::new(Registry::default());
11168 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
11169 let responses = handler
11170 .handle_control(
11171 ConnectionId::new(1),
11172 hello_frame("aft", PROTOCOL_VERSION, 7),
11173 )
11174 .unwrap();
11175 let ack = parse_ack(&responses[0]);
11176 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
11177
11178 let described = handler
11179 .handle_control_frame(
11180 &route_ctx(ConnectionId::new(2)).0,
11181 Frame::build(
11182 FrameType::Request,
11183 control_flags(),
11184 0,
11185 0,
11186 9,
11187 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11188 )
11189 .unwrap(),
11190 )
11191 .await
11192 .unwrap();
11193 let ClientControlResponse::ServerDescribe { machine_id, .. } =
11194 serde_json::from_slice(&described[0].body).unwrap()
11195 else {
11196 panic!("server.describe answered with another shape");
11197 };
11198 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
11199 }
11200
11201 #[test]
11202 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
11203 let registry = Arc::new(Registry::default());
11206 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
11207 crate::daemon_config::StorageConfig::Sqlite {
11208 data_home: std::path::PathBuf::from("/data"),
11209 },
11210 ));
11211
11212 let responses = handler
11213 .handle_control(
11214 ConnectionId::new(1),
11215 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
11216 )
11217 .unwrap();
11218 let ack = parse_ack(&responses[0]);
11219 assert_eq!(
11220 ack.storage,
11221 Some(serde_json::json!({
11222 "module_id": "alfonso-routing",
11223 "storage_namespace": "default",
11224 "isolation": { "kind": "module" },
11225 "backend": {
11226 "backend": "sqlite",
11227 "path": "/data/cortexkit/alfonso-routing/store.db"
11228 }
11229 })),
11230 "the delivered descriptor is the module's own sqlite store path"
11231 );
11232 }
11233
11234 #[test]
11235 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
11236 let registry = Arc::new(Registry::default());
11237 let handler = ControlHandler::new(Arc::clone(®istry));
11238 let conn = ConnectionId::new(1);
11239 let responses = handler
11240 .handle_control(
11241 conn,
11242 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11243 )
11244 .unwrap();
11245 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
11246 let registration = registry.get_module("aft").unwrap().unwrap();
11247 assert_eq!(registration.control_ops, module_baseline_control_ops());
11248
11249 let frame =
11250 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
11251 assert!(handler
11252 .guard_module_control_op(&frame, "aft", "route.bind")
11253 .unwrap()
11254 .is_none());
11255 let error = handler
11256 .guard_module_control_op(&frame, "aft", "test.synthetic")
11257 .unwrap()
11258 .expect("synthetic ungranted op should be rejected");
11259 assert_eq!(error.header.ty, FrameType::Error);
11260 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
11261 }
11262
11263 #[test]
11264 fn hello_control_ops_some_adds_optional_grants() {
11265 let registry = Arc::new(Registry::default());
11266 let handler = ControlHandler::new(Arc::clone(®istry));
11267 handler
11268 .handle_control(
11269 ConnectionId::new(1),
11270 hello_frame_with_control_ops(
11271 "aft",
11272 PROTOCOL_VERSION,
11273 7,
11274 Some(vec![
11275 "future.synthetic".to_string(),
11276 "route.bind".to_string(),
11277 ]),
11278 ),
11279 )
11280 .unwrap();
11281 let registration = registry.get_module("aft").unwrap().unwrap();
11282 assert_eq!(
11283 registration.control_ops,
11284 vec![
11285 "route.bind".to_string(),
11286 "route.status".to_string(),
11287 "future.synthetic".to_string(),
11288 ]
11289 );
11290 let frame =
11291 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
11292 assert!(handler
11293 .guard_module_control_op(&frame, "aft", "future.synthetic")
11294 .unwrap()
11295 .is_none());
11296 }
11297
11298 #[tokio::test]
11299 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
11300 let registry = Arc::new(Registry::default());
11301 let forwarding = Arc::new(ForwardingTable::default());
11302 let handler =
11303 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11304 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
11305 hello_via_sink(
11306 &handler,
11307 &module_ctx,
11308 &mut module_rx,
11309 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11310 )
11311 .await;
11312
11313 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
11314 let responses = handler
11315 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
11316 .await
11317 .unwrap();
11318 assert_eq!(responses.len(), 1);
11319 assert_eq!(responses[0].header.ty, FrameType::Error);
11320 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
11321 assert!(module_rx.try_recv().is_err());
11322 }
11323
11324 #[tokio::test]
11325 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
11326 let registry = Arc::new(Registry::default());
11327 let forwarding = Arc::new(ForwardingTable::default());
11328 let handler =
11329 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11330 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
11331 hello_via_sink(
11332 &handler,
11333 &module_ctx,
11334 &mut module_rx,
11335 hello_frame_with_control_ops(
11336 "aft",
11337 PROTOCOL_VERSION,
11338 7,
11339 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11340 ),
11341 )
11342 .await;
11343
11344 let project_root = unique_project_root("demux");
11345 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
11346 let route_handler = handler.clone();
11347 let route_task = tokio::spawn(async move {
11348 route_handler
11349 .handle_control_frame(
11350 &route_client_ctx,
11351 route_open_frame(100, "aft", project_root),
11352 )
11353 .await
11354 .unwrap()
11355 });
11356 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11357 .await
11358 .unwrap()
11359 .unwrap();
11360 assert!(matches!(
11361 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
11362 ModuleControlRequest::RouteBind { .. }
11363 ));
11364
11365 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
11366 let health_handler = handler.clone();
11367 let health_task = tokio::spawn(async move {
11368 health_handler
11369 .handle_control_frame(
11370 &health_client_ctx,
11371 supervisor_health_probe_frame(101, "aft"),
11372 )
11373 .await
11374 .unwrap()
11375 });
11376 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11377 .await
11378 .unwrap()
11379 .unwrap();
11380 assert_eq!(
11381 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11382 ModuleControlRequest::HealthCheck {}
11383 );
11384
11385 handler
11386 .handle_control_frame(
11387 &module_ctx,
11388 health_response(health_frame.header.corr, HealthStatus::Degraded),
11389 )
11390 .await
11391 .unwrap();
11392 let health_response = health_task.await.unwrap();
11393 assert_eq!(health_response.len(), 1);
11394 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11395 ClientControlResponse::SupervisorHealthProbe {
11396 module_id,
11397 status,
11398 detail,
11399 metrics,
11400 } => {
11401 assert_eq!(module_id, "aft");
11402 assert_eq!(status, HealthStatus::Degraded);
11403 assert_eq!(detail.as_deref(), Some("warming"));
11404 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
11405 }
11406 other => panic!("unexpected health response: {other:?}"),
11407 }
11408
11409 handler
11410 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11411 .await
11412 .unwrap();
11413 let route_response = route_task.await.unwrap();
11414 assert!(route_response.is_empty());
11415 let published = route_client_rx.recv().await.unwrap();
11416 assert!(matches!(
11417 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11418 ClientControlResponse::RouteOpen { .. }
11419 ));
11420 }
11421
11422 async fn relay_route_open(
11427 handler: &ControlHandler,
11428 client_connection: ConnectionId,
11429 client_egress: &FrameSink,
11430 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11431 corr: u64,
11432 module_id: &str,
11433 project_root_label: &str,
11434 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
11435 let ctx = RouteCtx {
11436 connection_id: client_connection,
11437 egress: client_egress.clone(),
11438 };
11439 let handler = handler.clone();
11440 let project_root = unique_project_root(project_root_label);
11441 let module_id = module_id.to_string();
11442 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
11443 let task = tokio::spawn(async move {
11444 let _guard = tracing::dispatcher::set_default(&dispatch);
11445 handler
11446 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
11447 .await
11448 .unwrap()
11449 });
11450 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
11451 .await
11452 .expect("module receives the relayed route.bind")
11453 .expect("module egress is open");
11454 (task, bind.frame)
11455 }
11456
11457 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
11458 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
11459 ModuleControlRequest::RouteBind {
11460 route_channel,
11461 epoch,
11462 ..
11463 } => (route_channel, epoch),
11464 other => panic!("expected a route.bind request, got {other:?}"),
11465 }
11466 }
11467
11468 fn published_route(frame: &Frame) -> (u16, u32) {
11469 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
11470 ClientControlResponse::RouteOpen {
11471 route_channel,
11472 route_epoch,
11473 ..
11474 } => (route_channel, route_epoch),
11475 other => panic!("expected a route.open response, got {other:?}"),
11476 }
11477 }
11478
11479 #[tokio::test]
11499 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
11500 let registry = Arc::new(Registry::default());
11501 let forwarding = Arc::new(ForwardingTable::default());
11502 let handler =
11503 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11504
11505 let module_connection = ConnectionId::new(30);
11506 let (module_ctx, mut module_rx) = route_ctx(module_connection);
11507 hello_via_sink(
11508 &handler,
11509 &module_ctx,
11510 &mut module_rx,
11511 hello_frame("aft", PROTOCOL_VERSION, 7),
11512 )
11513 .await;
11514
11515 let dying_client = ConnectionId::new(31);
11516 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
11517
11518 let (first_task, first_bind) = relay_route_open(
11521 &handler,
11522 dying_client,
11523 &dying_ctx.egress,
11524 &mut module_rx,
11525 100,
11526 "aft",
11527 "closing-first",
11528 )
11529 .await;
11530 handler
11531 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
11532 .await
11533 .unwrap();
11534 assert!(first_task.await.unwrap().is_empty());
11535 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
11536
11537 let (second_task, second_bind) = relay_route_open(
11539 &handler,
11540 dying_client,
11541 &dying_ctx.egress,
11542 &mut module_rx,
11543 101,
11544 "aft",
11545 "closing-second",
11546 )
11547 .await;
11548 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
11549
11550 assert!(forwarding
11553 .escalate_client_delivery_failure(
11554 dying_client,
11555 first_channel,
11556 first_epoch,
11557 CloseReason::new(
11558 "module_to_client_delivery_failed",
11559 "client egress refused a module frame",
11560 ),
11561 crate::forwarding::UndeliveredFrame {
11562 module_id: None,
11563 sink: &dying_ctx.egress,
11564 },
11565 )
11566 .unwrap());
11567 assert!(!dying_ctx.egress.is_closed());
11568
11569 let ack = handler
11571 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
11572 .await;
11573 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11574 if module_loop_error.is_some() {
11575 handler.cleanup_connection(module_connection).unwrap();
11579 }
11580 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11584 .await
11585 .ok()
11586 .flatten();
11587
11588 assert!(
11590 registry
11591 .get_module_by_connection(module_connection)
11592 .unwrap()
11593 .is_some(),
11594 "one client's closing connection ended the shared module connection: \
11595 {module_loop_error:?}"
11596 );
11597 let cotenant = ConnectionId::new(32);
11599 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11600 let (cotenant_task, cotenant_bind) = relay_route_open(
11601 &handler,
11602 cotenant,
11603 &cotenant_ctx.egress,
11604 &mut module_rx,
11605 102,
11606 "aft",
11607 "closing-cotenant",
11608 )
11609 .await;
11610 handler
11611 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
11612 .await
11613 .unwrap();
11614 assert!(cotenant_task.await.unwrap().is_empty());
11615 let (cotenant_channel, cotenant_epoch) =
11616 published_route(&cotenant_rx.recv().await.unwrap());
11617 assert!(matches!(
11618 forwarding
11619 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11620 .unwrap(),
11621 DataRoute::Client(DataRouteState::Bound(_))
11622 ));
11623
11624 let goodbye = post_ack_module_frame
11627 .expect("module receives a GOODBYE for the abandoned route channel");
11628 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11629 assert_eq!(goodbye.header.channel, abandoned_channel);
11630 assert_eq!(goodbye.header.epoch, abandoned_epoch);
11631
11632 assert!(dying_rx.try_recv().is_err());
11636 let second_response = second_task.await.unwrap();
11637 assert_eq!(second_response.len(), 1);
11638 assert_eq!(
11639 parse_error(&second_response[0])["code"],
11640 "target_unavailable"
11641 );
11642 }
11643
11644 async fn catalog_snapshot(
11647 handler: &ControlHandler,
11648 client_ctx: &RouteCtx,
11649 corr: u64,
11650 ) -> (u64, Vec<(String, bool)>) {
11651 let body =
11652 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
11653 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
11654 let reply = handler
11655 .handle_control_frame(client_ctx, request)
11656 .await
11657 .unwrap()
11658 .pop()
11659 .unwrap();
11660 let ClientControlResponse::CatalogList {
11661 generation,
11662 modules,
11663 ..
11664 } = serde_json::from_slice(&reply.body).unwrap()
11665 else {
11666 panic!("not a catalog.list reply");
11667 };
11668 (
11669 generation,
11670 modules
11671 .into_iter()
11672 .map(|entry| (entry.module_id, entry.ready))
11673 .collect(),
11674 )
11675 }
11676
11677 fn hello_held_before_forwarding() -> (
11683 ControlHandler,
11684 Arc<std::sync::Barrier>,
11685 std::thread::JoinHandle<Vec<Frame>>,
11686 RouteCtx,
11687 mpsc::Receiver<crate::router::OutboundFrame>,
11688 ) {
11689 let hold = Arc::new(std::sync::Barrier::new(2));
11690 let handler = ControlHandler::with_forwarding(
11691 Arc::new(Registry::default()),
11692 Arc::new(ForwardingTable::default()),
11693 )
11694 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
11695 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11696 let hello = {
11697 let handler = handler.clone();
11698 let module_ctx = module_ctx.clone();
11699 std::thread::spawn(move || {
11700 handler
11701 .handle_hello(
11702 module_ctx.connection_id,
11703 Some(module_ctx.egress.clone()),
11704 hello_frame("aft", PROTOCOL_VERSION, 7),
11705 )
11706 .unwrap()
11707 })
11708 };
11709 (handler, hold, hello, module_ctx, module_rx)
11710 }
11711
11712 #[tokio::test]
11721 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
11722 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
11723 let client = ConnectionId::new(41);
11724 let (client_ctx, mut client_rx) = route_ctx(client);
11725
11726 hold.wait();
11727 assert!(handler.registry.get_module("aft").unwrap().is_some());
11729 assert!(!handler
11730 .forwarding
11731 .has_live_module_connection("aft")
11732 .unwrap());
11733 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
11734 assert!(
11735 listed.is_empty(),
11736 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
11737 );
11738 let refused = handler
11739 .handle_control_frame(
11740 &client_ctx,
11741 route_open_frame(301, "aft", unique_project_root("listed-window")),
11742 )
11743 .await
11744 .unwrap();
11745 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
11746 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
11747 let duplicate = handler
11748 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
11749 .await
11750 .unwrap();
11751 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11752
11753 hold.wait();
11754 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11755 let ack = module_rx.try_recv().unwrap().frame;
11756 assert_eq!(ack.header.ty, FrameType::HelloAck);
11757 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11758 assert_eq!(listed, vec![("aft".to_string(), true)]);
11759
11760 let (open_task, bind) = relay_route_open(
11761 &handler,
11762 client,
11763 &client_ctx.egress,
11764 &mut module_rx,
11765 303,
11766 "aft",
11767 "listed-after",
11768 )
11769 .await;
11770 assert!(handler
11771 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11772 .await
11773 .unwrap()
11774 .is_empty());
11775 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
11776 published_route(&client_rx.try_recv().unwrap().frame);
11777 }
11778
11779 #[tokio::test]
11786 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
11787 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
11788 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11789 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11790
11791 hold.wait();
11792 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
11793 assert!(listed.is_empty());
11794 assert_eq!(
11795 during, before,
11796 "the generation moved while the module was not listed"
11797 );
11798
11799 hold.wait();
11800 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11801 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11802 assert_eq!(listed, vec![("aft".to_string(), true)]);
11803 assert_eq!(
11804 after,
11805 before + 1,
11806 "the module appeared without exactly one generation bump"
11807 );
11808 }
11809
11810 #[tokio::test]
11815 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
11816 let registry = Arc::new(Registry::default());
11817 let handler = ControlHandler::with_forwarding(
11818 Arc::clone(®istry),
11819 Arc::new(ForwardingTable::default()),
11820 );
11821 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11822 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11823 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11824 drop(module_rx);
11825
11826 let replies = handler
11827 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
11828 .await
11829 .unwrap();
11830 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
11831 assert!(registry.get_module("aft").unwrap().is_none());
11832 assert!(registry
11833 .get_module_by_connection(module_ctx.connection_id)
11834 .unwrap()
11835 .is_none());
11836 assert_eq!(
11837 catalog_snapshot(&handler, &client_ctx, 301).await,
11838 (before, Vec::new())
11839 );
11840 }
11841
11842 async fn route_open_held_after_bind_send() -> (
11846 ControlHandler,
11847 Arc<tokio::sync::Barrier>,
11848 RouteCtx,
11849 mpsc::Receiver<crate::router::OutboundFrame>,
11850 ) {
11851 let hold = Arc::new(tokio::sync::Barrier::new(2));
11852 let handler = ControlHandler::with_forwarding(
11853 Arc::new(Registry::default()),
11854 Arc::new(ForwardingTable::default()),
11855 )
11856 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
11857 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11858 hello_via_sink(
11859 &handler,
11860 &module_ctx,
11861 &mut module_rx,
11862 hello_frame("aft", PROTOCOL_VERSION, 7),
11863 )
11864 .await;
11865 (handler, hold, module_ctx, module_rx)
11866 }
11867
11868 #[tokio::test]
11875 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
11876 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11877 let client = ConnectionId::new(41);
11878 let (client_ctx, mut client_rx) = route_ctx(client);
11879
11880 let (open_task, bind) = relay_route_open(
11881 &handler,
11882 client,
11883 &client_ctx.egress,
11884 &mut module_rx,
11885 200,
11886 "aft",
11887 "acked-before-mark",
11888 )
11889 .await;
11890 let (module_channel, module_epoch) = route_bind_channel(&bind);
11891 hold.wait().await;
11893
11894 assert!(handler
11896 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11897 .await
11898 .unwrap()
11899 .is_empty());
11900 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
11901
11902 hold.wait().await;
11903 assert!(
11904 open_task.await.unwrap().is_empty(),
11905 "route.open answers accepted"
11906 );
11907
11908 match module_rx.try_recv() {
11911 Err(mpsc::error::TryRecvError::Empty) => {}
11912 Ok(outbound) => panic!(
11913 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
11914 outbound.frame.header.ty,
11915 outbound.frame.header.channel,
11916 outbound.frame.header.epoch
11917 ),
11918 Err(err) => panic!("module egress closed: {err}"),
11919 }
11920
11921 let router = Router::with_control_handler(Arc::new(handler.clone()));
11924 let request = Frame::build(
11925 FrameType::Request,
11926 Flags::new(true, Priority::Interactive, false),
11927 client_channel,
11928 client_epoch,
11929 201,
11930 b"first request".to_vec(),
11931 )
11932 .unwrap();
11933 router
11934 .route_for_connection(&client_ctx, request)
11935 .await
11936 .unwrap();
11937 let delivered = module_rx.try_recv().unwrap().frame;
11938 assert_eq!(delivered.header.ty, FrameType::Request);
11939 assert_eq!(
11940 (delivered.header.channel, delivered.header.epoch),
11941 (module_channel, module_epoch)
11942 );
11943 assert_eq!(delivered.body, b"first request");
11944 }
11945
11946 #[tokio::test]
11952 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
11953 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11954 let client = ConnectionId::new(42);
11955 let (client_ctx, _client_rx) = route_ctx(client);
11956
11957 let (open_task, bind) = relay_route_open(
11958 &handler,
11959 client,
11960 &client_ctx.egress,
11961 &mut module_rx,
11962 300,
11963 "aft",
11964 "released-before-mark",
11965 )
11966 .await;
11967 let (module_channel, module_epoch) = route_bind_channel(&bind);
11968 hold.wait().await;
11969
11970 handler.cleanup_connection(client).unwrap();
11971 assert!(
11972 module_rx.try_recv().is_err(),
11973 "client teardown must not GOODBYE a bind route.open is still queueing"
11974 );
11975
11976 hold.wait().await;
11977 let response = open_task.await.unwrap();
11978 assert_eq!(response.len(), 1);
11979 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11980
11981 let goodbye = module_rx
11982 .try_recv()
11983 .expect("module is told to drop the binding the daemon abandoned")
11984 .frame;
11985 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11986 assert_eq!(
11987 (goodbye.header.channel, goodbye.header.epoch),
11988 (module_channel, module_epoch)
11989 );
11990 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11991 }
11992
11993 #[tokio::test]
11998 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
11999 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12000 let client = ConnectionId::new(43);
12001 let (client_ctx, client_rx) = route_ctx(client);
12002
12003 let (open_task, bind) = relay_route_open(
12004 &handler,
12005 client,
12006 &client_ctx.egress,
12007 &mut module_rx,
12008 400,
12009 "aft",
12010 "gone-client-before-mark",
12011 )
12012 .await;
12013 let (module_channel, module_epoch) = route_bind_channel(&bind);
12014 hold.wait().await;
12015
12016 drop(client_rx);
12017 assert!(client_ctx.egress.is_closed());
12018 handler
12019 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
12020 .await
12021 .unwrap();
12022
12023 hold.wait().await;
12024 let response = open_task.await.unwrap();
12025 assert_eq!(response.len(), 1);
12026 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12027
12028 let goodbye = module_rx
12029 .try_recv()
12030 .expect("module is told to drop the binding it acked")
12031 .frame;
12032 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12033 assert_eq!(
12034 (goodbye.header.channel, goodbye.header.epoch),
12035 (module_channel, module_epoch)
12036 );
12037 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
12038 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
12039 }
12040
12041 #[test]
12048 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
12049 let handler = ControlHandler::default();
12050 let module_connection = ConnectionId::new(30);
12051 let client_connection = ConnectionId::new(31);
12052
12053 handler
12054 .refuse_to_end_module_connection_for_a_client(
12055 module_connection,
12056 77,
12057 ForwardingError::ConnectionClosing {
12058 connection_id: client_connection,
12059 },
12060 )
12061 .expect("a closing client must never end the module connection");
12062
12063 assert!(matches!(
12064 handler.refuse_to_end_module_connection_for_a_client(
12065 module_connection,
12066 78,
12067 ForwardingError::ConnectionClosing {
12068 connection_id: module_connection,
12069 },
12070 ),
12071 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
12072 connection_id
12073 })) if connection_id == module_connection
12074 ));
12075 assert!(matches!(
12076 handler.refuse_to_end_module_connection_for_a_client(
12077 module_connection,
12078 79,
12079 ForwardingError::Poisoned,
12080 ),
12081 Err(RouterError::Forwarding(ForwardingError::Poisoned))
12082 ));
12083 assert!(matches!(
12084 handler.refuse_to_end_module_connection_for_a_client(
12085 module_connection,
12086 80,
12087 ForwardingError::StaleModuleEndpoint,
12088 ),
12089 Err(RouterError::Forwarding(
12090 ForwardingError::StaleModuleEndpoint
12091 ))
12092 ));
12093 }
12094
12095 #[tokio::test]
12108 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
12109 let registry = Arc::new(Registry::default());
12110 let forwarding = Arc::new(ForwardingTable::default());
12111 let supervisor = SupervisorHandle::new();
12112 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12113 let handler =
12114 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12115 .with_supervisor(supervisor);
12116
12117 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12118 hello_via_sink(
12119 &handler,
12120 &target_ctx,
12121 &mut target_rx,
12122 hello_frame("target", PROTOCOL_VERSION, 1),
12123 )
12124 .await;
12125
12126 let wrong_nonce = handler
12130 .handle_control_frame(
12131 &route_ctx(ConnectionId::new(91)).0,
12132 route_open_frame_with_admission_facts(
12133 20,
12134 "target",
12135 unique_project_root("admission-facts"),
12136 Some(subc_control::ConsumerIdentity {
12137 module_id: "fed".to_string(),
12138 launch_nonce: "not-the-real-nonce".to_string(),
12139 }),
12140 None,
12141 ),
12142 )
12143 .await
12144 .unwrap();
12145 assert_eq!(
12146 parse_error(&wrong_nonce[0])["code"],
12147 "bad_consumer_identity",
12148 "a mismatched launch nonce must be refused, not stamped Reserved"
12149 );
12150
12151 let never_spawned = handler
12155 .handle_control_frame(
12156 &route_ctx(ConnectionId::new(92)).0,
12157 route_open_frame_with_admission_facts(
12158 21,
12159 "target",
12160 unique_project_root("admission-facts"),
12161 Some(subc_control::ConsumerIdentity {
12162 module_id: "never-spawned".to_string(),
12163 launch_nonce: "any-nonce".to_string(),
12164 }),
12165 None,
12166 ),
12167 )
12168 .await
12169 .unwrap();
12170 assert_eq!(
12171 parse_error(&never_spawned[0])["code"],
12172 "bad_consumer_identity",
12173 "an unspawned module_id must be refused rather than accepted for lack of a record"
12174 );
12175 }
12176
12177 #[tokio::test]
12200 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
12201 let registry = Arc::new(Registry::default());
12202 let forwarding = Arc::new(ForwardingTable::default());
12203 let supervisor = SupervisorHandle::new();
12204 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12205 let handler =
12206 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12207 .with_supervisor(supervisor);
12208
12209 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
12210 hello_via_sink(
12211 &handler,
12212 &target_ctx,
12213 &mut target_rx,
12214 hello_frame("target", PROTOCOL_VERSION, 1),
12215 )
12216 .await;
12217
12218 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
12219 let route_handler = handler.clone();
12220 let route_task = tokio::spawn(async move {
12221 route_handler
12222 .handle_control_frame(
12223 &client_ctx,
12224 route_open_frame_with_admission_facts(
12225 30,
12226 "target",
12227 unique_project_root("admission-facts"),
12228 Some(subc_control::ConsumerIdentity {
12229 module_id: "fed".to_string(),
12230 launch_nonce: "fed-nonce".to_string(),
12231 }),
12232 None,
12233 ),
12234 )
12235 .await
12236 .unwrap()
12237 });
12238
12239 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12246 .await
12247 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
12248 .expect("module control channel closed before route.bind");
12249 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12250 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
12251 panic!("expected route.bind")
12252 };
12253 assert_eq!(
12254 principal,
12255 Some(Principal::Reserved {
12256 module_id: "fed".to_string()
12257 }),
12258 "a correctly attested consumer must be stamped Reserved for its own id"
12259 );
12260
12261 handler
12262 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12263 .await
12264 .unwrap();
12265 assert!(route_task.await.unwrap().is_empty());
12266 assert!(
12267 matches!(
12268 serde_json::from_slice::<ClientControlResponse>(
12269 &client_rx.recv().await.unwrap().body
12270 )
12271 .unwrap(),
12272 ClientControlResponse::RouteOpen { .. }
12273 ),
12274 "the route must actually open, not merely avoid an error"
12275 );
12276 }
12277
12278 #[tokio::test(start_paused = true)]
12279 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
12280 let registry = Arc::new(Registry::default());
12281 let forwarding = Arc::new(ForwardingTable::default());
12282 let supervisor = SupervisorHandle::new();
12283 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12284 let handler =
12285 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12286 .with_supervisor(supervisor);
12287
12288 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
12289 hello_via_sink(
12290 &handler,
12291 &target_ctx,
12292 &mut target_rx,
12293 hello_frame("target", PROTOCOL_VERSION, 1),
12294 )
12295 .await;
12296
12297 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
12298 let direct_handler = handler.clone();
12299 let direct_open = tokio::spawn(async move {
12300 direct_handler
12301 .handle_control_frame(
12302 &direct_ctx,
12303 route_open_frame(2, "target", unique_project_root("route-census-direct")),
12304 )
12305 .await
12306 .unwrap()
12307 });
12308 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12309 .await
12310 .expect("no direct route.bind within 5s")
12311 .expect("target control channel closed before direct route.bind");
12312 handler
12313 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
12314 .await
12315 .unwrap();
12316 assert!(direct_open.await.unwrap().is_empty());
12317 let _ = direct_rx.recv().await.unwrap();
12318
12319 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
12320 let reserved_handler = handler.clone();
12321 let reserved_open = tokio::spawn(async move {
12322 reserved_handler
12323 .handle_control_frame(
12324 &reserved_ctx,
12325 route_open_frame_with_admission_facts(
12326 3,
12327 "target",
12328 unique_project_root("admission-facts"),
12329 Some(ConsumerIdentity {
12330 module_id: "fed".to_string(),
12331 launch_nonce: "fed-nonce".to_string(),
12332 }),
12333 None,
12334 ),
12335 )
12336 .await
12337 .unwrap()
12338 });
12339 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12340 .await
12341 .expect("no reserved route.bind within 5s")
12342 .expect("target control channel closed before reserved route.bind");
12343 handler
12344 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
12345 .await
12346 .unwrap();
12347 assert!(reserved_open.await.unwrap().is_empty());
12348 let _ = reserved_rx.recv().await.unwrap();
12349
12350 forwarding
12351 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
12352 .unwrap();
12353 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
12354 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
12355 module_id: Some("target".to_string()),
12356 })
12357 .unwrap();
12358 let census_frame =
12359 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
12360 let response = handler
12361 .handle_control_frame(&census_ctx, census_frame)
12362 .await
12363 .unwrap()
12364 .pop()
12365 .unwrap();
12366 let actual: Value = serde_json::from_slice(&response.body).unwrap();
12367 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
12368 assert!(matches!(
12369 decoded,
12370 ClientControlResponse::SupervisorRoutes { .. }
12371 ));
12372 let routes = actual["modules"][0]["routes"].as_array().unwrap();
12373 assert_eq!(routes.len(), 2);
12374 assert!(routes.iter().all(|route| route["draining"] == true));
12375 assert!(
12378 routes.iter().all(|route| route["drain_reason"] == "reload"),
12379 "draining routes must name the drain's reason: {routes:?}"
12380 );
12381 assert!(routes.iter().any(|route| {
12382 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
12383 }));
12384 assert!(routes.iter().any(|route| {
12385 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
12386 }));
12387
12388 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12389 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
12390 if std::env::var_os("UPDATE_GOLDEN").is_some() {
12391 std::fs::write(
12392 &golden_path,
12393 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
12394 )
12395 .unwrap();
12396 }
12397 let expected: Value =
12398 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
12399 assert_eq!(actual, expected);
12400 }
12401
12402 async fn query_live_roots(
12403 handler: &ControlHandler,
12404 module_ctx: &RouteCtx,
12405 ) -> ModuleControlResponseToModule {
12406 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
12407 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
12408 let response = handler
12409 .handle_control_frame(module_ctx, frame)
12410 .await
12411 .unwrap()
12412 .pop()
12413 .unwrap();
12414 serde_json::from_slice(&response.body).unwrap()
12415 }
12416
12417 #[tokio::test(start_paused = true)]
12418 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
12419 let registry = Arc::new(Registry::default());
12420 let forwarding = Arc::new(ForwardingTable::default());
12421 let handler = ControlHandler::with_forwarding(registry, forwarding);
12422 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
12423 hello_via_sink(
12424 &handler,
12425 &target_ctx,
12426 &mut target_rx,
12427 hello_frame("target", PROTOCOL_VERSION, 1),
12428 )
12429 .await;
12430 let root = unique_project_root("live-roots-known");
12431 let path = ProjectRootId::from_path_allowing_missing(root.path())
12432 .unwrap()
12433 .as_path()
12434 .to_path_buf();
12435 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
12436 let open_handler = handler.clone();
12437 let opened = tokio::spawn(async move {
12438 open_handler
12439 .handle_control_frame(&client_ctx, legacy_route_open_frame(2, "target", root))
12440 .await
12441 .unwrap()
12442 });
12443 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12444 .await
12445 .unwrap()
12446 .unwrap();
12447 handler
12448 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
12449 .await
12450 .unwrap();
12451 assert!(opened.await.unwrap().is_empty());
12452 let _ = client_rx.recv().await.unwrap();
12453
12454 let root = unique_project_root("live-roots-pending");
12455 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
12456 .unwrap()
12457 .as_path()
12458 .to_path_buf();
12459 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
12460 let open_handler = handler.clone();
12461 let pending = tokio::spawn(async move {
12462 open_handler
12463 .handle_control_frame(&client_ctx, legacy_route_open_frame(3, "target", root))
12464 .await
12465 .unwrap()
12466 });
12467 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12468 .await
12469 .unwrap()
12470 .unwrap();
12471 let actual = query_live_roots(&handler, &target_ctx).await;
12472 let ModuleControlResponseToModule::LiveRoots {
12473 roots,
12474 unknown_root_bindings,
12475 total_bindings,
12476 } = actual
12477 else {
12478 panic!("expected live roots")
12479 };
12480 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
12481 assert_eq!(unknown_root_bindings, 0);
12482 assert_eq!(
12483 roots.len(),
12484 2,
12485 "root-known arm must retain each canonical root"
12486 );
12487 assert_eq!(
12488 total_bindings,
12489 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12490 );
12491 let counts = roots
12492 .iter()
12493 .map(|root| (root.project_root.clone(), root.bound, root.pending))
12494 .collect::<Vec<_>>();
12495 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
12496 expected.sort_by(|a, b| a.0.cmp(&b.0));
12497 assert_eq!(
12498 counts, expected,
12499 "roots must sort by path and count pending separately"
12500 );
12501 handler
12502 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
12503 .await
12504 .unwrap();
12505 assert!(pending.await.unwrap().is_empty());
12506 }
12507
12508 #[tokio::test(start_paused = true)]
12509 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
12510 let forwarding = Arc::new(ForwardingTable::default());
12511 let handler =
12512 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12513 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
12514 hello_via_sink(
12515 &handler,
12516 &target_ctx,
12517 &mut target_rx,
12518 hello_frame("target", PROTOCOL_VERSION, 1),
12519 )
12520 .await;
12521 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
12522 let pending = forwarding
12523 .begin_route_bind_relay_for_test(
12524 client_ctx.connection_id,
12525 client_ctx.egress.clone(),
12526 2,
12527 "target",
12528 )
12529 .unwrap();
12530 forwarding
12531 .complete_pending_relay(
12532 target_ctx.connection_id,
12533 pending.corr,
12534 RouteBindRelayOutcome::Accepted,
12535 )
12536 .unwrap();
12537 let actual = query_live_roots(&handler, &target_ctx).await;
12538 let ModuleControlResponseToModule::LiveRoots {
12539 roots,
12540 unknown_root_bindings,
12541 total_bindings,
12542 } = actual
12543 else {
12544 panic!("expected live roots")
12545 };
12546 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
12547 assert_eq!(
12548 unknown_root_bindings, 1,
12549 "unknown-root arm must not read as no bindings"
12550 );
12551 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
12552 assert_eq!(
12553 total_bindings,
12554 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12555 );
12556 }
12557
12558 #[tokio::test(start_paused = true)]
12566 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
12567 let forwarding = Arc::new(ForwardingTable::default());
12568 let handler =
12569 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12570 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
12571 let replies = handler
12572 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
12573 .await
12574 .unwrap();
12575 let queued_by_hello = module_rx.len();
12576
12577 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
12578 let open_handler = handler.clone();
12579 let open = tokio::spawn(async move {
12580 open_handler
12581 .handle_control_frame(
12582 &client_ctx,
12583 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
12584 )
12585 .await
12586 .unwrap()
12587 });
12588 let mut spins = 0;
12590 while module_rx.len() == queued_by_hello {
12591 spins += 1;
12592 assert!(spins < 10_000, "route.open never queued a route.bind");
12593 tokio::task::yield_now().await;
12594 }
12595
12596 for reply in replies {
12598 module_ctx.egress.send(reply).await.unwrap();
12599 }
12600
12601 let first = module_rx.recv().await.unwrap().frame;
12602 assert_eq!(
12603 first.header.ty,
12604 FrameType::HelloAck,
12605 "the first frame a registering module reads must be its HELLO_ACK"
12606 );
12607 assert_eq!(first.header.corr, 7);
12608 let second = module_rx.recv().await.unwrap().frame;
12609 assert_eq!(second.header.ty, FrameType::Request);
12610 assert!(
12611 matches!(
12612 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
12613 ModuleControlRequest::RouteBind { .. }
12614 ),
12615 "the route.bind follows the ack"
12616 );
12617 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
12618
12619 handler
12620 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
12621 .await
12622 .unwrap();
12623 assert!(open.await.unwrap().is_empty());
12624 let _ = client_rx.recv().await.unwrap();
12625 }
12626
12627 #[tokio::test(start_paused = true)]
12628 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
12629 let handler = ControlHandler::with_forwarding(
12630 Arc::new(Registry::default()),
12631 Arc::new(ForwardingTable::default()),
12632 );
12633 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
12634 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
12635 hello_via_sink(
12636 &handler,
12637 &first_ctx,
12638 &mut first_rx,
12639 hello_frame("first", PROTOCOL_VERSION, 1),
12640 )
12641 .await;
12642 hello_via_sink(
12643 &handler,
12644 &second_ctx,
12645 &mut second_rx,
12646 hello_frame("second", PROTOCOL_VERSION, 2),
12647 )
12648 .await;
12649 let root = unique_project_root("second-only");
12650 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
12651 let cloned = handler.clone();
12652 let open = tokio::spawn(async move {
12653 cloned
12654 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
12655 .await
12656 .unwrap()
12657 });
12658 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
12659 .await
12660 .unwrap()
12661 .unwrap();
12662 let first = query_live_roots(&handler, &first_ctx).await;
12663 let second = query_live_roots(&handler, &second_ctx).await;
12664 assert!(
12665 matches!(
12666 first,
12667 ModuleControlResponseToModule::LiveRoots {
12668 total_bindings: 0,
12669 ..
12670 }
12671 ),
12672 "cross-module scope must not expose another module's roots"
12673 );
12674 assert!(
12675 matches!(
12676 second,
12677 ModuleControlResponseToModule::LiveRoots {
12678 total_bindings: 1,
12679 ..
12680 }
12681 ),
12682 "second module must see its pending route"
12683 );
12684 handler
12685 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
12686 .await
12687 .unwrap();
12688 assert!(open.await.unwrap().is_empty());
12689 }
12690
12691 #[tokio::test(start_paused = true)]
12692 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
12693 let handler = ControlHandler::with_forwarding(
12694 Arc::new(Registry::default()),
12695 Arc::new(ForwardingTable::default()),
12696 );
12697 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
12698 hello_via_sink(
12699 &handler,
12700 &target_ctx,
12701 &mut target_rx,
12702 hello_frame("target", PROTOCOL_VERSION, 1),
12703 )
12704 .await;
12705 let actual = query_live_roots(&handler, &target_ctx).await;
12706 let ModuleControlResponseToModule::LiveRoots {
12707 roots,
12708 unknown_root_bindings,
12709 total_bindings,
12710 } = actual
12711 else {
12712 panic!("expected live roots")
12713 };
12714 assert!(roots.is_empty());
12715 assert_eq!(unknown_root_bindings, 0);
12716 assert_eq!(total_bindings, 0);
12717 assert_eq!(
12718 total_bindings,
12719 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12720 );
12721 }
12722
12723 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
12730 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12731 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
12732 let text = std::fs::read_to_string(&path)
12733 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
12734 let vectors: Vec<(String, Value)> = text
12735 .lines()
12736 .filter(|line| !line.trim().is_empty())
12737 .map(|line| {
12738 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
12739 let id = entry["corpus_id"]
12740 .as_str()
12741 .expect("every vector carries a corpus_id")
12742 .to_string();
12743 (id, entry["package"].clone())
12744 })
12745 .collect();
12746 assert_eq!(
12750 vectors.len(),
12751 3,
12752 "vendored fed corpus changed size; re-sync from subc-federation"
12753 );
12754
12755 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
12770 let richest = vectors
12771 .iter()
12772 .filter_map(|(_, package)| package.as_object())
12773 .map(|object| {
12774 object
12775 .keys()
12776 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
12777 .count()
12778 })
12779 .max()
12780 .unwrap_or(0);
12781 assert!(
12782 richest >= 2,
12783 "vendored corpus no longer carries a package with unmodelled fields, \
12784 so the relay test can no longer distinguish a verbatim relay from a lossy one"
12785 );
12786
12787 vectors
12788 }
12789
12790 #[tokio::test]
12798 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
12799 for (corpus_id, package) in fed_admission_facts_vectors() {
12800 let registry = Arc::new(Registry::default());
12801 let forwarding = Arc::new(ForwardingTable::default());
12802 let supervisor = SupervisorHandle::new();
12803 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12804 let handler =
12805 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12806 .with_supervisor(supervisor)
12807 .with_admission_facts_config(
12808 Some("fed".to_string()),
12809 Some(vec!["target".to_string()]),
12810 );
12811
12812 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12813 hello_via_sink(
12814 &handler,
12815 &target_ctx,
12816 &mut target_rx,
12817 hello_frame("target", PROTOCOL_VERSION, 1),
12818 )
12819 .await;
12820
12821 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
12822 let route_handler = handler.clone();
12823 let expected = package.clone();
12824 let route_task = tokio::spawn(async move {
12825 route_handler
12826 .handle_control_frame(
12827 &client_ctx,
12828 route_open_frame_with_admission_facts(
12829 20,
12830 "target",
12831 unique_project_root("admission-facts"),
12832 Some(subc_control::ConsumerIdentity {
12833 module_id: "fed".to_string(),
12834 launch_nonce: "fed-nonce".to_string(),
12835 }),
12836 Some(package),
12837 ),
12838 )
12839 .await
12840 .unwrap()
12841 });
12842
12843 let bind_frame = target_rx.recv().await.unwrap();
12844 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12845 let ModuleControlRequest::RouteBind {
12846 admission_facts, ..
12847 } = bind
12848 else {
12849 panic!("{corpus_id}: expected route.bind")
12850 };
12851 assert_eq!(
12852 admission_facts,
12853 Some(expected),
12854 "{corpus_id}: relay must not add, drop or reshape any field"
12855 );
12856
12857 handler
12858 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12859 .await
12860 .unwrap();
12861 route_task.await.unwrap();
12862 }
12863 }
12864
12865 #[tokio::test]
12866 async fn route_bind_never_relays_the_openers_build_class() {
12867 let registry = Arc::new(Registry::default());
12868 let forwarding = Arc::new(ForwardingTable::default());
12869 let handler =
12870 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12871 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
12872 hello_via_sink(
12873 &handler,
12874 &target_ctx,
12875 &mut target_rx,
12876 hello_frame("target", PROTOCOL_VERSION, 1),
12877 )
12878 .await;
12879
12880 let mut identity = BindIdentity::new("unit".to_string(), "s".to_string());
12881 identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
12882 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12883 busy_refusals: None,
12884 target: RouteTarget::ToolProvider {
12885 module_id: "target".to_string(),
12886 },
12887 identity,
12888 consumer_identity: None,
12889 consumer_capabilities: None,
12890 role_versions: None,
12891 admission_facts: None,
12892 scope: None,
12893 })
12894 .unwrap();
12895 let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
12896
12897 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
12898 let route_handler = handler.clone();
12899 let route_task = tokio::spawn(async move {
12900 route_handler
12901 .handle_control_frame(&client_ctx, open)
12902 .await
12903 .unwrap()
12904 });
12905 let bind_frame = target_rx.recv().await.unwrap();
12906 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12907 let ModuleControlRequest::RouteBind { identity, .. } = bind else {
12908 panic!("expected route.bind")
12909 };
12910 assert_eq!(
12911 identity.build_class, None,
12912 "route.bind carried the build class"
12913 );
12914 assert!(
12915 !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
12916 "the route.bind bytes still name build_class"
12917 );
12918 handler
12919 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12920 .await
12921 .unwrap();
12922 route_task.await.unwrap();
12923 }
12924
12925 #[tokio::test]
12926 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
12927 let registry = Arc::new(Registry::default());
12928 let forwarding = Arc::new(ForwardingTable::default());
12929 let supervisor = SupervisorHandle::new();
12930 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12931 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
12932 let handler =
12933 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12934 .with_supervisor(supervisor)
12935 .with_admission_facts_config(
12936 Some("fed".to_string()),
12937 Some(vec!["target".to_string()]),
12938 );
12939
12940 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
12941 hello_via_sink(
12942 &handler,
12943 &target_ctx,
12944 &mut target_rx,
12945 hello_frame("target", PROTOCOL_VERSION, 1),
12946 )
12947 .await;
12948 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
12949 hello_via_sink(
12950 &handler,
12951 &other_ctx,
12952 &mut other_rx,
12953 hello_frame("other", PROTOCOL_VERSION, 2),
12954 )
12955 .await;
12956
12957 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
12958 let expected_facts = facts.clone();
12959 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
12960 let route_handler = handler.clone();
12961 let route_task = tokio::spawn(async move {
12962 route_handler
12963 .handle_control_frame(
12964 &client_ctx,
12965 route_open_frame_with_admission_facts(
12966 10,
12967 "target",
12968 unique_project_root("admission-facts"),
12969 Some(subc_control::ConsumerIdentity {
12970 module_id: "fed".to_string(),
12971 launch_nonce: "fed-nonce".to_string(),
12972 }),
12973 Some(facts.clone()),
12974 ),
12975 )
12976 .await
12977 .unwrap()
12978 });
12979 let bind_frame = target_rx.recv().await.unwrap();
12980 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12981 let ModuleControlRequest::RouteBind {
12982 admission_facts, ..
12983 } = bind
12984 else {
12985 panic!("expected route.bind")
12986 };
12987 assert_eq!(admission_facts, Some(expected_facts));
12988 handler
12989 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12990 .await
12991 .unwrap();
12992 assert!(route_task.await.unwrap().is_empty());
12993 assert!(matches!(
12994 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
12995 .unwrap(),
12996 ClientControlResponse::RouteOpen { .. }
12997 ));
12998
12999 let direct = handler
13000 .handle_control_frame(
13001 &route_ctx(ConnectionId::new(73)).0,
13002 route_open_frame_with_admission_facts(
13003 11,
13004 "target",
13005 unique_project_root("admission-facts"),
13006 None,
13007 Some(json!({"x": 1})),
13008 ),
13009 )
13010 .await
13011 .unwrap();
13012 assert_eq!(
13013 parse_error(&direct[0])["code"],
13014 "admission_facts_not_permitted"
13015 );
13016
13017 let different_reserved = handler
13018 .handle_control_frame(
13019 &route_ctx(ConnectionId::new(77)).0,
13020 route_open_frame_with_admission_facts(
13021 15,
13022 "target",
13023 unique_project_root("admission-facts"),
13024 Some(subc_control::ConsumerIdentity {
13025 module_id: "other".to_string(),
13026 launch_nonce: "other-nonce".to_string(),
13027 }),
13028 Some(json!({"x": 1})),
13029 ),
13030 )
13031 .await
13032 .unwrap();
13033 assert_eq!(
13034 parse_error(&different_reserved[0])["code"],
13035 "admission_facts_not_permitted"
13036 );
13037
13038 let other_target = handler
13039 .handle_control_frame(
13040 &route_ctx(ConnectionId::new(74)).0,
13041 route_open_frame_with_admission_facts(
13042 12,
13043 "other",
13044 unique_project_root("admission-facts"),
13045 Some(subc_control::ConsumerIdentity {
13046 module_id: "fed".to_string(),
13047 launch_nonce: "fed-nonce".to_string(),
13048 }),
13049 Some(json!({"x": 1})),
13050 ),
13051 )
13052 .await
13053 .unwrap();
13054 assert_eq!(
13055 parse_error(&other_target[0])["code"],
13056 "admission_facts_target_not_allowed"
13057 );
13058
13059 let nonexistent = handler
13060 .handle_control_frame(
13061 &route_ctx(ConnectionId::new(75)).0,
13062 route_open_frame_with_admission_facts(
13063 13,
13064 "missing",
13065 unique_project_root("admission-facts"),
13066 None,
13067 Some(json!({"x": 1})),
13068 ),
13069 )
13070 .await
13071 .unwrap();
13072 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
13073
13074 let described = handler
13075 .handle_control_frame(
13076 &route_ctx(ConnectionId::new(76)).0,
13077 Frame::build(
13078 FrameType::Request,
13079 control_flags(),
13080 0,
13081 0,
13082 14,
13083 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13084 )
13085 .unwrap(),
13086 )
13087 .await
13088 .unwrap();
13089 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13090 serde_json::from_slice(&described[0].body).unwrap()
13091 else {
13092 panic!("expected server.describe response")
13093 };
13094 assert!(capabilities
13095 .iter()
13096 .any(|cap| cap == "admission_facts_relay_v1"));
13097 }
13098
13099 #[tokio::test]
13100 async fn admission_facts_without_configured_carrier_are_rejected() {
13101 let registry = Arc::new(Registry::default());
13102 let forwarding = Arc::new(ForwardingTable::default());
13103 let handler = ControlHandler::with_forwarding(registry, forwarding);
13104 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
13105 hello_via_sink(
13106 &handler,
13107 &target_ctx,
13108 &mut target_rx,
13109 hello_frame("target", PROTOCOL_VERSION, 1),
13110 )
13111 .await;
13112
13113 let responses = handler
13114 .handle_control_frame(
13115 &route_ctx(ConnectionId::new(79)).0,
13116 route_open_frame_with_admission_facts(
13117 16,
13118 "target",
13119 unique_project_root("admission-facts"),
13120 None,
13121 Some(json!({"x": 1})),
13122 ),
13123 )
13124 .await
13125 .unwrap();
13126 assert_eq!(
13127 parse_error(&responses[0])["code"],
13128 "admission_facts_not_permitted"
13129 );
13130 }
13131
13132 #[tokio::test]
13133 async fn route_open_relays_consumer_capabilities_verbatim() {
13134 let registry = Arc::new(Registry::default());
13135 let forwarding = Arc::new(ForwardingTable::default());
13136 let handler =
13137 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13138 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
13139 hello_via_sink(
13140 &handler,
13141 &module_ctx,
13142 &mut module_rx,
13143 hello_frame("aft", PROTOCOL_VERSION, 7),
13144 )
13145 .await;
13146
13147 let expected = vec!["elicitation".to_string(), "roots".to_string()];
13148 let expected_for_request = expected.clone();
13149 let project_root = unique_project_root("consumer-capabilities-present");
13150 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
13151 let route_handler = handler.clone();
13152 let route_task = tokio::spawn(async move {
13153 route_handler
13154 .handle_control_frame(
13155 &client_ctx,
13156 route_open_frame_with_consumer_capabilities(
13157 401,
13158 "aft",
13159 project_root,
13160 Some(expected_for_request),
13161 ),
13162 )
13163 .await
13164 .unwrap()
13165 });
13166 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13167 .await
13168 .unwrap()
13169 .unwrap();
13170 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13171 let ModuleControlRequest::RouteBind {
13172 consumer_capabilities,
13173 ..
13174 } = bind
13175 else {
13176 panic!("expected route.bind request, got {bind:?}");
13177 };
13178 assert_eq!(consumer_capabilities, Some(expected.clone()));
13179
13180 handler
13181 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13182 .await
13183 .unwrap();
13184 let route_response = route_task.await.unwrap();
13185 assert!(route_response.is_empty());
13186 let published = client_rx.recv().await.unwrap();
13187 assert!(matches!(
13188 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13189 ClientControlResponse::RouteOpen { .. }
13190 ));
13191 }
13192
13193 #[tokio::test]
13194 async fn route_open_without_consumer_capabilities_relays_none() {
13195 let registry = Arc::new(Registry::default());
13196 let forwarding = Arc::new(ForwardingTable::default());
13197 let handler =
13198 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13199 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
13200 hello_via_sink(
13201 &handler,
13202 &module_ctx,
13203 &mut module_rx,
13204 hello_frame("aft", PROTOCOL_VERSION, 7),
13205 )
13206 .await;
13207
13208 let project_root = unique_project_root("consumer-capabilities-absent");
13209 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
13210 let route_handler = handler.clone();
13211 let route_task = tokio::spawn(async move {
13212 route_handler
13213 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
13214 .await
13215 .unwrap()
13216 });
13217 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13218 .await
13219 .unwrap()
13220 .unwrap();
13221 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13222 let ModuleControlRequest::RouteBind {
13223 consumer_capabilities,
13224 ..
13225 } = bind
13226 else {
13227 panic!("expected route.bind request, got {bind:?}");
13228 };
13229 assert_eq!(consumer_capabilities, None);
13230
13231 handler
13232 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13233 .await
13234 .unwrap();
13235 let route_response = route_task.await.unwrap();
13236 assert!(route_response.is_empty());
13237 let published = client_rx.recv().await.unwrap();
13238 assert!(matches!(
13239 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13240 ClientControlResponse::RouteOpen { .. }
13241 ));
13242 }
13243
13244 async fn bind_role_versions_for(
13248 sent: Option<BTreeMap<String, String>>,
13249 connection: u64,
13250 ) -> Option<BTreeMap<String, String>> {
13251 let registry = Arc::new(Registry::default());
13252 let forwarding = Arc::new(ForwardingTable::default());
13253 let handler =
13254 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13255 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
13256 hello_via_sink(
13257 &handler,
13258 &module_ctx,
13259 &mut module_rx,
13260 hello_frame("aft", PROTOCOL_VERSION, 7),
13261 )
13262 .await;
13263 let project_root = unique_project_root("role-versions");
13264 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
13265 let route_handler = handler.clone();
13266 let route_task = tokio::spawn(async move {
13267 route_handler
13268 .handle_control_frame(
13269 &client_ctx,
13270 route_open_frame_with_role_versions(403, "aft", project_root, sent),
13271 )
13272 .await
13273 .unwrap()
13274 });
13275 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13276 .await
13277 .expect("a well-formed route.open reaches the module as a bind")
13278 .unwrap();
13279 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13280 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
13281 panic!("expected route.bind request, got {bind:?}");
13282 };
13283 handler
13284 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13285 .await
13286 .unwrap();
13287 assert!(route_task.await.unwrap().is_empty());
13288 let published = client_rx.recv().await.unwrap();
13289 assert!(matches!(
13290 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13291 ClientControlResponse::RouteOpen { .. }
13292 ));
13293 role_versions
13294 }
13295
13296 #[tokio::test]
13297 async fn route_open_relays_role_versions_verbatim() {
13298 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
13299 assert_eq!(
13300 bind_role_versions_for(Some(sent.clone()), 141).await,
13301 Some(sent)
13302 );
13303 }
13304
13305 #[tokio::test]
13308 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
13309 assert_eq!(bind_role_versions_for(None, 143).await, None);
13310 assert_eq!(
13311 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
13312 None
13313 );
13314 }
13315
13316 #[tokio::test]
13317 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
13318 let registry = Arc::new(Registry::default());
13319 let forwarding = Arc::new(ForwardingTable::default());
13320 let handler =
13321 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13322 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
13323 hello_via_sink(
13324 &handler,
13325 &module_ctx,
13326 &mut module_rx,
13327 hello_frame("aft", PROTOCOL_VERSION, 7),
13328 )
13329 .await;
13330
13331 let nine: BTreeMap<String, String> = (0..9)
13332 .map(|index| (format!("role-{index}"), "v1".to_string()))
13333 .collect();
13334 for (label, malformed) in [
13335 (
13336 "invalid role name",
13337 role_versions(&[("Tool_Provider", "v1")]),
13338 ),
13339 ("invalid version", role_versions(&[("tool-provider", "v0")])),
13340 ("nine entries", nine),
13341 ] {
13342 let responses = tokio::time::timeout(
13345 Duration::from_secs(1),
13346 handler.handle_control_frame(
13347 &route_ctx(ConnectionId::new(148)).0,
13348 route_open_frame_with_role_versions(
13349 404,
13350 "aft",
13351 unique_project_root("role-versions-malformed"),
13352 Some(malformed),
13353 ),
13354 ),
13355 )
13356 .await
13357 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
13358 .unwrap();
13359 assert_eq!(responses.len(), 1, "{label}");
13360 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
13361 let error = parse_error(&responses[0]);
13362 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
13363 assert_eq!(
13364 error["detail"]["field"], "role_versions",
13365 "{label}: {error}"
13366 );
13367 assert!(
13368 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
13369 "{label}: a malformed declaration is terminal"
13370 );
13371 assert!(
13372 module_rx.try_recv().is_err(),
13373 "{label}: the module must never see a bind"
13374 );
13375 }
13376 }
13377
13378 #[tokio::test]
13382 async fn route_role_versions_capability_is_advertised() {
13383 let handler = ControlHandler::new(Arc::new(Registry::default()));
13384 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
13385 let ack = hello_via_sink(
13386 &handler,
13387 &ctx,
13388 &mut rx,
13389 hello_frame("m", PROTOCOL_VERSION, 1),
13390 )
13391 .await;
13392 let ack = parse_ack(&ack);
13393 assert!(
13394 ack.subc_capabilities
13395 .iter()
13396 .any(|c| c == "route-role-versions/v1"),
13397 "{:?}",
13398 ack.subc_capabilities
13399 );
13400
13401 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13402 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13403 let reply = handler
13404 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
13405 .await
13406 .unwrap()
13407 .pop()
13408 .unwrap();
13409 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13410 serde_json::from_slice(&reply.body).unwrap()
13411 else {
13412 panic!("not a server.describe reply");
13413 };
13414 assert!(
13415 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
13416 "{capabilities:?}"
13417 );
13418 }
13419
13420 #[tokio::test]
13424 async fn catalog_events_capability_is_advertised() {
13425 let handler = ControlHandler::new(Arc::new(Registry::default()));
13426 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13427 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13428 let reply = handler
13429 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
13430 .await
13431 .unwrap()
13432 .pop()
13433 .unwrap();
13434 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13435 serde_json::from_slice(&reply.body).unwrap()
13436 else {
13437 panic!("not a server.describe reply");
13438 };
13439 assert!(
13440 capabilities
13441 .iter()
13442 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
13443 "{capabilities:?}"
13444 );
13445 }
13446
13447 #[tokio::test]
13448 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
13449 let handler = ControlHandler::new(Arc::new(Registry::default()));
13450 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
13451 let ack = hello_via_sink(
13452 &handler,
13453 &module,
13454 &mut module_rx,
13455 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
13456 )
13457 .await;
13458 let ack = parse_ack(&ack);
13459 assert!(
13460 ack.subc_capabilities
13461 .iter()
13462 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13463 "{:?}",
13464 ack.subc_capabilities
13465 );
13466 assert!(ack
13467 .subc_capabilities
13468 .iter()
13469 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13470
13471 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
13472 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13473 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13474 let reply = handler
13475 .handle_control_frame(&client, frame)
13476 .await
13477 .unwrap()
13478 .pop()
13479 .unwrap();
13480 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13481 serde_json::from_slice(&reply.body).unwrap()
13482 else {
13483 panic!("not a server.describe reply");
13484 };
13485 assert!(
13486 capabilities
13487 .iter()
13488 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13489 "{capabilities:?}"
13490 );
13491 assert!(capabilities
13492 .iter()
13493 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13494 }
13495
13496 #[tokio::test]
13497 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
13498 let handler = ControlHandler::new(Arc::new(Registry::default()));
13499 let (module, mut rx) = route_ctx(ConnectionId::new(156));
13500 let ack = hello_via_sink(
13501 &handler,
13502 &module,
13503 &mut rx,
13504 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
13505 )
13506 .await;
13507 assert!(parse_ack(&ack)
13508 .subc_capabilities
13509 .iter()
13510 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13511 assert!(parse_ack(&ack)
13512 .subc_capabilities
13513 .iter()
13514 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13515 let (client, _rx) = route_ctx(ConnectionId::new(157));
13516 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13517 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13518 let reply = handler
13519 .handle_control_frame(&client, frame)
13520 .await
13521 .unwrap()
13522 .pop()
13523 .unwrap();
13524 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13525 serde_json::from_slice(&reply.body).unwrap()
13526 else {
13527 panic!("server.describe reply")
13528 };
13529 assert!(capabilities
13530 .iter()
13531 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13532 assert!(capabilities
13533 .iter()
13534 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13535 }
13536
13537 #[tokio::test]
13538 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
13539 let handler = ControlHandler::new(Arc::new(Registry::default()));
13540 let (module, mut rx) = route_ctx(ConnectionId::new(158));
13541 let ack = hello_via_sink(
13542 &handler,
13543 &module,
13544 &mut rx,
13545 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
13546 )
13547 .await;
13548 assert!(parse_ack(&ack)
13549 .subc_capabilities
13550 .iter()
13551 .any(|cap| cap == "scope-workdir/v1"));
13552 let (client, _rx) = route_ctx(ConnectionId::new(159));
13553 let frame = Frame::build(
13554 FrameType::Request,
13555 control_flags(),
13556 0,
13557 0,
13558 5,
13559 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13560 )
13561 .unwrap();
13562 let reply = handler
13563 .handle_control_frame(&client, frame)
13564 .await
13565 .unwrap()
13566 .remove(0);
13567 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13568 serde_json::from_slice(&reply.body).unwrap()
13569 else {
13570 panic!("server.describe reply")
13571 };
13572 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
13573 }
13574
13575 #[tokio::test]
13576 async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
13577 let handler = ControlHandler::new(Arc::new(Registry::default()));
13578 let (module, mut rx) = route_ctx(ConnectionId::new(160));
13579 let ack = hello_via_sink(
13580 &handler,
13581 &module,
13582 &mut rx,
13583 hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
13584 )
13585 .await;
13586 assert!(parse_ack(&ack)
13587 .subc_capabilities
13588 .iter()
13589 .any(|cap| cap == "worker-runs/v1"));
13590 let (client, _rx) = route_ctx(ConnectionId::new(161));
13591 let frame = Frame::build(
13592 FrameType::Request,
13593 control_flags(),
13594 0,
13595 0,
13596 5,
13597 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13598 )
13599 .unwrap();
13600 let reply = handler
13601 .handle_control_frame(&client, frame)
13602 .await
13603 .unwrap()
13604 .remove(0);
13605 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13606 serde_json::from_slice(&reply.body).unwrap()
13607 else {
13608 panic!("server.describe reply")
13609 };
13610 assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
13611 }
13612
13613 #[tokio::test]
13614 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
13615 let handler = ControlHandler::new(Arc::new(Registry::default()));
13616 let (module, mut rx) = route_ctx(ConnectionId::new(154));
13617 let ack = hello_via_sink(
13618 &handler,
13619 &module,
13620 &mut rx,
13621 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
13622 )
13623 .await;
13624 assert!(parse_ack(&ack)
13625 .subc_capabilities
13626 .iter()
13627 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13628 let (client, _rx) = route_ctx(ConnectionId::new(155));
13629 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13630 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13631 let reply = handler
13632 .handle_control_frame(&client, frame)
13633 .await
13634 .unwrap()
13635 .pop()
13636 .unwrap();
13637 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13638 serde_json::from_slice(&reply.body).unwrap()
13639 else {
13640 panic!("not a server.describe reply")
13641 };
13642 assert!(capabilities
13643 .iter()
13644 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13645 }
13646
13647 #[tokio::test]
13648 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
13649 let registry = Arc::new(Registry::default());
13650 let forwarding = Arc::new(ForwardingTable::default());
13651 let handler =
13652 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13653 .with_health_probe_timeout(Duration::from_secs(5));
13654 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
13655 hello_via_sink(
13656 &handler,
13657 &module_ctx,
13658 &mut module_rx,
13659 non_routable_hello_frame_with_control_ops(
13660 "mcp",
13661 300,
13662 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13663 ),
13664 )
13665 .await;
13666 assert!(registry
13667 .get_module("mcp")
13668 .unwrap()
13669 .unwrap()
13670 .manifest
13671 .provides
13672 .is_empty());
13673
13674 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
13675 let route_response = handler
13676 .handle_control_frame(
13677 &route_client_ctx,
13678 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
13679 )
13680 .await
13681 .unwrap();
13682 assert_eq!(route_response[0].header.ty, FrameType::Error);
13683 assert_eq!(
13684 parse_error(&route_response[0])["code"],
13685 "target_unavailable"
13686 );
13687 assert!(parse_error(&route_response[0])["message"]
13688 .as_str()
13689 .unwrap()
13690 .contains("does not provide the requested target"));
13691 assert!(module_rx.try_recv().is_err());
13692
13693 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
13694 let health_handler = handler.clone();
13695 let health_task = tokio::spawn(async move {
13696 health_handler
13697 .handle_control_frame(
13698 &health_client_ctx,
13699 supervisor_health_probe_frame(302, "mcp"),
13700 )
13701 .await
13702 .unwrap()
13703 });
13704 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13705 .await
13706 .unwrap()
13707 .unwrap();
13708 assert_eq!(
13709 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13710 ModuleControlRequest::HealthCheck {}
13711 );
13712 handler
13713 .handle_control_frame(
13714 &module_ctx,
13715 health_response(health_frame.header.corr, HealthStatus::Ok),
13716 )
13717 .await
13718 .unwrap();
13719 let health_response = health_task.await.unwrap();
13720 assert_eq!(health_response[0].header.ty, FrameType::Response);
13721 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13722 ClientControlResponse::SupervisorHealthProbe {
13723 module_id, status, ..
13724 } => {
13725 assert_eq!(module_id, "mcp");
13726 assert_eq!(status, HealthStatus::Ok);
13727 }
13728 other => panic!("unexpected health response: {other:?}"),
13729 }
13730
13731 forwarding
13736 .cleanup_connection(module_ctx.connection_id)
13737 .unwrap();
13738 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
13739 let cleanup_response = tokio::time::timeout(
13740 Duration::from_millis(200),
13741 handler.handle_control_frame(
13742 &cleanup_probe_ctx,
13743 supervisor_health_probe_frame(303, "mcp"),
13744 ),
13745 )
13746 .await
13747 .expect("probe should fail immediately when the control lane is gone")
13748 .unwrap();
13749 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
13750 assert_eq!(
13751 parse_error(&cleanup_response[0])["code"],
13752 "target_unavailable"
13753 );
13754 assert!(parse_error(&cleanup_response[0])["message"]
13755 .as_str()
13756 .unwrap()
13757 .contains("no module connection"));
13758
13759 handler
13760 .cleanup_connection(module_ctx.connection_id)
13761 .unwrap();
13762 }
13763
13764 #[tokio::test]
13765 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
13766 let registry = Arc::new(Registry::default());
13767 let supervisor_handle = SupervisorHandle::new();
13768 let supervisor =
13769 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13770 .with_handle(supervisor_handle.clone())
13771 .with_connection_file_path(
13772 std::env::temp_dir()
13773 .join(format!("subc-route-open-warming-{}", std::process::id())),
13774 );
13775 let module = supervisor
13776 .supervise_configured(
13777 ModuleSpec {
13778 carry_containment: false,
13779 module_id: "warming".to_string(),
13780 program: fake_aft_stub_path(),
13781 args: Vec::new(),
13782 env: Vec::new(),
13783 reserved: false,
13784 reserved_prefixes: Vec::new(),
13785 protocol: ModuleProtocol::Subc,
13786 overlap: Default::default(),
13787 },
13788 true,
13789 )
13790 .unwrap();
13791 assert_eq!(module.state().unwrap(), ModuleState::Running);
13792
13793 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13794 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
13795 let response = handler
13796 .handle_control_frame(
13797 &ctx,
13798 route_open_frame(304, "warming", unique_project_root("warming")),
13799 )
13800 .await
13801 .unwrap();
13802 module.stop().await.unwrap();
13803
13804 assert_eq!(response[0].header.ty, FrameType::Error);
13805 let error = parse_error(&response[0]);
13806 assert_eq!(error["code"], "module_warming");
13807 assert!(error["message"]
13808 .as_str()
13809 .unwrap()
13810 .contains("state=running, enabled=true, live=false"));
13811 }
13812
13813 #[test]
13814 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
13815 let handler = ControlHandler::new(Arc::new(Registry::default()));
13816 let capture = EventCapture::default();
13817 let _subscriber =
13818 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13819 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
13820 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
13821 let pending = (0..limit).collect::<Vec<_>>();
13822 let response = handler
13823 .route_open_capacity_refusal(
13824 &ctx,
13825 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
13826 "busy",
13827 pending.len(),
13828 limit,
13829 )
13830 .unwrap();
13831 assert_eq!(parse_error(&response)["code"], "target_unavailable");
13832 let event = capture
13833 .events()
13834 .into_iter()
13835 .find(|event| {
13836 event.target == "control"
13837 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
13838 })
13839 .expect("connection admission refusal event");
13840 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13841 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13842 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13845 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13846 }
13847
13848 #[test]
13849 fn route_open_target_cap_logs_admission_reason_and_capacity() {
13850 let handler = ControlHandler::new(Arc::new(Registry::default()));
13851 let capture = EventCapture::default();
13852 let _subscriber =
13853 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13854 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
13855 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
13856 let guards = (0..limit)
13857 .map(|_| {
13858 handler
13859 .route_bind_concurrency
13860 .try_admit("busy", limit)
13861 .unwrap()
13862 })
13863 .collect::<Vec<_>>();
13864 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
13865 Err(in_flight) => in_flight,
13866 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
13867 };
13868 let response = handler
13869 .route_open_target_capacity_refusal(
13870 &ctx,
13871 &RouteOpenCaller::unknown(),
13872 &route_open_frame(397, "busy", unique_project_root("target-cap")),
13873 "busy",
13874 in_flight,
13875 )
13876 .unwrap();
13877 assert_eq!(parse_error(&response)["code"], "target_unavailable");
13878 let event = capture
13879 .events()
13880 .into_iter()
13881 .find(|event| {
13882 event.target == "control"
13883 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
13884 })
13885 .expect("target admission refusal event");
13886 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13887 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13888 drop(guards);
13889 }
13890
13891 #[cfg(unix)]
13892 #[tokio::test(start_paused = true)]
13893 async fn supervisor_list_shows_a_failed_modules_next_retry() {
13894 let registry = Arc::new(Registry::default());
13895 let handle = SupervisorHandle::new();
13896 let supervisor =
13897 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
13898 .with_handle(handle.clone());
13899 let module = supervisor
13900 .spawn(crate::ModuleSpec {
13901 carry_containment: false,
13902 module_id: "retry-list".into(),
13903 program: PathBuf::from("/bin/sh"),
13904 args: vec!["-c".into(), "exit 17".into()],
13905 env: vec![],
13906 reserved: false,
13907 reserved_prefixes: vec![],
13908 protocol: ModuleProtocol::None,
13909 overlap: Default::default(),
13910 })
13911 .unwrap();
13912 let deadline = std::time::Instant::now() + Duration::from_secs(10);
13913 while module.state().unwrap() != crate::ModuleState::Failed {
13914 assert!(
13915 std::time::Instant::now() < deadline,
13916 "waiting for budget failure"
13917 );
13918 tokio::task::yield_now().await;
13919 }
13920 let handler = ControlHandler::new(registry).with_supervisor(handle);
13921 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13922 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13923 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13924 assert_eq!(body["modules"][0]["state"], "failed");
13925 let retry = body["modules"][0]["next_retry_at_ms"]
13926 .as_u64()
13927 .expect("scheduled retry on the wire");
13928 let now = std::time::SystemTime::now()
13929 .duration_since(std::time::UNIX_EPOCH)
13930 .unwrap()
13931 .as_millis() as u64;
13932 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
13933 module.set_enabled(false).await.unwrap();
13934 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13935 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13936 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13937 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
13938 }
13939
13940 #[tokio::test]
13941 async fn slow_step_before_pid_read_does_not_spend_its_budget() {
13942 let _guard = crate::subtree_resources::fault_read_guard().await;
13943 let dir = cortexkit_test_support::ScratchDir::new("pid-read-budget");
13944 let supervisor =
13945 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13946 let module = supervisor
13947 .spawn(crate::ModuleSpec {
13948 carry_containment: false,
13949 module_id: "pid-read-budget".into(),
13950 program: std::env::current_exe().unwrap(),
13951 args: vec![
13952 "--exact".into(),
13953 "control::tests::pid_resource_read_fixture".into(),
13954 ],
13955 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
13956 .into_iter()
13957 .map(|key| (key.into(), dir.display().to_string()))
13958 .chain(std::iter::once((
13959 "SUBC_SUBTREE_PID_FIXTURE".into(),
13960 "1".into(),
13961 )))
13962 .collect(),
13963 reserved: false,
13964 reserved_prefixes: Vec::new(),
13965 protocol: ModuleProtocol::None,
13966 overlap: Default::default(),
13967 })
13968 .unwrap();
13969 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
13970 while !matches!(
13971 module.child_resource_usage(),
13972 subc_control::ChildResourceUsage::Measured(_)
13973 ) {
13974 assert!(
13975 tokio::time::Instant::now() < ready_deadline,
13976 "fixture process was not measurable"
13977 );
13978 tokio::time::sleep(Duration::from_millis(10)).await;
13979 }
13980 tokio::time::sleep(crate::subtree_resources::READ_TIMEOUT * 2).await;
13981 let reading = crate::subtree_resources::read_process(module.clone()).await;
13982 module.stop().await.unwrap();
13983 assert!(
13984 matches!(reading, subc_control::ChildResourceUsage::Measured(_)),
13985 "{reading:?}"
13986 );
13987 }
13988
13989 #[test]
13990 fn pid_resource_read_fixture() {
13991 if std::env::var_os("SUBC_SUBTREE_PID_FIXTURE").is_some() {
13992 loop {
13993 std::thread::park_timeout(Duration::from_secs(600));
13994 }
13995 }
13996 }
13997
13998 #[tokio::test]
13999 async fn supervisor_list_slow_subtree_does_not_hold_connection_reader() {
14000 let _guard = crate::subtree_resources::fault_read_guard().await;
14001 let registry = Arc::new(Registry::default());
14002 let handle = SupervisorHandle::new();
14003 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14004 .with_handle(handle.clone());
14005 let module = supervisor
14006 .supervise_configured(
14007 crate::ModuleSpec {
14008 carry_containment: false,
14009 module_id: "memory-read".into(),
14010 program: PathBuf::from("disabled-module"),
14011 args: Vec::new(),
14012 env: Vec::new(),
14013 reserved: false,
14014 reserved_prefixes: Vec::new(),
14015 protocol: ModuleProtocol::Subc,
14016 overlap: Default::default(),
14017 },
14018 false,
14019 )
14020 .unwrap();
14021 let (target, release, began) = crate::subtree_resources::held();
14022 module.set_subtree_target_for_test(target);
14023 let handler = ControlHandler::new(registry).with_supervisor(handle);
14024 let (ctx, mut rx) = route_ctx(ConnectionId::new(112));
14025 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14026 let replies = handler
14027 .handle_client_control_request(
14028 &ctx,
14029 frame,
14030 ClientControlRequest::SupervisorList {},
14031 &mut None,
14032 )
14033 .await
14034 .unwrap();
14035 assert!(
14036 replies.is_empty(),
14037 "list reply must be detached from the connection reader"
14038 );
14039 began.await.unwrap();
14040 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14041 let replies = handler
14042 .handle_client_control_request(
14043 &ctx,
14044 frame,
14045 ClientControlRequest::SupervisorSpawnSnapshot {},
14046 &mut None,
14047 )
14048 .await
14049 .unwrap();
14050 assert_eq!(
14051 replies.len(),
14052 1,
14053 "another request must complete while the subtree read stalls"
14054 );
14055 let reply = tokio::time::timeout(Duration::from_secs(5), rx.recv())
14056 .await
14057 .unwrap()
14058 .unwrap();
14059 release.send(()).unwrap();
14060 drop(release);
14061 let body: Value = serde_json::from_slice(&reply.frame.body).unwrap();
14062 assert_eq!(body["modules"][0]["subtree"]["source"], "unavailable");
14063 assert_eq!(body["modules"][0]["subtree"]["reason"], "timeout");
14064 assert_eq!(
14065 body["modules"][0]["subtree"]["process_count"], 1,
14066 "a count obtained before the stalled read must survive timeout"
14067 );
14068 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14069 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14070 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14071 assert_eq!(
14072 body["modules"][0]["subtree"]["source"], "process_sum",
14073 "the next request must read afresh"
14074 );
14075 }
14076
14077 #[tokio::test]
14078 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
14079 let registry = Arc::new(Registry::default());
14080 let handle = SupervisorHandle::new();
14081 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14082 .with_handle(handle.clone());
14083 supervisor
14084 .supervise_configured(
14085 crate::ModuleSpec {
14086 carry_containment: false,
14087 module_id: "drift".into(),
14088 program: PathBuf::from("disabled-module"),
14089 args: Vec::new(),
14090 env: Vec::new(),
14091 reserved: false,
14092 reserved_prefixes: Vec::new(),
14093 protocol: ModuleProtocol::Subc,
14094 overlap: Default::default(),
14095 },
14096 false,
14097 )
14098 .unwrap();
14099 let handler = ControlHandler::new(registry).with_supervisor(handle);
14100 for configured in [None, Some(false), Some(true)] {
14101 handler.capability_evaluator.configure(
14102 configured.map(|enabled| ("drift".to_string(), enabled)),
14103 BTreeMap::new(),
14104 );
14105 let frame =
14106 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14107 let replies = handler
14108 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
14109 .await
14110 .unwrap();
14111 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14112 assert!(body.get("configured_enabled").is_none());
14113 assert_eq!(body["op"], "supervisor.ack");
14114 let frame =
14115 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14116 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14117 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14118 assert_eq!(
14119 body["modules"][0].get("configured_enabled"),
14120 configured.map(Value::Bool).as_ref()
14121 );
14122 assert_eq!(body["modules"][0]["subtree"]["source"], "unavailable");
14123 assert_eq!(body["modules"][0]["subtree"]["reason"], "not_running");
14124 }
14125 }
14126
14127 #[tokio::test(flavor = "current_thread")]
14128 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
14129 let handler = ControlHandler::new(Arc::new(Registry::default()));
14130 let capture = EventCapture::default();
14131 let _subscriber =
14132 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14133 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
14134 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14135
14136 let _ = handler
14137 .handle_client_control_request(
14138 &ctx,
14139 frame,
14140 ClientControlRequest::SupervisorSetEnabled {
14141 module_id: "broca".to_string(),
14142 enabled: false,
14143 },
14144 &mut None,
14145 )
14146 .await
14147 .unwrap();
14148
14149 let events = capture
14150 .events()
14151 .into_iter()
14152 .filter(|event| {
14153 event.target == "control"
14154 && event.fields.get("message").map(String::as_str)
14155 == Some("supervisor request received")
14156 })
14157 .collect::<Vec<_>>();
14158 assert_eq!(events.len(), 1, "one supervisor request log line");
14159 let event = &events[0];
14160 assert_eq!(event.level, tracing::Level::INFO);
14161 assert_eq!(
14162 event.fields.get("op"),
14163 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
14164 );
14165 assert_eq!(
14166 event.fields.get("module_id"),
14167 Some(&"\"broca\"".to_string())
14168 );
14169 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
14170 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
14171 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
14172 }
14173
14174 #[tokio::test(flavor = "current_thread")]
14175 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
14176 let registry = Arc::new(Registry::default());
14177 let module_connection = ConnectionId::new(303);
14178 registry
14179 .register_with_control_ops(
14180 manifest("aft", PROTOCOL_VERSION),
14181 PROTOCOL_VERSION,
14182 module_connection,
14183 module_baseline_control_ops(),
14184 )
14185 .unwrap();
14186 let handler = ControlHandler::new(registry);
14187 let capture = EventCapture::default();
14188 let _subscriber =
14189 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14190 let (ctx, _rx) = route_ctx(module_connection);
14191 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14192
14193 let _ = handler
14194 .handle_client_control_request(
14195 &ctx,
14196 frame,
14197 ClientControlRequest::SupervisorSetEnabled {
14198 module_id: "broca".to_string(),
14199 enabled: false,
14200 },
14201 &mut None,
14202 )
14203 .await
14204 .unwrap();
14205
14206 let callers = capture
14207 .events()
14208 .into_iter()
14209 .filter(|event| {
14210 event.fields.get("message").map(String::as_str)
14211 == Some("supervisor request received")
14212 })
14213 .map(|event| event.fields.get("caller").cloned())
14214 .collect::<Vec<_>>();
14215 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
14216 }
14217
14218 #[tokio::test(flavor = "current_thread")]
14219 async fn supervisor_list_does_not_log_request_received() {
14220 let handler = ControlHandler::new(Arc::new(Registry::default()));
14221 let capture = EventCapture::default();
14222 let _subscriber =
14223 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14224 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
14225 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14226
14227 handler
14228 .handle_client_control_request(
14229 &ctx,
14230 frame,
14231 ClientControlRequest::SupervisorList {},
14232 &mut None,
14233 )
14234 .await
14235 .unwrap();
14236
14237 assert!(capture.events().into_iter().all(|event| {
14238 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14239 }));
14240 }
14241
14242 #[tokio::test(flavor = "current_thread")]
14243 async fn supervisor_rescan_preview_does_not_log_request_received() {
14244 let handler = ControlHandler::new(Arc::new(Registry::default()));
14245 let capture = EventCapture::default();
14246 let _subscriber =
14247 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14248 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
14249 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14250
14251 handler
14252 .handle_client_control_request(
14253 &ctx,
14254 frame,
14255 ClientControlRequest::SupervisorRescan { preview: true },
14256 &mut None,
14257 )
14258 .await
14259 .unwrap();
14260
14261 assert!(capture.events().into_iter().all(|event| {
14262 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14263 }));
14264 }
14265
14266 struct ApproveOperator;
14267 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
14268 fn prompt(
14269 &self,
14270 _: &str,
14271 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
14272 ) -> crate::operator_confirm::ProviderResult {
14273 crate::operator_confirm::ProviderResult::Approved
14274 }
14275 }
14276
14277 #[tokio::test]
14278 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
14279 assert_eq!(
14280 ControlHandler::default().privileged_operator_prompt,
14281 PrivilegedOperatorPrompt::Log
14282 );
14283 for (initial, changed, running) in [
14284 ("log", "enforce", PrivilegedOperatorPrompt::Log),
14285 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
14286 ] {
14287 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
14288 let path = dir.join("subc.jsonc");
14289 std::fs::write(
14290 &path,
14291 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
14292 )
14293 .unwrap();
14294 let config = crate::daemon_config::load(&path).unwrap().unwrap();
14295 let registry = Arc::new(Registry::default());
14296 let handle = SupervisorHandle::new();
14297 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
14298 .with_handle(handle.clone());
14299 let handler = ControlHandler::new(registry)
14300 .with_supervisor(handle)
14301 .with_privileged_operator_prompt(config.privileged_operator_prompt)
14302 .with_supervisor_rescan(supervisor, &path, config.port);
14303 handler.forwarding.operator_confirms().configure(
14304 Arc::new(ApproveOperator),
14305 crate::operator_confirm::OperatorLimits::default(),
14306 );
14307 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
14308
14309 for (disk_mode, needs_restart) in [
14310 (Some(initial), false),
14311 (Some(changed), true),
14312 (Some(changed), true),
14313 (None, running == PrivilegedOperatorPrompt::Enforce),
14314 (Some(initial), false),
14315 ] {
14316 let mut doc = json!({"version":1});
14317 if let Some(mode) = disk_mode {
14318 doc["privileged_operator_prompt"] = json!(mode);
14319 }
14320 std::fs::write(&path, doc.to_string()).unwrap();
14321 for preview in [true, false] {
14322 let request = ClientControlRequest::SupervisorRescan { preview };
14323 let frame = Frame::build(
14324 FrameType::Request,
14325 control_flags(),
14326 0,
14327 0,
14328 1,
14329 serde_json::to_vec(&request).unwrap(),
14330 )
14331 .unwrap();
14332 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
14333 assert_eq!(frames.len(), 1);
14334 let ClientControlResponse::SupervisorRescan { result } =
14335 serde_json::from_slice(&frames[0].body).unwrap()
14336 else {
14337 panic!("expected a rescan response");
14338 };
14339 let expected = if needs_restart {
14340 vec!["privileged_operator_prompt"]
14341 } else {
14342 Vec::new()
14343 };
14344 assert_eq!(result.restart_required, expected);
14345 assert_eq!(handler.privileged_operator_prompt, running);
14346 assert_eq!(
14347 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
14348 running
14349 );
14350 }
14351 }
14352 }
14353 }
14354
14355 #[tokio::test]
14356 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
14357 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
14358 let path = dir.join("subc.jsonc");
14359 let registry = Arc::new(Registry::default());
14360 let handle = SupervisorHandle::new();
14361 let supervisor =
14362 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
14363 let handler = ControlHandler::new(registry)
14364 .with_supervisor(handle)
14365 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
14366 .with_supervisor_rescan(supervisor, &path, None);
14367 handler.forwarding.operator_confirms().configure(
14368 Arc::new(ApproveOperator),
14369 crate::operator_confirm::OperatorLimits::default(),
14370 );
14371 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
14372 for preview in [true, false] {
14373 std::fs::write(
14374 &path,
14375 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
14376 )
14377 .unwrap();
14378 let request = ClientControlRequest::SupervisorRescan { preview };
14379 let frame = Frame::build(
14380 FrameType::Request,
14381 control_flags(),
14382 0,
14383 0,
14384 1,
14385 serde_json::to_vec(&request).unwrap(),
14386 )
14387 .unwrap();
14388 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
14389 assert_eq!(frames.len(), 1);
14390 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
14391 assert_eq!(
14392 handler.privileged_operator_prompt,
14393 PrivilegedOperatorPrompt::Enforce
14394 );
14395 }
14396 }
14397
14398 #[tokio::test]
14403 async fn route_open_refusal_names_the_check_that_refused() {
14404 let handler = ControlHandler::new(Arc::new(Registry::default()));
14405 let capture = EventCapture::default();
14406 let _subscriber =
14407 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14408 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14409 let response = handler
14410 .handle_control_frame(
14411 &ctx,
14412 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
14413 )
14414 .await
14415 .unwrap();
14416
14417 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14418 let event = capture
14419 .events()
14420 .into_iter()
14421 .find(|event| {
14422 event.target == "control"
14423 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
14424 })
14425 .expect("route.open refusal event");
14426 assert_eq!(
14427 event.fields.get("reason"),
14428 Some(&"\"not_registered\"".to_string())
14429 );
14430 }
14431
14432 #[tokio::test]
14433 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
14434 let registry = Arc::new(Registry::default());
14435 let supervisor_handle = SupervisorHandle::new();
14436 let supervisor =
14437 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
14438 .with_handle(supervisor_handle.clone())
14439 .with_connection_file_path(std::env::temp_dir().join(format!(
14440 "subc-route-open-refusal-info-{}",
14441 std::process::id()
14442 )));
14443 let module = supervisor
14444 .supervise_configured(
14445 ModuleSpec {
14446 carry_containment: false,
14447 module_id: "warming".to_string(),
14448 program: fake_aft_stub_path(),
14449 args: Vec::new(),
14450 env: Vec::new(),
14451 reserved: false,
14452 reserved_prefixes: Vec::new(),
14453 protocol: ModuleProtocol::Subc,
14454 overlap: Default::default(),
14455 },
14456 true,
14457 )
14458 .unwrap();
14459 assert_eq!(module.state().unwrap(), ModuleState::Running);
14460
14461 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14462 assert!(handler
14463 .counters()
14464 .snapshot()
14465 .get("route_open_refused_by_code")
14466 .is_none());
14467 let capture = EventCapture::default();
14468 let _subscriber =
14469 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14470 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
14471 let response = handler
14472 .handle_control_frame(
14473 &ctx,
14474 route_open_frame(394, "warming", unique_project_root("refusal-info")),
14475 )
14476 .await
14477 .unwrap();
14478 module.stop().await.unwrap();
14479
14480 assert_eq!(parse_error(&response[0])["code"], "module_warming");
14481 let event = capture
14482 .events()
14483 .into_iter()
14484 .find(|event| {
14485 event.target == "control"
14486 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
14487 })
14488 .expect("route.open refusal event");
14489 assert_eq!(
14490 event.fields.get("module_id"),
14491 Some(&"\"warming\"".to_string())
14492 );
14493 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
14494 assert_eq!(
14495 event.fields.get("reason"),
14496 Some(&"\"supervised_not_registered\"".to_string())
14497 );
14498 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
14499 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
14500 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
14501 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
14502 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
14503 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
14504 assert_eq!(
14505 handler.counters().snapshot()["route_open_refused_by_code"],
14506 json!({ "module_warming": 1 })
14507 );
14508 }
14509
14510 const OUTAGE_START: &str = "route.open refusing module: not serving";
14511 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
14512
14513 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
14514 capture
14515 .events()
14516 .into_iter()
14517 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
14518 .collect()
14519 }
14520
14521 fn supervise_stub(
14522 registry: &Arc<Registry>,
14523 module_id: &str,
14524 enabled: bool,
14525 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
14526 let supervisor_handle = SupervisorHandle::new();
14527 let supervisor =
14528 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
14529 .with_handle(supervisor_handle.clone())
14530 .with_connection_file_path(std::env::temp_dir().join(format!(
14531 "subc-route-outage-{module_id}-{}",
14532 std::process::id()
14533 )));
14534 let module = supervisor
14535 .supervise_configured(
14536 ModuleSpec {
14537 carry_containment: false,
14538 module_id: module_id.to_string(),
14539 program: fake_aft_stub_path(),
14540 args: Vec::new(),
14541 env: Vec::new(),
14542 reserved: false,
14543 reserved_prefixes: Vec::new(),
14544 protocol: ModuleProtocol::Subc,
14545 overlap: Default::default(),
14546 },
14547 enabled,
14548 )
14549 .unwrap();
14550 (supervisor_handle, module)
14551 }
14552
14553 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
14554 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
14555 module_id: module_id.to_string(),
14556 drain_timeout_ms: Some(50),
14557 })
14558 .unwrap();
14559 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
14560 }
14561
14562 #[test]
14566 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
14567 let registry = Arc::new(Registry::default());
14568 let forwarding = Arc::new(ForwardingTable::default());
14569 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14570 let second = ControlHandler::with_forwarding(registry, forwarding);
14571 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
14572 }
14573
14574 #[tokio::test(flavor = "current_thread")]
14578 async fn route_open_unknown_module_refusals_add_no_outage_state() {
14579 let handler = ControlHandler::new(Arc::new(Registry::default()));
14580 let capture = EventCapture::default();
14581 let _subscriber =
14582 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14583 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
14584 for corr in 0..8 {
14585 let response = handler
14586 .handle_control_frame(
14587 &ctx,
14588 route_open_frame(
14589 380 + corr,
14590 &format!("nobody-{corr}"),
14591 unique_project_root("outage-unknown"),
14592 ),
14593 )
14594 .await
14595 .unwrap();
14596 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14597 }
14598
14599 assert_eq!(handler.route_outages.tracked_module_count(), 0);
14600 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
14601 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
14602 }
14603
14604 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14609 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
14610 let registry = Arc::new(Registry::default());
14611 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
14612 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14613 let capture = EventCapture::default();
14614 let _subscriber =
14615 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14616 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
14617 handler.route_outages.record_accepted("outage-restart");
14620
14621 let response = handler
14622 .handle_control_frame(
14623 &ctx,
14624 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
14625 )
14626 .await
14627 .unwrap();
14628 assert_eq!(response[0].header.ty, FrameType::Error);
14629 let starts = outage_lines(&capture, OUTAGE_START);
14630 assert_eq!(starts.len(), 1, "{starts:?}");
14631 assert_eq!(starts[0].level, tracing::Level::WARN);
14632 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
14633 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
14634 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
14635 handler.route_outages.record_accepted("outage-restart");
14636 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
14637
14638 let restart = handler
14639 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
14640 .await
14641 .unwrap();
14642 assert_eq!(
14643 restart[0].header.ty,
14644 FrameType::Response,
14645 "{:?}",
14646 parse_error(&restart[0])
14647 );
14648 handler
14649 .handle_control_frame(
14650 &ctx,
14651 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
14652 )
14653 .await
14654 .unwrap();
14655 module.stop().await.unwrap();
14656
14657 let starts = outage_lines(&capture, OUTAGE_START);
14658 assert_eq!(starts.len(), 2, "{starts:?}");
14659 assert_eq!(starts[1].level, tracing::Level::INFO);
14660 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
14661 }
14662
14663 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14667 async fn failed_operator_restart_leaves_no_operator_mark() {
14668 let registry = Arc::new(Registry::default());
14669 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
14670 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14671 let capture = EventCapture::default();
14672 let _subscriber =
14673 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14674 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
14675 handler.route_outages.record_accepted("outage-disabled");
14676
14677 let restart = handler
14678 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
14679 .await
14680 .unwrap();
14681 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
14682 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
14683
14684 handler
14685 .handle_control_frame(
14686 &ctx,
14687 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
14688 )
14689 .await
14690 .unwrap();
14691 let starts = outage_lines(&capture, OUTAGE_START);
14692 assert_eq!(starts.len(), 1, "{starts:?}");
14693 assert_eq!(starts[0].level, tracing::Level::WARN);
14694 }
14695
14696 #[tokio::test(flavor = "current_thread")]
14697 async fn route_open_unknown_module_escapes_target_module_id() {
14698 let handler = ControlHandler::new(Arc::new(Registry::default()));
14699 let capture = EventCapture::default();
14700 let _subscriber =
14701 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14702 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
14703 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14704 let response = handler
14705 .handle_control_frame(
14706 &ctx,
14707 route_open_frame(
14708 395,
14709 hostile_module_id,
14710 unique_project_root("hostile-target-module-id"),
14711 ),
14712 )
14713 .await
14714 .unwrap();
14715
14716 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14717 let event = capture
14718 .events()
14719 .into_iter()
14720 .find(|event| {
14721 event.target == "control"
14722 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
14723 })
14724 .expect("route.open unknown-module refusal event");
14725 let logged = event.fields.get("module_id").expect("module_id field");
14726 assert!(!logged.bytes().any(|byte| byte < 0x20));
14727 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
14728 }
14729
14730 #[tokio::test(flavor = "current_thread")]
14733 async fn route_open_refusal_carries_daemon_origin() {
14734 let handler = ControlHandler::new(Arc::new(Registry::default()));
14735 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14736 let response = handler
14737 .handle_control_frame(
14738 &ctx,
14739 route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
14740 )
14741 .await
14742 .unwrap();
14743
14744 assert_eq!(response.len(), 1);
14745 assert_eq!(response[0].header.ty, FrameType::Error);
14746 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14747 assert!(
14748 response[0].header.flags.is_daemon_origin(),
14749 "route.open refusal must carry daemon origin: {:?}",
14750 response[0].header.flags
14751 );
14752 }
14753
14754 #[tokio::test(flavor = "current_thread")]
14758 async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
14759 let registry = Arc::new(Registry::default());
14760 let forwarding = Arc::new(ForwardingTable::default());
14761 let handler =
14762 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14763 let module_connection = ConnectionId::new(95);
14764 let (module_ctx, mut module_rx) = route_ctx(module_connection);
14765 hello_via_sink(
14766 &handler,
14767 &module_ctx,
14768 &mut module_rx,
14769 hello_frame("aft", PROTOCOL_VERSION, 395),
14770 )
14771 .await;
14772 forwarding
14773 .begin_module_drain("aft", RouteCloseReason::Restart)
14774 .unwrap()
14775 .expect("the module endpoint drains");
14776
14777 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
14778 let response = handler
14779 .handle_control_frame(
14780 &client_ctx,
14781 route_open_frame(396, "aft", unique_project_root("drain-origin")),
14782 )
14783 .await
14784 .unwrap();
14785
14786 assert_eq!(response.len(), 1);
14787 assert_eq!(response[0].header.ty, FrameType::Error);
14788 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14789 assert!(
14790 response[0].header.flags.is_daemon_origin(),
14791 "drain module_reloading must carry daemon origin: {:?}",
14792 response[0].header.flags
14793 );
14794 }
14795
14796 #[tokio::test(flavor = "current_thread")]
14800 async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
14801 let registry = Arc::new(Registry::default());
14802 let forwarding = Arc::new(ForwardingTable::default());
14803 let handler =
14804 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14805 let module_connection = ConnectionId::new(95);
14806 let (module_ctx, mut module_rx) = route_ctx(module_connection);
14807 hello_via_sink(
14808 &handler,
14809 &module_ctx,
14810 &mut module_rx,
14811 hello_frame("aft", PROTOCOL_VERSION, 395),
14812 )
14813 .await;
14814
14815 let client_connection = ConnectionId::new(96);
14816 let (client_ctx, _client_rx) = route_ctx(client_connection);
14817 let (route_task, bind) = relay_route_open(
14818 &handler,
14819 client_connection,
14820 &client_ctx.egress,
14821 &mut module_rx,
14822 396,
14823 "aft",
14824 "spoofed-reloading",
14825 )
14826 .await;
14827 let rejection = Frame::build(
14830 FrameType::Error,
14831 crate::router::daemon_error_flags(),
14832 0,
14833 0,
14834 bind.header.corr,
14835 serde_json::to_vec(&ErrorBody::new(
14836 "module_reloading",
14837 "module refused route.bind",
14838 ))
14839 .unwrap(),
14840 )
14841 .unwrap();
14842 handler
14843 .handle_control_frame(&module_ctx, rejection)
14844 .await
14845 .unwrap();
14846
14847 let response = route_task.await.unwrap();
14848 assert_eq!(response.len(), 1);
14849 assert_eq!(response[0].header.ty, FrameType::Error);
14850 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14851 assert!(
14852 !response[0].header.flags.is_daemon_origin(),
14853 "a module's own rejection must not carry daemon origin: {:?}",
14854 response[0].header.flags
14855 );
14856 }
14857
14858 #[tokio::test(flavor = "current_thread")]
14859 async fn route_open_module_rejection_uses_daemon_counter_key() {
14860 let registry = Arc::new(Registry::default());
14861 let forwarding = Arc::new(ForwardingTable::default());
14862 let handler =
14863 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14864 let module_connection = ConnectionId::new(95);
14865 let (module_ctx, mut module_rx) = route_ctx(module_connection);
14866 hello_via_sink(
14867 &handler,
14868 &module_ctx,
14869 &mut module_rx,
14870 hello_frame("aft", PROTOCOL_VERSION, 395),
14871 )
14872 .await;
14873
14874 let client_connection = ConnectionId::new(96);
14875 let (client_ctx, _client_rx) = route_ctx(client_connection);
14876 let capture = EventCapture::default();
14877 let _subscriber =
14878 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14879 let (route_task, bind) = relay_route_open(
14880 &handler,
14881 client_connection,
14882 &client_ctx.egress,
14883 &mut module_rx,
14884 396,
14885 "aft",
14886 "hostile-module-code",
14887 )
14888 .await;
14889 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
14890 let rejection = Frame::build(
14891 FrameType::Error,
14892 control_flags(),
14893 0,
14894 0,
14895 bind.header.corr,
14896 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
14897 )
14898 .unwrap();
14899 handler
14900 .handle_control_frame(&module_ctx, rejection)
14901 .await
14902 .unwrap();
14903
14904 let response = route_task.await.unwrap();
14905 assert_eq!(parse_error(&response[0])["code"], hostile_code);
14906 let counters = handler.counters().snapshot();
14907 assert_eq!(
14908 counters["route_open_refused_by_code"],
14909 json!({ "module_rejected": 1 })
14910 );
14911 assert!(counters["route_open_refused_by_code"]
14912 .get(hostile_code)
14913 .is_none());
14914
14915 let event = capture
14916 .events()
14917 .into_iter()
14918 .find(|event| {
14919 event.target == "control"
14920 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
14921 })
14922 .expect("route.open module-rejection refusal event");
14923 let logged = event.fields.get("module_code").expect("module_code field");
14924 assert!(!logged.bytes().any(|byte| byte < 0x20));
14925 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
14926 }
14927
14928 #[tokio::test]
14929 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
14930 let registry = Arc::new(Registry::default());
14931 let supervisor_handle = SupervisorHandle::new();
14932 let missing_program = std::env::temp_dir().join(format!(
14933 "subc-route-open-missing-program-{}",
14934 std::process::id()
14935 ));
14936 let supervisor =
14937 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
14938 .with_handle(supervisor_handle.clone());
14939 let module = supervisor
14940 .supervise_configured(
14941 ModuleSpec {
14942 carry_containment: false,
14943 module_id: "failed".to_string(),
14944 program: missing_program,
14945 args: Vec::new(),
14946 env: Vec::new(),
14947 reserved: false,
14948 reserved_prefixes: Vec::new(),
14949 protocol: ModuleProtocol::Subc,
14950 overlap: Default::default(),
14951 },
14952 true,
14953 )
14954 .unwrap();
14955 assert_eq!(module.state().unwrap(), ModuleState::Failed);
14956
14957 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14958 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
14959 let response = handler
14960 .handle_control_frame(
14961 &ctx,
14962 route_open_frame(305, "failed", unique_project_root("failed")),
14963 )
14964 .await
14965 .unwrap();
14966
14967 assert_eq!(response[0].header.ty, FrameType::Error);
14968 let error = parse_error(&response[0]);
14969 assert_eq!(error["code"], "target_unavailable");
14970 assert!(error["message"]
14971 .as_str()
14972 .unwrap()
14973 .contains("state=failed, enabled=true, live=false"));
14974 }
14975
14976 #[tokio::test]
14977 async fn route_open_role_mismatch_remains_target_unavailable() {
14978 let registry = Arc::new(Registry::default());
14979 let handler = ControlHandler::new(Arc::clone(®istry));
14980 handler
14981 .handle_control(
14982 ConnectionId::new(41),
14983 non_routable_hello_frame_with_control_ops("health-only", 306, None),
14984 )
14985 .unwrap();
14986
14987 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
14988 let response = handler
14989 .handle_control_frame(
14990 &ctx,
14991 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
14992 )
14993 .await
14994 .unwrap();
14995
14996 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14997 assert!(parse_error(&response[0])["message"]
14998 .as_str()
14999 .unwrap()
15000 .contains("does not provide the requested target"));
15001 }
15002
15003 #[tokio::test]
15004 async fn route_open_inactive_registration_remains_target_unavailable() {
15005 let registry = Arc::new(Registry::default());
15006 let handler = ControlHandler::new(Arc::clone(®istry));
15007 handler
15008 .handle_control(
15009 ConnectionId::new(43),
15010 hello_frame("inactive", PROTOCOL_VERSION, 308),
15011 )
15012 .unwrap();
15013 assert!(registry
15014 .set_module_state_for_test("inactive", ChannelState::Closed)
15015 .unwrap());
15016
15017 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
15018 let response = handler
15019 .handle_control_frame(
15020 &ctx,
15021 route_open_frame(309, "inactive", unique_project_root("inactive")),
15022 )
15023 .await
15024 .unwrap();
15025
15026 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
15027 assert!(parse_error(&response[0])["message"]
15028 .as_str()
15029 .unwrap()
15030 .contains("is not active"));
15031 }
15032
15033 #[tokio::test]
15034 async fn late_health_reply_is_recorded_through_the_module_response_path() {
15035 let registry = Arc::new(Registry::default());
15036 let forwarding = Arc::new(ForwardingTable::default());
15037 let supervisor_handle = SupervisorHandle::new();
15038 let supervisor =
15039 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
15040 .with_forwarding(Arc::clone(&forwarding))
15041 .with_handle(supervisor_handle.clone());
15042 let module = supervisor
15043 .supervise_configured(
15044 crate::ModuleSpec {
15045 carry_containment: false,
15046 module_id: "late-health-response".to_string(),
15047 program: PathBuf::from("disabled-module"),
15048 args: Vec::new(),
15049 env: Vec::new(),
15050 reserved: false,
15051 reserved_prefixes: Vec::new(),
15052 protocol: ModuleProtocol::Subc,
15053 overlap: Default::default(),
15054 },
15055 false,
15056 )
15057 .unwrap();
15058 let handler =
15059 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15060 .with_supervisor(supervisor_handle);
15061 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
15062 handler
15063 .handle_control_frame(
15064 &module_ctx,
15065 hello_frame_with_control_ops(
15066 "late-health-response",
15067 PROTOCOL_VERSION,
15068 7,
15069 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15070 ),
15071 )
15072 .await
15073 .unwrap();
15074 let probe_started_at = Instant::now() - Duration::from_millis(80);
15075 let pending = forwarding
15076 .begin_health_probe_rpc_for(
15077 "late-health-response",
15078 MODULE_CONTROL_OP_HEALTH_CHECK,
15079 probe_started_at,
15080 Instant::now() - Duration::from_millis(1),
15081 )
15082 .unwrap();
15083 assert!(forwarding
15084 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
15085 .unwrap());
15086
15087 let responses = handler
15088 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
15089 .await
15090 .unwrap();
15091
15092 assert!(responses.is_empty());
15093 let health = module.status().unwrap().health;
15094 assert_eq!(health.late_answer_count, 1);
15095 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
15096 }
15097
15098 #[tokio::test]
15099 async fn health_probe_timeout_and_module_death_are_typed() {
15100 let registry = Arc::new(Registry::default());
15101 let forwarding = Arc::new(ForwardingTable::default());
15102 let handler =
15103 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15104 .with_health_probe_timeout(Duration::from_millis(50));
15105 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
15106 hello_via_sink(
15107 &handler,
15108 &module_ctx,
15109 &mut module_rx,
15110 hello_frame_with_control_ops(
15111 "aft",
15112 PROTOCOL_VERSION,
15113 7,
15114 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15115 ),
15116 )
15117 .await;
15118
15119 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
15120 let responses = handler
15121 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
15122 .await
15123 .unwrap();
15124 assert_eq!(responses[0].header.ty, FrameType::Error);
15125 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
15126 let _ = module_rx.try_recv();
15127
15128 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
15129 let health_handler = handler.clone();
15130 let death_task = tokio::spawn(async move {
15131 health_handler
15132 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
15133 .await
15134 .unwrap()
15135 });
15136 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15137 .await
15138 .unwrap()
15139 .unwrap();
15140 handler
15141 .cleanup_connection(module_ctx.connection_id)
15142 .unwrap();
15143 let responses = death_task.await.unwrap();
15144 assert_eq!(responses[0].header.ty, FrameType::Error);
15145 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
15146 }
15147
15148 #[test]
15149 fn hello_requires_exact_protocol_version() {
15150 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
15151 let registry = Arc::new(Registry::default());
15152 let handler = ControlHandler::new(Arc::clone(®istry));
15153 let responses = handler
15154 .handle_control(
15155 ConnectionId::new(connection),
15156 hello_frame("aft", offered, 9),
15157 )
15158 .unwrap();
15159
15160 assert_eq!(responses.len(), 1);
15161 assert_eq!(responses[0].header.ty, FrameType::Error);
15162 let error = parse_error(&responses[0]);
15163 assert_eq!(error["code"], "version_unsupported");
15164 assert!(registry.get_module("aft").unwrap().is_none());
15165 assert_eq!(registry.active_registration_count().unwrap(), 0);
15166 }
15167 }
15168
15169 #[test]
15170 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
15171 let registry = Arc::new(Registry::default());
15172 let forwarding = Arc::new(ForwardingTable::default());
15173 let handler =
15174 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15175 let module_connection = ConnectionId::new(301);
15176 let registration = registry
15177 .register_with_control_ops(
15178 manifest("aft-push", PROTOCOL_VERSION),
15179 PROTOCOL_VERSION,
15180 module_connection,
15181 module_baseline_control_ops(),
15182 )
15183 .unwrap();
15184 let (module_tx, _module_rx) = mpsc::channel(8);
15185 let endpoint = forwarding
15186 .register_module_connection(
15187 module_connection,
15188 "aft-push".to_string(),
15189 PROTOCOL_VERSION,
15190 manifest_concurrency(®istration.manifest),
15191 FrameSink::new(module_tx),
15192 )
15193 .unwrap();
15194
15195 let unknown = Frame::build(
15197 FrameType::Push,
15198 control_flags(),
15199 0,
15200 0,
15201 5,
15202 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
15203 )
15204 .unwrap();
15205 let out = handler.handle_status_update(endpoint, unknown).unwrap();
15206 assert!(
15207 out.is_empty(),
15208 "unknown push op must be ignored, got {out:?}"
15209 );
15210
15211 let malformed = Frame::build(
15213 FrameType::Push,
15214 control_flags(),
15215 0,
15216 0,
15217 6,
15218 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
15219 )
15220 .unwrap();
15221 let out = handler.handle_status_update(endpoint, malformed).unwrap();
15222 assert_eq!(out.len(), 1);
15223 assert_eq!(out[0].header.ty, FrameType::Error);
15224 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
15225 }
15226
15227 #[test]
15228 fn hello_rejected_when_connection_already_owns_client_routes() {
15229 let registry = Arc::new(Registry::default());
15230 let forwarding = Arc::new(ForwardingTable::default());
15231 let handler =
15232 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15233 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
15235 let client_connection = ConnectionId::new(202);
15236
15237 let responses = handler
15240 .handle_control(
15241 client_connection,
15242 hello_frame("aft-second", PROTOCOL_VERSION, 9),
15243 )
15244 .unwrap();
15245 assert_eq!(responses[0].header.ty, FrameType::Error);
15246 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
15247 assert!(registry.get_module("aft-second").unwrap().is_none());
15248 }
15249
15250 #[tokio::test]
15251 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
15252 let registry = Arc::new(Registry::default());
15253 let forwarding = Arc::new(ForwardingTable::default());
15254 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
15255 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
15256 hello_via_sink(
15257 &handler,
15258 &module_ctx,
15259 &mut module_rx,
15260 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
15261 )
15262 .await;
15263 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
15264 let pending = forwarding
15265 .begin_route_bind_relay_for_test(
15266 client_ctx.connection_id,
15267 client_ctx.egress.clone(),
15268 2,
15269 "alpha",
15270 )
15271 .unwrap();
15272 forwarding
15273 .complete_pending_relay(
15274 module_ctx.connection_id,
15275 pending.corr,
15276 RouteBindRelayOutcome::Accepted,
15277 )
15278 .unwrap();
15279 client_rx.try_recv().unwrap();
15280 for module_id in ["beta", "alpha"] {
15281 let replies = handler
15282 .handle_control_frame(
15283 &module_ctx,
15284 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
15285 )
15286 .await
15287 .unwrap();
15288 assert_eq!(replies.len(), 1, "second HELLO must be refused");
15289 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
15290 }
15291 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
15292 assert!(registry.get_module("beta").unwrap().is_none());
15293 assert!(matches!(
15294 forwarding
15295 .lookup_data_route(
15296 client_ctx.connection_id,
15297 pending.client_channel,
15298 pending.client_epoch,
15299 )
15300 .unwrap(),
15301 DataRoute::Client(DataRouteState::Bound(_))
15302 ));
15303 assert!(handler
15304 .hello_launch_nonces
15305 .lock()
15306 .unwrap()
15307 .presented(module_ctx.connection_id, Some("alpha-nonce")));
15308 assert!(module_rx.try_recv().is_err());
15309 }
15310
15311 #[test]
15312 fn reserved_module_hello_requires_matching_launch_nonce() {
15313 let registry = Arc::new(Registry::default());
15314 let supervisor = SupervisorHandle::new();
15315 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
15318 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
15319
15320 let no_nonce = handler
15322 .handle_control(
15323 ConnectionId::new(1),
15324 hello_frame("vault", PROTOCOL_VERSION, 1),
15325 )
15326 .unwrap();
15327 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
15328 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
15329 assert!(registry.get_module("vault").unwrap().is_none());
15330
15331 let wrong = handler
15333 .handle_control(
15334 ConnectionId::new(2),
15335 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
15336 )
15337 .unwrap();
15338 assert_eq!(wrong[0].header.ty, FrameType::Error);
15339 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
15340 assert!(registry.get_module("vault").unwrap().is_none());
15341
15342 let ok = handler
15344 .handle_control(
15345 ConnectionId::new(3),
15346 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
15347 )
15348 .unwrap();
15349 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
15350 assert!(registry.get_module("vault").unwrap().is_some());
15351 }
15352
15353 #[test]
15354 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
15355 let registry = Arc::new(Registry::default());
15356 let supervisor = SupervisorHandle::new();
15357 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
15358 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
15359 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
15360
15361 let squat = handler
15362 .handle_control(
15363 ConnectionId::new(1),
15364 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
15365 )
15366 .unwrap();
15367 assert_eq!(squat[0].header.ty, FrameType::Error);
15368 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
15369 assert!(parse_error(&squat[0])["message"]
15370 .as_str()
15371 .unwrap()
15372 .contains("fed:"));
15373
15374 let accepted_peer = handler
15375 .handle_control(
15376 ConnectionId::new(2),
15377 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
15378 )
15379 .unwrap();
15380 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
15381
15382 let accepted_short = handler
15383 .handle_control(
15384 ConnectionId::new(3),
15385 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
15386 )
15387 .unwrap();
15388 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
15389
15390 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
15391 let response = handler
15392 .handle_control(
15393 ConnectionId::new(conn),
15394 hello_frame(module_id, PROTOCOL_VERSION, conn),
15395 )
15396 .unwrap();
15397 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
15398 }
15399 }
15400
15401 #[test]
15402 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
15403 let registry = Arc::new(Registry::default());
15404 let supervisor = SupervisorHandle::new();
15405 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
15406 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
15407 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
15408 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
15409
15410 let owner_nonce = handler
15411 .handle_control(
15412 ConnectionId::new(1),
15413 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
15414 )
15415 .unwrap();
15416 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
15417 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
15418 assert!(registry.get_module("fed:special").unwrap().is_none());
15419
15420 let exact_nonce = handler
15421 .handle_control(
15422 ConnectionId::new(2),
15423 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
15424 )
15425 .unwrap();
15426 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
15427 assert!(registry.get_module("fed:special").unwrap().is_some());
15428 }
15429
15430 #[test]
15431 fn non_reserved_module_ignores_launch_nonce() {
15432 let registry = Arc::new(Registry::default());
15433 let handler = ControlHandler::new(Arc::clone(®istry));
15436 let no_nonce = handler
15437 .handle_control(
15438 ConnectionId::new(1),
15439 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
15440 )
15441 .unwrap();
15442 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
15443 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
15444
15445 let echoed_nonce = handler
15446 .handle_control(
15447 ConnectionId::new(2),
15448 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
15449 )
15450 .unwrap();
15451 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
15452 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
15453 }
15454
15455 #[test]
15456 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
15457 let handler = ControlHandler::default();
15458 let conn = ConnectionId::new(1);
15459 let malformed = Frame::build(
15460 FrameType::Hello,
15461 control_flags(),
15462 0,
15463 0,
15464 3,
15465 b"{not json".to_vec(),
15466 )
15467 .unwrap();
15468
15469 let error = handler.handle_control(conn, malformed).unwrap();
15470 assert_eq!(error[0].header.ty, FrameType::Error);
15471 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
15472
15473 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
15474 let pong = handler.handle_control(conn, ping).unwrap();
15475 assert_eq!(pong[0].header.ty, FrameType::Pong);
15476 assert_eq!(pong[0].header.corr, 4);
15477 }
15478
15479 #[test]
15480 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
15481 let registry = Arc::new(Registry::default());
15482 let handler = ControlHandler::new(Arc::clone(®istry));
15483
15484 handler
15485 .handle_control(
15486 ConnectionId::new(1),
15487 hello_frame("aft", PROTOCOL_VERSION, 1),
15488 )
15489 .unwrap();
15490 let duplicate = handler
15491 .handle_control(
15492 ConnectionId::new(2),
15493 hello_frame("aft", PROTOCOL_VERSION, 2),
15494 )
15495 .unwrap();
15496
15497 assert_eq!(duplicate[0].header.ty, FrameType::Error);
15498 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15499 let registration = registry.get_module("aft").unwrap().unwrap();
15500 assert_eq!(registration.connection_id, ConnectionId::new(1));
15501 }
15502
15503 #[test]
15504 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
15505 let registry = Arc::new(Registry::default());
15506 let forwarding = Arc::new(ForwardingTable::default());
15507 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
15508 let handler =
15509 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15510 .with_process_liveness(process_liveness);
15511 let (ctx, route_channel, route_epoch) =
15512 bind_liveness_route(®istry, &forwarding, "aft-dead");
15513 let responses = handler
15514 .handle_route_poll(
15515 &ctx,
15516 route_poll_frame(41, PollKind::Liveness, route_channel),
15517 route_channel,
15518 route_epoch,
15519 PollKind::Liveness,
15520 )
15521 .unwrap();
15522
15523 assert_eq!(responses.len(), 1);
15524 assert_eq!(responses[0].header.ty, FrameType::Response);
15525 assert_route_poll_liveness(&responses[0], false);
15526 }
15527
15528 #[test]
15529 fn liveness_poll_without_process_source_uses_bound_route() {
15530 let registry = Arc::new(Registry::default());
15531 let forwarding = Arc::new(ForwardingTable::default());
15532 let handler =
15533 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15534 let (ctx, route_channel, route_epoch) =
15535 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
15536 let responses = handler
15537 .handle_route_poll(
15538 &ctx,
15539 route_poll_frame(42, PollKind::Liveness, route_channel),
15540 route_channel,
15541 route_epoch,
15542 PollKind::Liveness,
15543 )
15544 .unwrap();
15545
15546 assert_route_poll_liveness(&responses[0], true);
15547 }
15548
15549 #[test]
15550 fn liveness_poll_untracked_process_source_uses_bound_route() {
15551 let registry = Arc::new(Registry::default());
15552 let forwarding = Arc::new(ForwardingTable::default());
15553 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
15554 let handler =
15555 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15556 .with_process_liveness(process_liveness);
15557 let (ctx, route_channel, route_epoch) =
15558 bind_liveness_route(®istry, &forwarding, "aft-untracked");
15559 let responses = handler
15560 .handle_route_poll(
15561 &ctx,
15562 route_poll_frame(43, PollKind::Liveness, route_channel),
15563 route_channel,
15564 route_epoch,
15565 PollKind::Liveness,
15566 )
15567 .unwrap();
15568
15569 assert_route_poll_liveness(&responses[0], true);
15570 }
15571
15572 #[tokio::test]
15573 async fn unknown_op_returns_unknown_control_op() {
15574 let handler = ControlHandler::default();
15575 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
15576 let request = Frame::build(
15577 FrameType::Request,
15578 control_flags(),
15579 0,
15580 0,
15581 55,
15582 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
15583 )
15584 .unwrap();
15585
15586 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15587
15588 assert_eq!(response.len(), 1);
15589 assert_eq!(response[0].header.ty, FrameType::Error);
15590 assert_eq!(response[0].header.corr, 55);
15591 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
15592 }
15593
15594 #[tokio::test]
15595 async fn supervisor_provenance_rejects_unknown_exact_module() {
15596 let handler = ControlHandler::default();
15597 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
15598 let request = Frame::build(
15599 FrameType::Request,
15600 control_flags(),
15601 0,
15602 0,
15603 57,
15604 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
15605 )
15606 .unwrap();
15607
15608 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15609
15610 assert_eq!(response.len(), 1);
15611 assert_eq!(response[0].header.ty, FrameType::Error);
15612 assert_eq!(response[0].header.corr, 57);
15613 let error = parse_error(&response[0]);
15614 assert_eq!(error["code"], "unknown_module");
15615 assert_eq!(error["message"], "module_id 'missing' is not supervised");
15616 }
15617
15618 #[test]
15619 fn provenance_probe_override_keeps_handler_tests_deterministic() {
15620 let expected = subc_control::RunningImageAgreement::Unavailable {
15621 reason: subc_control::RunningImageUnavailableReason::HashFailed,
15622 };
15623 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
15624 assert_eq!(handler.provenance_probe_override, Some(expected));
15625 }
15626
15627 #[test]
15628 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
15629 let verdict = reload_verdict(
15630 std::path::Path::new("/bin/new"),
15631 Some(std::path::Path::new("/bin/old")),
15632 subc_control::RunningImageAgreement::Unavailable {
15633 reason: subc_control::RunningImageUnavailableReason::HashFailed,
15634 },
15635 );
15636 assert!(matches!(
15637 verdict.path,
15638 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
15639 if configured == std::path::Path::new("/bin/new")
15640 && spawned_from == std::path::Path::new("/bin/old")
15641 ));
15642 }
15643
15644 #[test]
15645 fn launch_spec_without_running_digest_is_unavailable_not_match() {
15646 use subc_control::{
15647 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
15648 };
15649 let configured = crate::supervise::ModuleSpec {
15650 carry_containment: false,
15651 module_id: "legacy".into(),
15652 program: "/module".into(),
15653 args: Vec::new(),
15654 env: Vec::new(),
15655 reserved: false,
15656 reserved_prefixes: Vec::new(),
15657 protocol: subc_control::ModuleProtocol::Subc,
15658 overlap: Default::default(),
15659 };
15660 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
15661 assert_eq!(
15662 launch_spec_agreement(&configured, false, Some(&digest)),
15663 Agreement::Unavailable {
15664 reason: Reason::NotRunning
15665 }
15666 );
15667 assert_eq!(
15668 launch_spec_agreement(&configured, true, None),
15669 Agreement::Unavailable {
15670 reason: Reason::SpawnedSpecUnavailable
15671 }
15672 );
15673 assert_eq!(
15674 launch_spec_agreement(&configured, true, Some(&digest)),
15675 Agreement::Match
15676 );
15677 }
15678
15679 #[cfg(windows)]
15680 #[test]
15681 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
15682 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
15683 let configured = dir.join("module.exe");
15684 std::fs::write(&configured, b"executable path fixture").unwrap();
15685 let spawned = std::fs::canonicalize(&configured).unwrap();
15686 assert_ne!(
15687 configured, spawned,
15688 "fixture must exercise different path spellings"
15689 );
15690 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
15691 let lower_case = configured.to_string_lossy().to_lowercase();
15692 for spelling in [
15693 &configured,
15694 Path::new(&forward_slashes),
15695 Path::new(&lower_case),
15696 ] {
15697 let verdict = reload_verdict(
15698 spelling,
15699 Some(&spawned),
15700 subc_control::RunningImageAgreement::Unavailable {
15701 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15702 },
15703 );
15704 assert_eq!(
15705 verdict.path,
15706 ReloadPathAgreement::Match,
15707 "configured {spelling:?}, spawned {spawned:?}"
15708 );
15709 }
15710 }
15711
15712 #[cfg(windows)]
15713 #[test]
15714 fn windows_reload_path_keeps_different_existing_program_pending() {
15715 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
15716 let old = dir.join("old.exe");
15717 let new = dir.join("new.exe");
15718 std::fs::write(&old, b"same executable content").unwrap();
15719 std::fs::copy(&old, &new).unwrap();
15720 let spawned = std::fs::canonicalize(&old).unwrap();
15721 let image = subc_control::RunningImageAgreement::Unavailable {
15722 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15723 };
15724 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
15725 assert!(
15726 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
15727 if configured == new && spawned_from == spawned)
15728 );
15729 let missing = dir.join("missing.exe");
15730 assert!(matches!(
15731 reload_verdict(&missing, Some(&spawned), image).path,
15732 ReloadPathAgreement::Mismatch { .. }
15733 ));
15734 }
15735
15736 #[test]
15737 fn reload_verdict_detects_replaced_image_at_same_path() {
15738 let image = subc_control::RunningImageAgreement::Mismatch {
15739 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
15740 digest: "old".into(),
15741 },
15742 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
15743 digest: "new".into(),
15744 },
15745 };
15746 let verdict = reload_verdict(
15747 std::path::Path::new("/bin/same"),
15748 Some(std::path::Path::new("/bin/same")),
15749 image.clone(),
15750 );
15751 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15752 assert_eq!(verdict.image, image);
15753 }
15754
15755 #[test]
15756 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
15757 let image = subc_control::RunningImageAgreement::Unavailable {
15758 reason: subc_control::RunningImageUnavailableReason::NotRunning,
15759 };
15760 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
15761 assert_eq!(
15762 verdict.path,
15763 subc_control::ReloadPathAgreement::Unavailable {
15764 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
15765 }
15766 );
15767 assert_eq!(verdict.image, image);
15768
15769 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
15770 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15771 };
15772 let verdict = reload_verdict(
15773 std::path::Path::new("/bin/same"),
15774 Some(std::path::Path::new("/bin/same")),
15775 unconfirmed.clone(),
15776 );
15777 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15778 assert_eq!(verdict.image, unconfirmed);
15779 }
15780
15781 #[test]
15782 fn reload_verdict_preserves_each_image_unavailability_reason() {
15783 use subc_control::RunningImageUnavailableReason as Reason;
15784
15785 for reason in [
15786 Reason::NotRunning,
15787 Reason::UnsupportedPlatform,
15788 Reason::RunningExecutableUnreadable,
15789 Reason::SpawnedPathUnreadable,
15790 Reason::HashFailed,
15791 Reason::ProcessIdentityUnconfirmed,
15792 Reason::Unknown("future_probe_reason".to_string()),
15793 ] {
15794 let image = subc_control::RunningImageAgreement::Unavailable {
15795 reason: reason.clone(),
15796 };
15797 let verdict = reload_verdict(
15798 std::path::Path::new("/bin/same"),
15799 Some(std::path::Path::new("/bin/same")),
15800 image.clone(),
15801 );
15802 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15803 assert_eq!(verdict.image, image, "{reason:?}");
15804 }
15805 }
15806
15807 #[tokio::test]
15808 async fn malformed_control_bodies_return_invalid_control_body() {
15809 let handler = ControlHandler::default();
15810 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
15811
15812 for (corr, body) in [
15813 (56, br#"{"route_channel":1}"#.as_slice()),
15814 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
15815 (
15816 58,
15817 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
15818 ),
15819 ] {
15820 let request = Frame::build(
15821 FrameType::Request,
15822 control_flags(),
15823 0,
15824 0,
15825 corr,
15826 body.to_vec(),
15827 )
15828 .unwrap();
15829 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15830
15831 assert_eq!(response.len(), 1);
15832 assert_eq!(response[0].header.ty, FrameType::Error);
15833 assert_eq!(response[0].header.corr, corr);
15834 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
15835 }
15836 }
15837
15838 #[tokio::test]
15839 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
15840 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15841 let registry = Arc::new(Registry::default());
15842 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
15843 let router = Router::with_control_handler(Arc::clone(&control));
15844 let connection = router.begin_connection();
15845 let (ctx, mut rx) = route_ctx(connection.id());
15846
15847 router
15848 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
15849 .await
15850 .unwrap();
15851 let response = rx.recv().await.unwrap();
15852 let ack = parse_ack(&response);
15853 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
15854 let channel = 1;
15855
15856 let goodbye =
15857 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
15858 router.route_for_connection(&ctx, goodbye).await.unwrap();
15859 assert!(rx.try_recv().is_err());
15860 assert!(registry.get_module("aft").unwrap().is_none());
15861
15862 router
15863 .route_for_connection(&ctx, channel_request(channel, 13))
15864 .await
15865 .unwrap();
15866 let error_frame = rx.recv().await.unwrap();
15867 assert_eq!(error_frame.header.ty, FrameType::Error);
15868 assert_eq!(error_frame.header.channel, channel);
15869 let captured = crate::router::test_log::captured_logs(&logs);
15870 assert_eq!(
15871 captured
15872 .lines()
15873 .filter(|line| {
15874 line.contains("module_id=aft")
15875 && line.contains("reason=explicit_goodbye")
15876 && line.contains("module registration ended")
15877 })
15878 .count(),
15879 1,
15880 "unexpected GOODBYE registry log: {captured}"
15881 );
15882 }
15883
15884 #[tokio::test]
15885 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
15886 let registry = Arc::new(Registry::default());
15887 let handler = ControlHandler::new(registry).with_capability_config(
15888 [("prov".to_string(), true), ("cons".to_string(), true)],
15889 BTreeMap::new(),
15890 );
15891 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
15892 register_capability_manifest(
15893 &handler,
15894 &provider_ctx,
15895 &mut provider_rx,
15896 capability_manifest("prov", &["thing/v1"], &[]),
15897 1,
15898 )
15899 .await;
15900 let mut consumer = capability_manifest("cons", &[], &[]);
15901 consumer.capabilities.as_mut().unwrap().requires.push(
15902 subc_protocol::manifest::CapabilityRequirement {
15903 capability: "thing/v1".to_string(),
15904 need: subc_protocol::manifest::CapabilityNeed::Required,
15905 },
15906 );
15907 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
15908 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
15909 assert_eq!(
15910 handler.capability_evaluator.verdict("cons", "thing/v1"),
15911 Some(CapabilityVerdict::Provided)
15912 );
15913 let (mut client_rx, _) = open_route_for_capability_test(
15914 &handler,
15915 &provider_ctx,
15916 &mut provider_rx,
15917 703,
15918 3,
15919 "prov",
15920 None,
15921 )
15922 .await;
15923 let goodbye =
15924 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
15925 handler
15926 .handle_control_frame(&provider_ctx, goodbye)
15927 .await
15928 .unwrap();
15929 assert_eq!(
15930 handler.capability_evaluator.verdict("cons", "thing/v1"),
15931 Some(CapabilityVerdict::NeverProvided)
15932 );
15933 let closed = client_rx
15934 .try_recv()
15935 .expect("GOODBYE pushes route.closed before route GOODBYE");
15936 assert!(
15937 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
15938 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
15939 );
15940 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
15941 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
15942 }
15943
15944 struct RouteClosedFixture {
15945 handler: ControlHandler,
15946 #[cfg(unix)]
15947 supervisor: Supervisor,
15948 module_ctx: RouteCtx,
15949 #[cfg(unix)]
15950 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15951 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15952 route: Arc<crate::forwarding::RouteBinding>,
15953 }
15954
15955 async fn route_closed_fixture() -> RouteClosedFixture {
15956 let registry = Arc::new(Registry::default());
15957 let handler = ControlHandler::new(Arc::clone(®istry));
15958 #[cfg(unix)]
15959 let supervisor = Supervisor::new(registry, RestartPolicy::default())
15960 .with_forwarding(Arc::clone(&handler.forwarding));
15961 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
15962 register_capability_manifest(
15963 &handler,
15964 &module_ctx,
15965 &mut module_rx,
15966 capability_manifest("prov", &[], &[]),
15967 1,
15968 )
15969 .await;
15970 let (client_rx, bind) = open_route_for_capability_test(
15971 &handler,
15972 &module_ctx,
15973 &mut module_rx,
15974 702,
15975 2,
15976 "prov",
15977 None,
15978 )
15979 .await;
15980 let ModuleControlRequest::RouteBind {
15981 route_channel: channel,
15982 epoch: route_epoch,
15983 ..
15984 } = bind
15985 else {
15986 panic!("expected route.bind");
15987 };
15988 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
15989 handler
15990 .forwarding
15991 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
15992 .unwrap()
15993 else {
15994 panic!("the acknowledged route must be bound");
15995 };
15996 RouteClosedFixture {
15997 handler,
15998 #[cfg(unix)]
15999 supervisor,
16000 module_ctx,
16001 #[cfg(unix)]
16002 module_rx,
16003 client_rx,
16004 route,
16005 }
16006 }
16007
16008 impl RouteClosedFixture {
16009 #[cfg(unix)]
16010 async fn cut(&mut self) {
16011 let notices = async {
16012 let mut draining = self.module_rx.recv().await.unwrap();
16013 assert!(matches!(
16014 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
16015 &draining.body
16016 )
16017 .unwrap(),
16018 subc_protocol::session::ModuleControlCommand::Draining {
16019 reason: RouteCloseReason::Disable,
16020 ..
16021 }
16022 ));
16023 draining.flushed.take().unwrap().send(()).unwrap();
16024 let mut closing = self.client_rx.recv().await.unwrap();
16025 assert!(matches!(
16026 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
16027 ClientControlPush::RouteClosing {
16028 reason: RouteCloseReason::Restart,
16029 ..
16030 }
16031 ));
16032 closing.flushed.take().unwrap().send(()).unwrap();
16033 };
16034 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
16035 result.unwrap();
16036 }
16037
16038 fn disconnect(&mut self) -> ClientControlPush {
16039 self.handler
16040 .cleanup_connection(self.module_ctx.connection_id)
16041 .unwrap();
16042 let closed = self.client_rx.try_recv().unwrap();
16043 assert_eq!(closed.header.ty, FrameType::Push);
16044 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
16045 assert!(matches!(
16046 &push,
16047 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
16048 if channels == &[self.route.client_channel]
16049 ));
16050 assert_eq!(
16051 self.client_rx.try_recv().unwrap().header.ty,
16052 FrameType::Goodbye
16053 );
16054 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
16055 push
16056 }
16057 }
16058
16059 #[cfg(unix)]
16060 #[tokio::test(start_paused = true)]
16061 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
16062 let mut fixture = route_closed_fixture().await;
16063 fixture.cut().await;
16064 assert!(matches!(
16065 fixture.disconnect(),
16066 ClientControlPush::RouteClosed {
16067 reason: RouteCloseReason::Restart,
16068 drained: true,
16069 ..
16070 }
16071 ));
16072 }
16073
16074 #[cfg(unix)]
16075 #[tokio::test(start_paused = true)]
16076 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
16077 let mut fixture = route_closed_fixture().await;
16078 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
16079 let started = tokio::time::Instant::now();
16080 fixture.cut().await;
16081 assert!(started.elapsed() >= Duration::from_secs(2));
16082 assert_eq!(fixture.route.flow.in_flight(), 1);
16083 assert!(matches!(
16084 fixture.disconnect(),
16085 ClientControlPush::RouteClosed {
16086 reason: RouteCloseReason::Restart,
16087 drained: false,
16088 ..
16089 }
16090 ));
16091 }
16092
16093 #[tokio::test]
16094 async fn module_connection_crash_reports_crash_and_not_drained() {
16095 let mut fixture = route_closed_fixture().await;
16096 assert!(matches!(
16100 fixture.disconnect(),
16101 ClientControlPush::RouteClosed {
16102 reason: RouteCloseReason::Crash,
16103 drained: false,
16104 ..
16105 }
16106 ));
16107 }
16108
16109 #[tokio::test]
16110 async fn dropping_router_connection_releases_registration() {
16111 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16112 let registry = Arc::new(Registry::default());
16113 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
16114 let router = Router::with_control_handler(Arc::clone(&control));
16115 let connection = router.begin_connection();
16116 let connection_id = connection.id();
16117 let (ctx, mut rx) = route_ctx(connection_id);
16118
16119 router
16120 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
16121 .await
16122 .unwrap();
16123 let response = rx.recv().await.unwrap();
16124 let ack = parse_ack(&response);
16125 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
16126 assert!(registry.get_module("aft").unwrap().is_some());
16127
16128 drop(connection);
16129
16130 assert!(registry.get_module("aft").unwrap().is_none());
16131 assert_eq!(registry.active_registration_count().unwrap(), 0);
16132
16133 control
16134 .cleanup_connection(ConnectionId::new(u64::MAX))
16135 .unwrap();
16136 let captured = crate::router::test_log::captured_logs(&logs);
16137 println!("captured registry lifecycle logs:\n{captured}");
16138 let events: Vec<_> = captured
16139 .lines()
16140 .filter(|line| {
16141 line.contains("module registered module_id=aft ")
16142 || line.contains("module registration ended")
16143 })
16144 .collect();
16145 assert_eq!(
16146 events.len(),
16147 2,
16148 "unexpected registry lifecycle logs: {captured}"
16149 );
16150 assert!(events[0].contains("module registered module_id=aft "));
16151 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
16152 assert!(events[1].contains(&format!(
16153 "module_id=aft connection_id={}",
16154 connection_id.get()
16155 )));
16156 assert!(events[1].contains("reason=connection_closed"));
16157 assert!(events[1].contains("module registration ended"));
16158 assert_eq!(
16159 captured
16160 .lines()
16161 .filter(|line| line.contains("module registered module_id=aft "))
16162 .count(),
16163 1,
16164 "legacy registration admission line must appear once: {captured}"
16165 );
16166 }
16167
16168 #[tokio::test]
16169 async fn hello_registration_keeps_legacy_module_registered_line_once() {
16170 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16171 let registry = Arc::new(Registry::default());
16172 let control = ControlHandler::new(Arc::clone(®istry));
16173 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
16174 hello_via_sink(
16175 &control,
16176 &ctx,
16177 &mut rx,
16178 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
16179 )
16180 .await;
16181
16182 let captured = crate::router::test_log::captured_logs(&logs);
16183 let admissions: Vec<_> = captured
16184 .lines()
16185 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
16186 .collect();
16187 assert_eq!(
16188 admissions.len(),
16189 1,
16190 "legacy admission line must remain exactly once: {captured}"
16191 );
16192 assert!(admissions[0].contains("routable_provider=true"));
16193 assert!(admissions[0].contains("connection_id=777"));
16194 }
16195
16196 fn capability_manifest(
16197 module_id: &str,
16198 provides: &[&str],
16199 must_never_reach: &[&str],
16200 ) -> ModuleManifest {
16201 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
16202 manifest.capabilities = Some(CapabilityDeclarations {
16203 provides: provides
16204 .iter()
16205 .map(|capability| (*capability).to_string())
16206 .collect(),
16207 requires: Vec::new(),
16208 must_never_reach: must_never_reach
16209 .iter()
16210 .map(|capability| (*capability).to_string())
16211 .collect(),
16212 });
16213 manifest
16214 }
16215
16216 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
16217 Frame::build(
16218 FrameType::Hello,
16219 control_flags(),
16220 0,
16221 0,
16222 corr,
16223 serde_json::to_vec(&ModuleHelloBody {
16224 protocol_ver: manifest.protocol_ver,
16225 manifest,
16226 control_ops: None,
16227 launch_nonce: None,
16228 })
16229 .expect("capability test HELLO serializes"),
16230 )
16231 .expect("capability test HELLO frame builds")
16232 }
16233
16234 fn catalog_update_with_capabilities_frame(
16235 corr: u64,
16236 capabilities: CapabilityDeclarations,
16237 ) -> Frame {
16238 Frame::build(
16239 FrameType::Request,
16240 control_flags(),
16241 0,
16242 0,
16243 corr,
16244 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
16245 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
16246 capabilities: Some(capabilities),
16247 ready: None,
16248 })
16249 .expect("capability catalog.update serializes"),
16250 )
16251 .expect("capability catalog.update frame builds")
16252 }
16253
16254 async fn register_capability_manifest(
16255 handler: &ControlHandler,
16256 ctx: &RouteCtx,
16257 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16258 manifest: ModuleManifest,
16259 corr: u64,
16260 ) {
16261 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
16262 }
16263
16264 async fn open_route_for_capability_test(
16265 handler: &ControlHandler,
16266 target_ctx: &RouteCtx,
16267 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16268 client_connection_id: u64,
16269 corr: u64,
16270 target_module_id: &str,
16271 consumer_identity: Option<ConsumerIdentity>,
16272 ) -> (
16273 mpsc::Receiver<crate::router::OutboundFrame>,
16274 ModuleControlRequest,
16275 ) {
16276 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
16277 let route_handler = handler.clone();
16278 let target_module_id = target_module_id.to_string();
16279 let route_task = tokio::spawn(async move {
16280 route_handler
16281 .handle_control_frame(
16282 &client_ctx,
16283 route_open_frame_with_admission_facts(
16284 corr,
16285 &target_module_id,
16286 unique_project_root("admission-facts"),
16287 consumer_identity,
16288 None,
16289 ),
16290 )
16291 .await
16292 .expect("capability test route.open succeeds")
16293 });
16294 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
16295 .await
16296 .expect("capability test route.open must reach route.bind")
16297 .expect("target control receiver stays open");
16298 let bind_request: ModuleControlRequest =
16299 serde_json::from_slice(&bind.body).expect("route.bind decodes");
16300 handler
16301 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
16302 .await
16303 .expect("capability test route.bind ACK succeeds");
16304 assert!(route_task.await.expect("route.open task joins").is_empty());
16305 let opened = client_rx
16306 .recv()
16307 .await
16308 .expect("successful route.open publishes a response");
16309 assert!(matches!(
16310 serde_json::from_slice::<ClientControlResponse>(&opened.body),
16311 Ok(ClientControlResponse::RouteOpen { .. })
16312 ));
16313 (client_rx, bind_request)
16314 }
16315
16316 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
16317 assert_eq!(frame.header.ty, FrameType::Push);
16318 assert_eq!(frame.header.channel, 0);
16319 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
16320 .expect("route.closed control push decodes");
16321 let ClientControlPush::RouteClosed { channels, .. } = &push else {
16322 panic!("expected route.closed");
16323 };
16324 assert_eq!(channels.len(), 1, "exactly one violating route closed");
16325 let closed_channel = channels[0];
16326 let channels = channels.clone();
16327 assert_eq!(
16328 push,
16329 ClientControlPush::RouteClosed {
16330 module_id: target_module_id.to_string(),
16331 channels,
16332 reason: RouteCloseReason::CapabilityDenied,
16333 drained: false,
16334 abandoned: 0,
16335 excluded_subscriptions: 0,
16336 terminal: Some(false),
16337 }
16338 );
16339 closed_channel
16340 }
16341
16342 #[tokio::test]
16343 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
16344 let registry = Arc::new(Registry::default());
16345 let forwarding = Arc::new(ForwardingTable::default());
16346 let supervisor = SupervisorHandle::new();
16347 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16348 let handler =
16349 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16350 .with_supervisor(supervisor);
16351 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
16352 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
16353 register_capability_manifest(
16354 &handler,
16355 &target_ctx,
16356 &mut target_rx,
16357 capability_manifest("target", &["credentials-provider/v1"], &[]),
16358 1,
16359 )
16360 .await;
16361 register_capability_manifest(
16362 &handler,
16363 &opener_ctx,
16364 &mut opener_rx,
16365 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16366 2,
16367 )
16368 .await;
16369
16370 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
16371 let replies = handler
16372 .handle_control_frame(
16373 &client_ctx,
16374 route_open_frame_with_admission_facts(
16375 3,
16376 "target",
16377 unique_project_root("admission-facts"),
16378 Some(ConsumerIdentity {
16379 module_id: "opener".to_string(),
16380 launch_nonce: "opener-nonce".to_string(),
16381 }),
16382 None,
16383 ),
16384 )
16385 .await
16386 .expect("denied route.open returns a typed frame");
16387 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
16388 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16389 assert!(
16390 target_rx.try_recv().is_err(),
16391 "forbidden route.open must not relay route.bind"
16392 );
16393 }
16394
16395 #[tokio::test]
16396 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
16397 let registry = Arc::new(Registry::default());
16398 let forwarding = Arc::new(ForwardingTable::default());
16399 let supervisor = SupervisorHandle::new();
16400 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16401 let handler =
16402 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16403 .with_supervisor(supervisor);
16404 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
16405 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
16406 register_capability_manifest(
16407 &handler,
16408 &target_ctx,
16409 &mut target_rx,
16410 capability_manifest("target", &["credentials-provider/v1"], &[]),
16411 1,
16412 )
16413 .await;
16414 register_capability_manifest(
16415 &handler,
16416 &old_opener_ctx,
16417 &mut old_opener_rx,
16418 capability_manifest("opener", &[], &[]),
16419 2,
16420 )
16421 .await;
16422 let (mut client_rx, _) = open_route_for_capability_test(
16423 &handler,
16424 &target_ctx,
16425 &mut target_rx,
16426 712,
16427 3,
16428 "target",
16429 Some(ConsumerIdentity {
16430 module_id: "opener".to_string(),
16431 launch_nonce: "opener-nonce".to_string(),
16432 }),
16433 )
16434 .await;
16435 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16436
16437 handler
16438 .cleanup_connection(old_opener_ctx.connection_id)
16439 .expect("old opener registration cleans up");
16440 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
16441 register_capability_manifest(
16442 &handler,
16443 &new_opener_ctx,
16444 &mut new_opener_rx,
16445 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16446 4,
16447 )
16448 .await;
16449
16450 let closed_channel = assert_capability_denied_push(
16451 client_rx
16452 .try_recv()
16453 .expect("HELLO deny addition must emit route.closed")
16454 .frame,
16455 "target",
16456 );
16457 let goodbye = client_rx
16458 .try_recv()
16459 .expect("HELLO deny addition must send the client a route GOODBYE")
16460 .frame;
16461 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16462 assert_eq!(goodbye.header.channel, closed_channel);
16463 assert_ne!(goodbye.header.channel, 0);
16464 assert_ne!(goodbye.header.epoch, 0);
16465 assert!(client_rx.try_recv().is_err());
16466 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16467 assert!(matches!(
16468 target_rx.try_recv(),
16469 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
16470 ));
16471 }
16472
16473 #[tokio::test]
16474 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
16475 let registry = Arc::new(Registry::default());
16476 let forwarding = Arc::new(ForwardingTable::default());
16477 let supervisor = SupervisorHandle::new();
16478 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16479 let handler =
16480 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16481 .with_supervisor(supervisor);
16482 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
16483 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
16484 register_capability_manifest(
16485 &handler,
16486 &target_ctx,
16487 &mut target_rx,
16488 capability_manifest("target", &[], &[]),
16489 1,
16490 )
16491 .await;
16492 register_capability_manifest(
16493 &handler,
16494 &opener_ctx,
16495 &mut opener_rx,
16496 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16497 2,
16498 )
16499 .await;
16500 let (mut client_rx, _) = open_route_for_capability_test(
16501 &handler,
16502 &target_ctx,
16503 &mut target_rx,
16504 722,
16505 3,
16506 "target",
16507 Some(ConsumerIdentity {
16508 module_id: "opener".to_string(),
16509 launch_nonce: "opener-nonce".to_string(),
16510 }),
16511 )
16512 .await;
16513 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16514
16515 let replies = handler
16516 .handle_control_frame(
16517 &target_ctx,
16518 catalog_update_with_capabilities_frame(
16519 4,
16520 CapabilityDeclarations {
16521 provides: vec!["credentials-provider/v1".to_string()],
16522 requires: Vec::new(),
16523 must_never_reach: Vec::new(),
16524 },
16525 ),
16526 )
16527 .await
16528 .expect("claim catalog.update succeeds");
16529 assert!(matches!(
16530 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
16531 Ok(ModuleControlResponseToModule::CatalogUpdate {})
16532 ));
16533 let closed_channel = assert_capability_denied_push(
16534 client_rx
16535 .try_recv()
16536 .expect("claim addition must emit route.closed")
16537 .frame,
16538 "target",
16539 );
16540 let goodbye = client_rx
16541 .try_recv()
16542 .expect("claim addition must send the client a route GOODBYE")
16543 .frame;
16544 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16545 assert_eq!(goodbye.header.channel, closed_channel);
16546 assert_ne!(goodbye.header.channel, 0);
16547 assert_ne!(goodbye.header.epoch, 0);
16548 assert!(client_rx.try_recv().is_err());
16549 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16550 assert!(matches!(
16551 target_rx.try_recv(),
16552 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
16553 ));
16554 }
16555
16556 #[tokio::test]
16557 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
16558 let registry = Arc::new(Registry::default());
16559 let forwarding = Arc::new(ForwardingTable::default());
16560 let supervisor = SupervisorHandle::new();
16561 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16562 let handler =
16563 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16564 .with_supervisor(supervisor);
16565 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
16566 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
16567 register_capability_manifest(
16568 &handler,
16569 &target_ctx,
16570 &mut target_rx,
16571 capability_manifest("target", &["credentials-provider/v1"], &[]),
16572 1,
16573 )
16574 .await;
16575 register_capability_manifest(
16576 &handler,
16577 &opener_ctx,
16578 &mut opener_rx,
16579 capability_manifest("opener", &[], &[]),
16580 2,
16581 )
16582 .await;
16583 let (mut client_rx, _) = open_route_for_capability_test(
16584 &handler,
16585 &target_ctx,
16586 &mut target_rx,
16587 732,
16588 3,
16589 "target",
16590 Some(ConsumerIdentity {
16591 module_id: "opener".to_string(),
16592 launch_nonce: "opener-nonce".to_string(),
16593 }),
16594 )
16595 .await;
16596 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16597
16598 handler
16599 .handle_control_frame(
16600 &target_ctx,
16601 catalog_update_with_capabilities_frame(
16602 4,
16603 CapabilityDeclarations {
16604 provides: Vec::new(),
16605 requires: Vec::new(),
16606 must_never_reach: Vec::new(),
16607 },
16608 ),
16609 )
16610 .await
16611 .expect("claim removal catalog.update succeeds");
16612 assert_eq!(
16613 forwarding.active_binding_count().unwrap(),
16614 1,
16615 "removing an attested target claim must leave the route census unchanged"
16616 );
16617 assert!(
16618 client_rx.try_recv().is_err(),
16619 "claim removal must not emit route.closed or a client route GOODBYE"
16620 );
16621 assert!(
16622 target_rx.try_recv().is_err(),
16623 "claim removal must not send the target a route GOODBYE"
16624 );
16625 }
16626
16627 #[tokio::test]
16630 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
16631 let registry = Arc::new(Registry::default());
16632 let forwarding = Arc::new(ForwardingTable::default());
16633 let supervisor = SupervisorHandle::new();
16634 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16635 let handler =
16636 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16637 .with_supervisor(supervisor);
16638 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
16639 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
16640 register_capability_manifest(
16641 &handler,
16642 &target_ctx,
16643 &mut target_rx,
16644 capability_manifest("target", &["credentials-provider/v1"], &[]),
16645 1,
16646 )
16647 .await;
16648 register_capability_manifest(
16649 &handler,
16650 &opener_ctx,
16651 &mut opener_rx,
16652 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16653 2,
16654 )
16655 .await;
16656
16657 let (_client_rx, bind) = open_route_for_capability_test(
16658 &handler,
16659 &target_ctx,
16660 &mut target_rx,
16661 742,
16662 3,
16663 "target",
16664 None,
16665 )
16666 .await;
16667 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
16668 panic!("direct scope-honesty route must bind");
16669 };
16670 assert_eq!(principal, Some(Principal::Direct));
16671 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16672 }
16673
16674 #[tokio::test]
16677 async fn must_never_reach_self_route_is_capability_forbidden() {
16678 let registry = Arc::new(Registry::default());
16679 let forwarding = Arc::new(ForwardingTable::default());
16680 let supervisor = SupervisorHandle::new();
16681 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
16682 let handler =
16683 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16684 .with_supervisor(supervisor);
16685 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
16686 register_capability_manifest(
16687 &handler,
16688 &self_ctx,
16689 &mut self_rx,
16690 capability_manifest(
16691 "self-provider",
16692 &["credentials-provider/v1"],
16693 &["credentials-provider/v1"],
16694 ),
16695 1,
16696 )
16697 .await;
16698
16699 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
16700 let replies = handler
16701 .handle_control_frame(
16702 &client_ctx,
16703 route_open_frame_with_admission_facts(
16704 2,
16705 "self-provider",
16706 unique_project_root("admission-facts"),
16707 Some(ConsumerIdentity {
16708 module_id: "self-provider".to_string(),
16709 launch_nonce: "self-nonce".to_string(),
16710 }),
16711 None,
16712 ),
16713 )
16714 .await
16715 .expect("self-route refusal returns a typed frame");
16716 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
16717 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16718 assert!(
16719 self_rx.try_recv().is_err(),
16720 "self denial must not relay route.bind"
16721 );
16722 }
16723
16724 #[test]
16725 fn unsupported_channel_zero_frame_returns_error() {
16726 let handler = ControlHandler::default();
16727 let request = Frame::build(
16728 FrameType::Request,
16729 control_flags(),
16730 0,
16731 0,
16732 21,
16733 b"opaque".to_vec(),
16734 )
16735 .unwrap();
16736
16737 let response = handler
16738 .handle_control(ConnectionId::new(1), request)
16739 .unwrap();
16740
16741 assert_eq!(response[0].header.ty, FrameType::Error);
16742 assert_eq!(
16743 parse_error(&response[0])["code"],
16744 "unsupported_control_frame"
16745 );
16746 }
16747
16748 mod swap {
16753 use super::*;
16754
16755 const INCUMBENT: ConnectionId = ConnectionId::new(30);
16756 const CANDIDATE: ConnectionId = ConnectionId::new(40);
16757
16758 struct Swap {
16759 registry: Arc<Registry>,
16760 forwarding: Arc<ForwardingTable>,
16761 handler: ControlHandler,
16762 incumbent_ctx: RouteCtx,
16763 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16764 candidate_ctx: RouteCtx,
16765 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16766 }
16767
16768 async fn swap_with_incumbent() -> Swap {
16769 let registry = Arc::new(Registry::default());
16770 let forwarding = Arc::new(ForwardingTable::default());
16771 let handler =
16772 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
16773 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
16774 hello_via_sink(
16775 &handler,
16776 &incumbent_ctx,
16777 &mut incumbent_rx,
16778 hello_frame("aft", PROTOCOL_VERSION, 7),
16779 )
16780 .await;
16781 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
16782 Swap {
16783 registry,
16784 forwarding,
16785 handler,
16786 incumbent_ctx,
16787 incumbent_rx,
16788 candidate_ctx,
16789 candidate_rx,
16790 }
16791 }
16792
16793 fn register_candidate(swap: &Swap, ready: Option<bool>) {
16794 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
16795 candidate_manifest.ready = ready;
16796 let registration = swap
16797 .registry
16798 .register_candidate_with_control_ops(
16799 candidate_manifest,
16800 PROTOCOL_VERSION,
16801 CANDIDATE,
16802 module_baseline_control_ops(),
16803 )
16804 .unwrap();
16805 swap.forwarding
16806 .register_candidate_module_connection(
16807 CANDIDATE,
16808 "aft".to_string(),
16809 PROTOCOL_VERSION,
16810 manifest_concurrency(®istration.manifest),
16811 swap.candidate_ctx.egress.clone(),
16812 )
16813 .unwrap();
16814 }
16815
16816 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
16817 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
16818 swap.registry.promote_candidate("aft").unwrap().unwrap();
16819 cutover.incumbent.unwrap()
16820 }
16821
16822 #[tokio::test]
16831 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
16832 let mut swap = swap_with_incumbent().await;
16833 let hold = Arc::new(std::sync::Barrier::new(2));
16834 let handler = swap
16835 .handler
16836 .clone()
16837 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
16838 handler
16839 .supervisor
16840 .open_swap("aft", "candidate-nonce".to_string());
16841 let hello = {
16842 let handler = handler.clone();
16843 let candidate_ctx = swap.candidate_ctx.clone();
16844 std::thread::spawn(move || {
16845 handler
16846 .handle_hello(
16847 candidate_ctx.connection_id,
16848 Some(candidate_ctx.egress.clone()),
16849 hello_frame_with_nonce(
16850 "aft",
16851 PROTOCOL_VERSION,
16852 8,
16853 Some("candidate-nonce"),
16854 ),
16855 )
16856 .unwrap()
16857 })
16858 };
16859
16860 hold.wait();
16861 assert!(swap
16864 .registry
16865 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
16866 .unwrap()
16867 .is_some());
16868 assert!(swap
16869 .registry
16870 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
16871 .unwrap()
16872 .is_none());
16873 assert!(swap
16874 .forwarding
16875 .module_endpoint_for_connection(CANDIDATE)
16876 .unwrap()
16877 .is_none());
16878
16879 hold.wait();
16880 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
16881 assert_eq!(
16882 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
16883 FrameType::HelloAck
16884 );
16885 let published = swap
16886 .registry
16887 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
16888 .unwrap()
16889 .expect("candidate is published once its HELLO completes");
16890 assert_eq!(published.connection_id, CANDIDATE);
16891 assert!(swap
16892 .forwarding
16893 .module_endpoint_for_connection(CANDIDATE)
16894 .unwrap()
16895 .is_some());
16896 }
16897
16898 fn keyed_total(counters: &Value, key: &str) -> u64 {
16899 counters[key]
16900 .as_object()
16901 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
16902 .unwrap_or(0)
16903 }
16904
16905 #[tokio::test]
16906 async fn replacement_logs_old_end_and_new_admission_once() {
16907 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16908 let mut swap = swap_with_incumbent().await;
16909 swap.handler
16910 .supervisor
16911 .open_swap("aft", "candidate-nonce".to_string());
16912 hello_via_sink(
16913 &swap.handler,
16914 &swap.candidate_ctx,
16915 &mut swap.candidate_rx,
16916 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
16917 )
16918 .await;
16919 cutover(&swap);
16920 swap.handler.cleanup_connection(INCUMBENT).unwrap();
16921
16922 let captured = crate::router::test_log::captured_logs(&logs);
16923 println!("captured replacement registry logs:\n{captured}");
16924 let new_admissions: Vec<_> = captured
16925 .lines()
16926 .filter(|line| {
16927 line.contains("connection_id=40")
16928 && line.contains("swap candidate registered; not routable until cutover")
16929 })
16930 .collect();
16931 assert_eq!(
16932 new_admissions.len(),
16933 1,
16934 "unexpected admission logs: {captured}"
16935 );
16936 assert!(new_admissions[0].contains("module_id=aft"));
16937 assert!(new_admissions[0].contains("connection_id=40"));
16938 assert!(
16939 new_admissions[0].contains("swap candidate registered; not routable until cutover")
16940 );
16941 assert!(new_admissions[0].contains("ready=true"));
16942
16943 let old_admissions: Vec<_> = captured
16944 .lines()
16945 .filter(|line| {
16946 line.contains("module registered module_id=aft ")
16947 && line.contains("connection_id=30")
16948 })
16949 .collect();
16950 assert_eq!(
16951 old_admissions.len(),
16952 1,
16953 "unexpected incumbent admission: {captured}"
16954 );
16955
16956 let old_ends: Vec<_> = captured
16957 .lines()
16958 .filter(|line| {
16959 line.contains("connection_id=30") && line.contains("module registration ended")
16960 })
16961 .collect();
16962 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
16963 assert!(old_ends[0].contains("module_id=aft"));
16964 assert!(old_ends[0].contains("connection_id=30"));
16965 assert!(old_ends[0].contains("reason=replaced"));
16966 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
16967 assert_eq!(
16968 captured
16969 .lines()
16970 .filter(|line| line.contains("module registration promoted"))
16971 .count(),
16972 1,
16973 "unexpected promotion logs: {captured}"
16974 );
16975 }
16976
16977 #[tokio::test]
16983 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
16984 let mut swap = swap_with_incumbent().await;
16985 let handler = swap.handler.clone();
16986
16987 let cotenant = ConnectionId::new(31);
16989 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
16990 let (cotenant_task, cotenant_bind) = relay_route_open(
16991 &handler,
16992 cotenant,
16993 &cotenant_ctx.egress,
16994 &mut swap.incumbent_rx,
16995 100,
16996 "aft",
16997 "swap-cotenant",
16998 )
16999 .await;
17000 handler
17001 .handle_control_frame(
17002 &swap.incumbent_ctx,
17003 route_bind_ack(cotenant_bind.header.corr),
17004 )
17005 .await
17006 .unwrap();
17007 assert!(cotenant_task.await.unwrap().is_empty());
17008 let (cotenant_channel, cotenant_epoch) =
17009 published_route(&cotenant_rx.recv().await.unwrap());
17010
17011 let caller = ConnectionId::new(32);
17013 let (caller_ctx, mut caller_rx) = route_ctx(caller);
17014 let (caller_task, caller_bind) = relay_route_open(
17015 &handler,
17016 caller,
17017 &caller_ctx.egress,
17018 &mut swap.incumbent_rx,
17019 101,
17020 "aft",
17021 "swap-caller",
17022 )
17023 .await;
17024 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
17025
17026 register_candidate(&swap, None);
17027 cutover(&swap);
17028
17029 let ack = handler
17031 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
17032 .await;
17033 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
17034 if module_loop_error.is_some() {
17035 handler.cleanup_connection(INCUMBENT).unwrap();
17038 }
17039
17040 assert!(
17042 cotenant_rx.try_recv().is_err(),
17043 "the co-tenant route on the incumbent was torn down by one late ack: \
17044 {module_loop_error:?}"
17045 );
17046 assert!(matches!(
17047 swap.forwarding
17048 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
17049 .unwrap(),
17050 DataRoute::Client(DataRouteState::Bound(_))
17051 ));
17052 assert_eq!(module_loop_error, None);
17053 assert!(swap
17054 .registry
17055 .get_module_by_connection(INCUMBENT)
17056 .unwrap()
17057 .is_some());
17058
17059 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
17061 .await
17062 .expect("the incumbent is told to drop the abandoned binding")
17063 .unwrap()
17064 .frame;
17065 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17066 assert_eq!(goodbye.header.channel, abandoned_channel);
17067 assert_eq!(goodbye.header.epoch, abandoned_epoch);
17068 assert!(swap.incumbent_rx.try_recv().is_err());
17069
17070 let response = caller_task.await.unwrap();
17072 assert_eq!(response.len(), 1);
17073 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17074 assert!(caller_rx.try_recv().is_err());
17075
17076 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
17080 let counters = handler.counters().snapshot();
17081 assert_eq!(
17082 keyed_total(&counters, "route_open_accepted_by_principal"),
17083 1
17084 );
17085 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
17086 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
17087 }
17088
17089 #[tokio::test]
17093 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
17094 let mut swap = swap_with_incumbent().await;
17095 register_candidate(&swap, None);
17096 let incumbent = cutover(&swap);
17097 swap.forwarding
17098 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
17099 .unwrap()
17100 .expect("the incumbent is still registered");
17101
17102 let client = ConnectionId::new(33);
17103 let (client_ctx, mut client_rx) = route_ctx(client);
17104 let route_handler = swap.handler.clone();
17105 let open_ctx = RouteCtx {
17106 connection_id: client,
17107 egress: client_ctx.egress.clone(),
17108 };
17109 let mut route_task = tokio::spawn(async move {
17110 route_handler
17111 .handle_control_frame(
17112 &open_ctx,
17113 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
17114 )
17115 .await
17116 .unwrap()
17117 });
17118 let bind = tokio::select! {
17119 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
17120 response = &mut route_task => {
17121 let response = response.unwrap();
17122 panic!(
17123 "post-cutover route.open was refused instead of relayed to the candidate: {}",
17124 parse_error(&response[0])["code"]
17125 );
17126 }
17127 };
17128 swap.handler
17129 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
17130 .await
17131 .unwrap();
17132 assert!(route_task.await.unwrap().is_empty());
17133 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
17134 match swap
17135 .forwarding
17136 .lookup_data_route(client, channel, epoch)
17137 .unwrap()
17138 {
17139 DataRoute::Client(DataRouteState::Bound(route)) => {
17140 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
17141 }
17142 other => panic!("expected a bound route on the candidate, got {other:?}"),
17143 }
17144 assert!(swap.incumbent_rx.try_recv().is_err());
17145 }
17146
17147 #[tokio::test]
17152 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
17153 let swap = swap_with_incumbent().await;
17154 register_candidate(&swap, Some(false));
17155 let update = Frame::build(
17156 FrameType::Request,
17157 control_flags(),
17158 0,
17159 0,
17160 55,
17161 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
17162 provides: manifest("aft", PROTOCOL_VERSION).provides,
17163 capabilities: None,
17164 ready: Some(true),
17165 })
17166 .unwrap(),
17167 )
17168 .unwrap();
17169
17170 let replies = swap
17171 .handler
17172 .handle_control_frame(&swap.candidate_ctx, update)
17173 .await
17174 .unwrap();
17175
17176 assert_eq!(replies.len(), 1);
17177 assert_eq!(
17178 replies[0].header.ty,
17179 FrameType::Response,
17180 "candidate catalog.update was refused: {:?}",
17181 serde_json::from_slice::<Value>(&replies[0].body).ok()
17182 );
17183 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
17184 assert_eq!(
17185 swap.registry
17186 .get_module("aft")
17187 .unwrap()
17188 .unwrap()
17189 .connection_id,
17190 INCUMBENT
17191 );
17192 }
17193 }
17194
17195 mod swap_admission {
17199 use super::*;
17200
17201 const INCUMBENT_NONCE: &str = "incumbent-nonce";
17202 const CANDIDATE_NONCE: &str = "candidate-nonce";
17203
17204 fn handler_with_incumbent(
17205 module_id: &str,
17206 reserved: bool,
17207 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
17208 let registry = Arc::new(Registry::default());
17209 let supervisor = SupervisorHandle::new();
17210 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
17211 if reserved {
17212 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
17213 }
17214 let handler =
17215 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
17216 let incumbent = handler
17217 .handle_control(
17218 ConnectionId::new(1),
17219 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
17220 )
17221 .unwrap();
17222 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
17223 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
17224 (registry, supervisor, handler)
17225 }
17226
17227 #[test]
17233 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
17234 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
17235
17236 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
17237 let replies = handler
17238 .handle_control(
17239 ConnectionId::new(connection),
17240 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
17241 )
17242 .unwrap();
17243 assert_eq!(replies[0].header.ty, FrameType::Error);
17244 assert_eq!(
17245 parse_error(&replies[0])["code"],
17246 "swap_token_invalid",
17247 "nonce {nonce:?}"
17248 );
17249 }
17250 assert!(registry.get_candidate("aft").unwrap().is_none());
17251 assert_eq!(
17252 registry.get_module("aft").unwrap().unwrap().connection_id,
17253 ConnectionId::new(1)
17254 );
17255
17256 let admitted = handler
17259 .handle_control(
17260 ConnectionId::new(4),
17261 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
17262 )
17263 .unwrap();
17264 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
17265 assert_eq!(
17266 registry
17267 .get_candidate("aft")
17268 .unwrap()
17269 .unwrap()
17270 .connection_id,
17271 ConnectionId::new(4)
17272 );
17273 assert_eq!(
17274 registry.get_module("aft").unwrap().unwrap().connection_id,
17275 ConnectionId::new(1),
17276 "the candidate must not take the active slot"
17277 );
17278 let replayed = handler
17279 .handle_control(
17280 ConnectionId::new(5),
17281 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
17282 )
17283 .unwrap();
17284 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
17285
17286 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
17290 let squatter = handler
17291 .handle_control(
17292 ConnectionId::new(6),
17293 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
17294 )
17295 .unwrap();
17296 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
17297 assert!(
17298 registry.get_module("aft").unwrap().is_none(),
17299 "a squatter took the active slot of an id being swapped"
17300 );
17301 }
17302
17303 #[test]
17308 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
17309 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
17310
17311 let replies = handler
17312 .handle_control(
17313 ConnectionId::new(2),
17314 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17315 )
17316 .unwrap();
17317
17318 assert_eq!(
17319 replies[0].header.ty,
17320 FrameType::HelloAck,
17321 "reserved candidate refused: {:?}",
17322 serde_json::from_slice::<Value>(&replies[0].body).ok()
17323 );
17324 assert_eq!(
17325 registry
17326 .get_candidate("vault")
17327 .unwrap()
17328 .unwrap()
17329 .connection_id,
17330 ConnectionId::new(2)
17331 );
17332 }
17333
17334 #[test]
17337 fn without_an_open_swap_the_ordinary_gates_decide() {
17338 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
17339 supervisor.close_swap("vault");
17340
17341 let candidate = handler
17342 .handle_control(
17343 ConnectionId::new(2),
17344 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17345 )
17346 .unwrap();
17347 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
17348 let duplicate = handler
17349 .handle_control(
17350 ConnectionId::new(3),
17351 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
17352 )
17353 .unwrap();
17354 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
17355 assert!(registry.get_candidate("vault").unwrap().is_none());
17356 }
17357 }
17358
17359 mod scopes {
17363 mod idle_tests {
17364 use super::*;
17365 include!("scope_idle_tests.rs");
17366 }
17367 mod route_open_timings {
17368 use super::*;
17369 include!("route_open_timing_tests.rs");
17370 }
17371 mod route_open_refusal_log {
17372 use super::*;
17373 include!("route_open_refusal_log_tests.rs");
17374 }
17375 mod activity {
17376 include!("activity_scope_tests.rs");
17377 }
17378 mod delta {
17379 include!("scope_handler_delta_tests.rs");
17380 }
17381 mod call_modes {
17382 include!("scope_call_mode_tests.rs");
17383 }
17384 #[tokio::test]
17385 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
17386 let rig = rig().await;
17387 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
17388 "s".into(),
17389 1,
17390 "tool".into(),
17391 ))
17392 .unwrap();
17393 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
17394 let (direct, _rx) = wide_ctx(999);
17395 for (ctx, expected) in [
17396 (&rig.owner, FrameType::Response),
17397 (&direct, FrameType::Error),
17398 ] {
17399 let frame =
17400 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
17401 .unwrap();
17402 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
17403 assert_eq!(replies[0].header.ty, expected);
17404 if expected == FrameType::Response {
17405 let reply: subc_control::ScopeServedByReply =
17406 serde_json::from_slice(&replies[0].body).unwrap();
17407 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
17408 } else {
17409 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
17410 }
17411 }
17412 let reply = call(
17413 &rig.handler,
17414 &rig.owner,
17415 &ModuleControlRequestFromModule::ScopeDescribe {
17416 owner: Principal::Reserved {
17417 module_id: OWNER.into(),
17418 },
17419 scope_ref: "s".into(),
17420 },
17421 )
17422 .await;
17423 assert!(matches!(
17424 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
17425 ModuleControlResponseToModule::ScopeDescribe { .. }
17426 ));
17427 }
17428
17429 use subc_protocol::scope::{
17430 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
17431 ScopeStatus,
17432 };
17433
17434 use super::*;
17435
17436 const OWNER: &str = "prefrontal-core";
17437
17438 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
17439 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
17440 }
17441
17442 async fn call(
17443 handler: &ControlHandler,
17444 ctx: &RouteCtx,
17445 request: &ModuleControlRequestFromModule,
17446 ) -> Frame {
17447 let body = serde_json::to_vec(request).unwrap();
17448 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
17449 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
17450 assert_eq!(replies.len(), 1, "{replies:?}");
17451 replies.pop().unwrap()
17452 }
17453
17454 async fn sync(
17455 handler: &ControlHandler,
17456 ctx: &RouteCtx,
17457 generation: u64,
17458 scopes: Vec<ScopeRecord>,
17459 ) -> Result<ModuleControlResponseToModule, String> {
17460 let reply = call(
17461 handler,
17462 ctx,
17463 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
17464 )
17465 .await;
17466 match reply.header.ty {
17467 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
17468 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
17469 }
17470 }
17471
17472 async fn describe(
17473 handler: &ControlHandler,
17474 ctx: &RouteCtx,
17475 owner: &str,
17476 scope_ref: &str,
17477 ) -> ModuleControlResponseToModule {
17478 let reply = call(
17479 handler,
17480 ctx,
17481 &ModuleControlRequestFromModule::ScopeDescribe {
17482 owner: Principal::Reserved {
17483 module_id: owner.to_string(),
17484 },
17485 scope_ref: scope_ref.to_string(),
17486 },
17487 )
17488 .await;
17489 assert_eq!(
17490 reply.header.ty,
17491 FrameType::Response,
17492 "{:?}",
17493 parse_error(&reply)
17494 );
17495 serde_json::from_slice(&reply.body).unwrap()
17496 }
17497
17498 #[tokio::test]
17499 async fn scope_describe_never_echoes_bind_proof_or_hash() {
17500 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17501 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17502 let mut rig = rig().await;
17503 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17504 .await;
17505 let _bound = rig
17506 .bound(
17507 Some(AFT),
17508 PLEXUS,
17509 Some(
17510 rig.selector("s", Some(1))
17511 .with_bind_proof(Some(PROOF.into())),
17512 ),
17513 )
17514 .await;
17515 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
17516 let output = serde_json::to_string(&reply).unwrap();
17517 assert!(output.contains("\"status\":\"live\""));
17518 assert!(!output.contains(PROOF));
17519 assert!(!output.contains(HASH));
17520 }
17521
17522 async fn module(
17524 handler: &ControlHandler,
17525 connection: u64,
17526 module_id: &str,
17527 nonce: Option<&str>,
17528 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
17529 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
17530 hello_via_sink(
17531 handler,
17532 &ctx,
17533 &mut rx,
17534 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
17535 )
17536 .await;
17537 (ctx, rx)
17538 }
17539
17540 #[tokio::test]
17543 async fn a_client_connection_cannot_sync_or_describe() {
17544 let handler = ControlHandler::new(Arc::new(Registry::default()));
17545 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
17546 for request in [
17547 ModuleControlRequestFromModule::ScopeSync {
17548 generation: 1,
17549 scopes: vec![head("s", 1)],
17550 },
17551 ModuleControlRequestFromModule::ScopeDescribe {
17552 owner: Principal::Direct,
17553 scope_ref: "s".to_string(),
17554 },
17555 ] {
17556 let reply = call(&handler, &ctx, &request).await;
17557 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
17558 }
17559 assert!(
17560 !handler
17561 .scopes
17562 .read()
17563 .unwrap()
17564 .describe(
17565 &Principal::Reserved {
17566 module_id: OWNER.to_string()
17567 },
17568 "s"
17569 )
17570 .owner_synced
17571 );
17572 }
17573
17574 #[tokio::test]
17577 async fn a_module_without_a_supervised_launch_cannot_sync() {
17578 let handler = ControlHandler::new(Arc::new(Registry::default()));
17579 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
17580 assert_eq!(
17581 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
17582 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17583 );
17584 }
17585
17586 #[tokio::test]
17587 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
17588 let supervisor = SupervisorHandle::new();
17589 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17590 let handler = ControlHandler::new(Arc::new(Registry::default()))
17591 .with_supervisor(supervisor.clone());
17592 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
17593 sync(&handler, &incumbent, 1, vec![head("s", 1)])
17594 .await
17595 .expect("the current launch syncs");
17596
17597 supervisor.open_swap(OWNER, "n2".to_string());
17600 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
17601 assert_eq!(
17602 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
17603 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17604 );
17605 sync(&handler, &incumbent, 2, vec![head("s", 1)])
17606 .await
17607 .expect("the serving owner syncs during the swap");
17608
17609 supervisor.close_swap(OWNER);
17612 assert_eq!(
17613 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
17614 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17615 );
17616 sync(&handler, &incumbent, 3, vec![head("s", 1)])
17617 .await
17618 .expect("the serving owner syncs after the rollback");
17619 handler.cleanup_connection(candidate.connection_id).unwrap();
17620
17621 supervisor.open_swap(OWNER, "n3".to_string());
17626 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
17627 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
17628 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
17629 .await
17630 .expect("the promoted launch takes authority");
17631 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17632 panic!("unexpected reply {reply:?}");
17633 };
17634 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
17635 assert_eq!(
17636 sync(&handler, &incumbent, 4, Vec::new()).await,
17637 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17638 );
17639 }
17640
17641 #[tokio::test]
17644 async fn closing_the_authority_connection_frees_sync_authority() {
17645 let supervisor = SupervisorHandle::new();
17646 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17647 let handler = ControlHandler::new(Arc::new(Registry::default()))
17648 .with_supervisor(supervisor.clone());
17649 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
17650 sync(&handler, &first, 10, vec![head("s", 1)])
17651 .await
17652 .unwrap();
17653 handler.cleanup_connection(first.connection_id).unwrap();
17654
17655 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
17656 sync(&handler, &second, 1, vec![head("s", 1)])
17657 .await
17658 .expect("the next connection takes the released authority");
17659 }
17660
17661 #[tokio::test]
17662 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
17663 let supervisor = SupervisorHandle::new();
17664 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17665 let handler =
17666 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
17667 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
17668 sync(&handler, &first, 10, vec![head("s", 1)])
17669 .await
17670 .unwrap();
17671 handler
17672 .handle_control_frame(
17673 &first,
17674 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
17675 )
17676 .await
17677 .unwrap();
17678 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
17679 sync(&handler, &second, 1, vec![head("s", 1)])
17680 .await
17681 .expect("GOODBYE releases authority even if the old socket remains open");
17682 }
17683
17684 #[tokio::test]
17685 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
17686 let registry = Arc::new(Registry::default());
17687 let supervisor_handle = SupervisorHandle::new();
17688 let supervisor =
17689 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
17690 .with_handle(supervisor_handle.clone())
17691 .with_daemon_incarnation("incarnation-7".to_string());
17692 supervisor
17695 .supervise_configured(
17696 ModuleSpec {
17697 carry_containment: false,
17698 module_id: OWNER.to_string(),
17699 program: PathBuf::from("/nonexistent/prefrontal-core"),
17700 args: Vec::new(),
17701 env: Vec::new(),
17702 reserved: false,
17703 reserved_prefixes: Vec::new(),
17704 protocol: ModuleProtocol::Subc,
17705 overlap: Default::default(),
17706 },
17707 false,
17708 )
17709 .unwrap();
17710 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
17711 let handler =
17712 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17713 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
17714
17715 let ModuleControlResponseToModule::ScopeDescribe {
17717 status,
17718 daemon_incarnation,
17719 owner_synced,
17720 owner_configured,
17721 scope,
17722 ..
17723 } = describe(&handler, &reader, OWNER, "s").await
17724 else {
17725 panic!("not a describe reply");
17726 };
17727 assert_eq!(status, ScopeStatus::NotLive);
17728 assert_eq!(daemon_incarnation, "incarnation-7");
17729 assert!(!owner_synced);
17730 assert!(owner_configured);
17731 assert!(scope.is_none());
17732
17733 let ModuleControlResponseToModule::ScopeDescribe {
17736 status,
17737 owner_configured,
17738 ..
17739 } = describe(&handler, &reader, "ghost", "s").await
17740 else {
17741 panic!("not a describe reply");
17742 };
17743 assert_eq!(status, ScopeStatus::NotLive);
17744 assert!(!owner_configured);
17745
17746 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
17748 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
17749 let ModuleControlResponseToModule::ScopeDescribe {
17750 status,
17751 scope_epoch,
17752 owner_synced,
17753 scope,
17754 ..
17755 } = describe(&handler, &reader, OWNER, "s").await
17756 else {
17757 panic!("not a describe reply");
17758 };
17759 assert_eq!(status, ScopeStatus::Live);
17760 assert_eq!(scope_epoch, Some(4));
17761 assert!(owner_synced);
17762 let stamp = scope.expect("a live scope carries its stamp");
17763 assert!(
17764 stamp.owner_authorized,
17765 "prefrontal-core is the default authority"
17766 );
17767 assert_eq!(stamp.kind, ScopeKind::Head);
17768 }
17769
17770 #[tokio::test]
17771 async fn scope_authority_owners_decides_owner_authorized() {
17772 let supervisor = SupervisorHandle::new();
17773 supervisor.set_spawn_nonce("broca", "b1".to_string());
17774 let handler = ControlHandler::new(Arc::new(Registry::default()))
17775 .with_supervisor(supervisor)
17776 .with_scope_authority_owners(vec!["broca".to_string()]);
17777 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
17778 let mut gated = head("s", 1);
17779 gated.attributes.agent_id = Some("agent".to_string());
17780 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
17781 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
17782 describe(&handler, &broca, "broca", "s").await
17783 else {
17784 panic!("not a describe reply");
17785 };
17786 assert!(scope.unwrap().owner_authorized);
17787 }
17788
17789 #[tokio::test]
17793 async fn scope_ops_and_the_scopes_capability_are_advertised() {
17794 let handler = ControlHandler::new(Arc::new(Registry::default()));
17795 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
17796 let ack = hello_via_sink(
17797 &handler,
17798 &ctx,
17799 &mut rx,
17800 hello_frame("m", PROTOCOL_VERSION, 1),
17801 )
17802 .await;
17803 let ack = parse_ack(&ack);
17804 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
17805 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
17806 }
17807 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
17808
17809 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
17810 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
17811 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
17812 let reply = handler
17813 .handle_control_frame(&client, frame)
17814 .await
17815 .unwrap()
17816 .pop()
17817 .unwrap();
17818 let ClientControlResponse::ServerDescribe { capabilities, .. } =
17819 serde_json::from_slice(&reply.body).unwrap()
17820 else {
17821 panic!("not a server.describe reply");
17822 };
17823 assert!(
17824 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
17825 "{capabilities:?}"
17826 );
17827 }
17828
17829 const PLEXUS: &str = "plexus";
17832 const OTHER: &str = "other";
17833 const AFT: &str = "aft";
17834 const BROCA: &str = "broca";
17835 const MAGIC: &str = "magic-context";
17836
17837 fn nonce(module_id: &str) -> String {
17838 format!("nonce-{module_id}")
17839 }
17840
17841 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
17842 let (tx, rx) = mpsc::channel(64);
17843 (
17844 RouteCtx {
17845 connection_id: ConnectionId::new(connection),
17846 egress: FrameSink::new(tx),
17847 },
17848 rx,
17849 )
17850 }
17851
17852 struct Rig {
17856 handler: ControlHandler,
17857 forwarding: Arc<ForwardingTable>,
17858 owner: RouteCtx,
17859 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17860 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
17861 generation: u64,
17862 next_connection: u64,
17863 _supervisor: Supervisor,
17864 }
17865
17866 async fn rig() -> Rig {
17867 rig_with_flow_support(true).await
17868 }
17869
17870 async fn rig_with_flow_support(flow_support: bool) -> Rig {
17871 rig_with_scope_capabilities(flow_support, false).await
17872 }
17873
17874 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
17875 let mut provides = Vec::new();
17876 if flow_support {
17877 provides.push("flow-scopes/v1");
17878 }
17879 if run_support {
17880 provides.push("agent-run-scopes/v1");
17881 }
17882 rig_with_target_capabilities(&provides).await
17883 }
17884
17885 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
17886 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
17887 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17888 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17891 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
17892 Frame::build(
17893 FrameType::Hello,
17894 control_flags(),
17895 0,
17896 0,
17897 connection,
17898 serde_json::to_vec(&body).unwrap(),
17899 )
17900 .unwrap()
17901 }
17902
17903 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
17904 let registry = Arc::new(Registry::default());
17905 let forwarding = Arc::new(ForwardingTable::default());
17906 let supervisor_handle = SupervisorHandle::new();
17907 let supervisor =
17908 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
17909 .with_handle(supervisor_handle.clone());
17910 supervisor
17911 .supervise_configured(
17912 ModuleSpec {
17913 carry_containment: false,
17914 module_id: OWNER.to_string(),
17915 program: PathBuf::from("/nonexistent/prefrontal-core"),
17916 args: Vec::new(),
17917 env: Vec::new(),
17918 reserved: false,
17919 reserved_prefixes: Vec::new(),
17920 protocol: ModuleProtocol::Subc,
17921 overlap: Default::default(),
17922 },
17923 false,
17924 )
17925 .unwrap();
17926 for module_id in [OWNER, AFT, BROCA, MAGIC] {
17927 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
17928 }
17929 let handler =
17930 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17931 .with_supervisor(supervisor_handle);
17932 let (owner, mut owner_rx) = wide_ctx(1);
17933 hello_via_sink(
17934 &handler,
17935 &owner,
17936 &mut owner_rx,
17937 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
17938 )
17939 .await;
17940 let mut modules = BTreeMap::new();
17941 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
17942 let (ctx, mut rx) = wide_ctx(connection);
17943 let hello = hello_with_capabilities(module_id, connection, provides);
17944 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
17945 modules.insert(module_id.to_string(), (ctx, rx));
17946 }
17947 Rig {
17948 handler,
17949 forwarding,
17950 owner,
17951 _owner_rx: owner_rx,
17952 modules,
17953 generation: 0,
17954 next_connection: 100,
17955 _supervisor: supervisor,
17956 }
17957 }
17958
17959 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
17960 ScopeCarrier::new(Principal::Reserved {
17961 module_id: module_id.to_string(),
17962 })
17963 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
17964 }
17965
17966 fn session(scope_epoch: u64) -> ScopeRecord {
17969 let mut record = head("s", scope_epoch);
17970 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
17971 record.attributes.agent_id = Some("agent-1".to_string());
17972 record.attributes.delegates = true;
17973 record
17974 }
17975
17976 impl Rig {
17977 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
17978 self.generation += 1;
17979 sync(&self.handler, &self.owner, self.generation, scopes)
17980 .await
17981 .expect("the owner's sync is accepted");
17982 }
17983
17984 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
17985 ScopeSelector::new(
17986 Principal::Reserved {
17987 module_id: OWNER.to_string(),
17988 },
17989 scope_ref.to_string(),
17990 scope_epoch,
17991 )
17992 }
17993
17994 fn open_frame(
17995 &mut self,
17996 opener: Option<&str>,
17997 target: &str,
17998 scope: Option<ScopeSelector>,
17999 ) -> (
18000 RouteCtx,
18001 mpsc::Receiver<crate::router::OutboundFrame>,
18002 Frame,
18003 ) {
18004 self.next_connection += 1;
18005 let (ctx, rx) = wide_ctx(self.next_connection);
18006 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
18007 busy_refusals: None,
18008 target: RouteTarget::ToolProvider {
18009 module_id: target.to_string(),
18010 },
18011 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
18012 consumer_identity: opener.map(|module_id| ConsumerIdentity {
18013 module_id: module_id.to_string(),
18014 launch_nonce: nonce(module_id),
18015 }),
18016 consumer_capabilities: None,
18017 role_versions: None,
18018 admission_facts: None,
18019 scope,
18020 })
18021 .unwrap();
18022 let frame = Frame::build(
18023 FrameType::Request,
18024 control_flags(),
18025 0,
18026 0,
18027 self.next_connection,
18028 body,
18029 )
18030 .unwrap();
18031 (ctx, rx, frame)
18032 }
18033
18034 async fn refused(
18036 &mut self,
18037 opener: Option<&str>,
18038 target: &str,
18039 scope: Option<ScopeSelector>,
18040 ) -> String {
18041 self.refusal_body(opener, target, scope).await["code"]
18042 .as_str()
18043 .unwrap()
18044 .to_string()
18045 }
18046
18047 async fn refusal_body(
18048 &mut self,
18049 opener: Option<&str>,
18050 target: &str,
18051 scope: Option<ScopeSelector>,
18052 ) -> Value {
18053 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
18054 let replies = tokio::time::timeout(
18055 Duration::from_secs(2),
18056 self.handler.handle_control_frame(&ctx, frame),
18057 )
18058 .await
18059 .expect("the open must be refused before waiting for a bind ack")
18060 .unwrap();
18061 assert_eq!(replies.len(), 1, "{replies:?}");
18062 assert_eq!(replies[0].header.ty, FrameType::Error);
18063 let (_, module_rx) = self.modules.get_mut(target).unwrap();
18064 assert!(
18065 module_rx.try_recv().is_err(),
18066 "a refused open relays nothing"
18067 );
18068 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
18069 parse_error(&replies[0])
18070 }
18071
18072 async fn relayed(
18074 &mut self,
18075 opener: Option<&str>,
18076 target: &str,
18077 scope: Option<ScopeSelector>,
18078 ) -> Relayed {
18079 self.relayed_with_legacy_root(opener, target, scope, None)
18080 .await
18081 }
18082
18083 async fn relayed_with_legacy_root(
18084 &mut self,
18085 opener: Option<&str>,
18086 target: &str,
18087 scope: Option<ScopeSelector>,
18088 legacy_root: Option<&std::path::Path>,
18089 ) -> Relayed {
18090 let (ctx, rx, mut frame) = self.open_frame(opener, target, scope);
18091 if let Some(root) = legacy_root {
18092 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
18093 body["identity"]["project_root"] = serde_json::json!(root);
18094 frame.body = serde_json::to_vec(&body).unwrap();
18095 }
18096 let handler = self.handler.clone();
18097 let task_ctx = ctx.clone();
18098 let task = tokio::spawn(async move {
18099 handler
18100 .handle_control_frame(&task_ctx, frame)
18101 .await
18102 .unwrap()
18103 });
18104 let (_, module_rx) = self.modules.get_mut(target).unwrap();
18105 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
18106 .await
18107 .expect("the target receives the relayed route.bind")
18108 .unwrap()
18109 .frame;
18110 Relayed {
18111 target: target.to_string(),
18112 client: ctx,
18113 client_rx: rx,
18114 task,
18115 bind,
18116 }
18117 }
18118
18119 async fn ack(&self, relayed: &Relayed) {
18120 let (module, _) = &self.modules[&relayed.target];
18121 self.handler
18122 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
18123 .await
18124 .unwrap();
18125 }
18126
18127 async fn bound(
18129 &mut self,
18130 opener: Option<&str>,
18131 target: &str,
18132 scope: Option<ScopeSelector>,
18133 ) -> Bound {
18134 let relayed = self.relayed(opener, target, scope).await;
18135 self.commit_relayed(relayed).await
18136 }
18137
18138 async fn bound_with_legacy_root(
18139 &mut self,
18140 opener: Option<&str>,
18141 target: &str,
18142 scope: Option<ScopeSelector>,
18143 root: &std::path::Path,
18144 ) -> Bound {
18145 let relayed = self
18146 .relayed_with_legacy_root(opener, target, scope, Some(root))
18147 .await;
18148 self.commit_relayed(relayed).await
18149 }
18150
18151 async fn commit_relayed(&self, relayed: Relayed) -> Bound {
18152 self.ack(&relayed).await;
18153 let Relayed {
18154 target,
18155 client,
18156 mut client_rx,
18157 task,
18158 bind,
18159 } = relayed;
18160 assert!(
18161 task.await.unwrap().is_empty(),
18162 "the open is answered by commit"
18163 );
18164 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
18165 Bound {
18166 target,
18167 client,
18168 client_rx,
18169 channel,
18170 epoch,
18171 bind,
18172 }
18173 }
18174
18175 fn live(&self, route: &Bound) -> bool {
18176 matches!(
18177 self.forwarding
18178 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
18179 .unwrap(),
18180 DataRoute::Client(DataRouteState::Bound(_))
18181 )
18182 }
18183 }
18184
18185 struct Relayed {
18186 target: String,
18187 client: RouteCtx,
18188 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18189 task: tokio::task::JoinHandle<Vec<Frame>>,
18190 bind: Frame,
18191 }
18192
18193 struct Bound {
18194 target: String,
18195 client: RouteCtx,
18196 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18197 channel: u16,
18198 epoch: u32,
18199 bind: Frame,
18200 }
18201
18202 impl Bound {
18203 fn closed_reason(&mut self) -> RouteCloseReason {
18206 let mut reason = None;
18207 let mut goodbye = false;
18208 while let Ok(outbound) = self.client_rx.try_recv() {
18209 let frame = outbound.frame;
18210 match frame.header.ty {
18211 FrameType::Goodbye => {
18212 assert_eq!(
18213 (frame.header.channel, frame.header.epoch),
18214 (self.channel, self.epoch)
18215 );
18216 goodbye = true;
18217 }
18218 FrameType::Push => {
18219 let ClientControlPush::RouteClosed {
18220 reason: r,
18221 module_id,
18222 ..
18223 } = serde_json::from_slice(&frame.body).unwrap()
18224 else {
18225 panic!("unexpected push");
18226 };
18227 assert_eq!(module_id, self.target);
18228 reason = Some(r);
18229 }
18230 other => panic!("unexpected frame {other:?}"),
18231 }
18232 }
18233 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
18234 reason.expect("the client is told why the route closed")
18235 }
18236
18237 fn untouched(&mut self) -> bool {
18238 self.client_rx.try_recv().is_err()
18239 }
18240
18241 fn stamp(&self) -> Option<ScopeStamp> {
18242 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
18243 ModuleControlRequest::RouteBind { scope, .. } => scope,
18244 other => panic!("expected a route.bind, got {other:?}"),
18245 }
18246 }
18247 }
18248
18249 #[tokio::test]
18250 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
18251 ) {
18252 let mut rig = rig().await;
18253 let mut record = session(1);
18254 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
18255 record.child_owners = vec![Principal::Reserved {
18256 module_id: MAGIC.to_string(),
18257 }];
18258 rig.sync(vec![record]).await;
18259 let scope = || Some(rig_selector("s", Some(1)));
18260
18261 rig.bound(Some(OWNER), PLEXUS, scope()).await;
18264 rig.bound(Some(AFT), OTHER, scope()).await;
18265 rig.bound(Some(BROCA), PLEXUS, scope()).await;
18266
18267 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
18271 assert_eq!(
18272 rig.refused(opener, target, scope()).await,
18273 error_codes::SCOPE_NOT_CARRIER,
18274 "{opener:?} -> {target}"
18275 );
18276 }
18277 }
18278
18279 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
18280 ScopeSelector::new(
18281 Principal::Reserved {
18282 module_id: OWNER.to_string(),
18283 },
18284 scope_ref.to_string(),
18285 scope_epoch,
18286 )
18287 }
18288
18289 #[tokio::test]
18290 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
18291 let mut rig = rig().await;
18292 rig.sync(vec![session(1)]).await;
18293 for opener in [OWNER, AFT] {
18294 assert_eq!(
18295 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
18296 .await,
18297 error_codes::SCOPE_EPOCH_REQUIRED,
18298 "{opener}"
18299 );
18300 }
18301 }
18302
18303 #[tokio::test]
18304 async fn admission_separates_not_synced_not_live_and_ended() {
18305 let mut rig = rig().await;
18306 let code = rig
18308 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18309 .await;
18310 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
18311 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
18312
18313 let ghost = ScopeSelector::new(
18315 Principal::Reserved {
18316 module_id: "ghost".to_string(),
18317 },
18318 "s".to_string(),
18319 Some(1),
18320 );
18321 assert_eq!(
18322 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
18323 error_codes::SCOPE_NOT_LIVE
18324 );
18325
18326 rig.sync(vec![session(2)]).await;
18327 assert_eq!(
18328 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
18329 .await,
18330 error_codes::SCOPE_NOT_LIVE
18331 );
18332 for epoch in [1, 3] {
18333 assert_eq!(
18334 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
18335 .await,
18336 error_codes::SCOPE_ENDED,
18337 "epoch {epoch}"
18338 );
18339 }
18340 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
18342 .await;
18343 }
18344
18345 #[tokio::test]
18346 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
18347 let mut rig = rig().await;
18348 rig.sync(vec![session(1)]).await;
18349 let route = rig
18350 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18351 .await;
18352 let stamp = route.stamp().expect("a scoped bind carries the stamp");
18353 assert_eq!(stamp.scope_ref, "s");
18354 assert_eq!(stamp.scope_epoch, 1);
18355 assert_eq!(stamp.kind, ScopeKind::Head);
18356 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
18357 assert!(stamp.attributes.delegates);
18358 assert!(stamp.owner_authorized);
18359 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18360 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
18361
18362 let (broca, mut broca_rx) = wide_ctx(50);
18365 hello_via_sink(
18366 &rig.handler,
18367 &broca,
18368 &mut broca_rx,
18369 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
18370 )
18371 .await;
18372 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
18373 .await
18374 .unwrap();
18375 let own = ScopeSelector::new(
18376 Principal::Reserved {
18377 module_id: BROCA.to_string(),
18378 },
18379 "b".to_string(),
18380 Some(1),
18381 );
18382 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
18383 assert!(!route.stamp().unwrap().owner_authorized);
18384 }
18385
18386 #[tokio::test]
18387 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
18388 let mut rig = rig().await;
18389 let mut record = head("s", 1);
18390 record.carriers = vec![carrier(AFT, None)];
18391 let flow_id = "Flow:run-7/step_2!~";
18392 record.attributes.flow_id = Some(flow_id.to_string());
18393 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
18394 .await
18395 .unwrap();
18396 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
18397 panic!("not a sync reply");
18398 };
18399 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
18400 let route = rig
18401 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18402 .await;
18403 assert!(rig.live(&route), "the stamped bind committed");
18404 let stamp = route.stamp().expect("a flow scope carries a stamp");
18405 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
18406 assert_eq!(stamp.attributes.agent_id, None);
18407 assert!(!stamp.attributes.delegates);
18408 assert!(stamp.owner_authorized);
18409 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18410 assert_eq!(unscoped.stamp(), None);
18411 }
18412
18413 #[tokio::test]
18414 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
18415 let mut rig = rig_with_flow_support(false).await;
18416 let mut record = session(1);
18417 record.attributes.flow_id = Some("flow:7".to_string());
18418 rig.sync(vec![record]).await;
18419 for opener in [OWNER, AFT] {
18420 let body = rig
18421 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
18422 .await;
18423 assert_eq!(body["code"], "target_flow_unsupported");
18424 let message = body["message"].as_str().unwrap();
18425 for required in [PLEXUS, "flow-scopes/v1"] {
18426 assert!(message.contains(required), "{message}");
18427 }
18428 }
18429 }
18430
18431 #[tokio::test]
18432 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
18433 let mut rig = rig_with_flow_support(true).await;
18434 let mut record = session(1);
18435 record.attributes.flow_id = Some("flow:7".to_string());
18436 rig.sync(vec![record]).await;
18437 for opener in [OWNER, AFT] {
18438 let route = rig
18439 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
18440 .await;
18441 assert!(rig.live(&route));
18442 assert_eq!(
18443 route.stamp().unwrap().attributes.flow_id.as_deref(),
18444 Some("flow:7")
18445 );
18446 }
18447 }
18448
18449 #[tokio::test]
18450 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
18451 use std::future::Future;
18452
18453 let mut rig = rig_with_flow_support(true).await;
18454 let mut record = session(1);
18455 record.attributes.flow_id = Some("flow:7".to_string());
18456 rig.sync(vec![record]).await;
18457 let (client, mut client_rx, frame) =
18458 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
18459 for _ in 0..64 {
18462 client.egress.try_send(route_bind_ack(1)).unwrap();
18463 }
18464 let handler = rig.handler.clone();
18465 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18466 std::future::poll_fn(|cx| {
18467 assert!(open.as_mut().poll(cx).is_pending());
18468 std::task::Poll::Ready(())
18469 })
18470 .await;
18471 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18472
18473 let old_connection = rig.modules[PLEXUS].0.connection_id;
18474 rig.handler.cleanup_connection(old_connection).unwrap();
18475 let (replacement, mut replacement_rx) = wide_ctx(200);
18476 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
18477 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
18478 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
18479 let hello = Frame::build(
18480 FrameType::Hello,
18481 control_flags(),
18482 0,
18483 0,
18484 200,
18485 serde_json::to_vec(&body).unwrap(),
18486 )
18487 .unwrap();
18488 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
18489
18490 client_rx.try_recv().unwrap();
18491 let replies = tokio::time::timeout(Duration::from_secs(2), open)
18492 .await
18493 .expect("the replacement is refused without waiting for a bind ack")
18494 .unwrap();
18495 assert_eq!(replies.len(), 1);
18496 let body = parse_error(&replies[0]);
18497 assert_eq!(body["code"], "target_flow_unsupported");
18498 for required in [PLEXUS, "flow-scopes/v1"] {
18499 assert!(body["message"].as_str().unwrap().contains(required));
18500 }
18501 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
18502 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
18503 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18504 assert!(rig
18505 .handler
18506 .registry
18507 .get_module_by_connection(replacement.connection_id)
18508 .unwrap()
18509 .is_some());
18510 }
18511
18512 #[tokio::test]
18513 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
18514 use std::future::Future;
18515 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
18516 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18517 const NEXT_HASH: &str =
18518 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
18519 let mut rig = rig().await;
18520 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
18521 record.carriers.push(ScopeCarrier::new(Principal::Direct));
18522 rig.sync(vec![record.clone()]).await;
18523 let (client, mut rx, frame) = rig.open_frame(
18524 None,
18525 PLEXUS,
18526 Some(
18527 rig.selector("s", Some(1))
18528 .with_bind_proof(Some(PROOF.into())),
18529 ),
18530 );
18531 for _ in 0..64 {
18534 client.egress.try_send(route_bind_ack(1)).unwrap();
18535 }
18536 let handler = rig.handler.clone();
18537 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18538 std::future::poll_fn(|cx| {
18539 assert!(open.as_mut().poll(cx).is_pending());
18540 std::task::Poll::Ready(())
18541 })
18542 .await;
18543 record.bind_proof_sha256 = Some(NEXT_HASH.into());
18544 rig.sync(vec![record]).await;
18545 rx.try_recv().unwrap();
18546 let replies = tokio::time::timeout(Duration::from_secs(2), open)
18547 .await
18548 .unwrap()
18549 .unwrap();
18550 assert_eq!(
18551 parse_error(&replies[0])["code"],
18552 error_codes::SCOPE_BIND_PROOF_MISMATCH
18553 );
18554 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
18555 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
18556 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18557 }
18558
18559 #[tokio::test]
18560 async fn scope_bind_proof_reserved_openers_ignore_proof() {
18561 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18562 let mut rig = rig().await;
18563 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18564 .await;
18565 for opener in [OWNER, AFT] {
18566 for proof in [None, Some("malformed+proof=")] {
18567 let route = rig
18568 .bound(
18569 Some(opener),
18570 PLEXUS,
18571 Some(
18572 rig.selector("s", Some(1))
18573 .with_bind_proof(proof.map(str::to_string)),
18574 ),
18575 )
18576 .await;
18577 assert!(rig.live(&route));
18578 }
18579 }
18580 }
18581
18582 #[tokio::test]
18583 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
18584 use std::future::Future;
18585 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18586 const NEXT_HASH: &str =
18587 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
18588 let mut rig = rig().await;
18589 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18590 .await;
18591 let (client, mut rx, frame) =
18592 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
18593 for _ in 0..64 {
18594 client.egress.try_send(route_bind_ack(1)).unwrap();
18595 }
18596 let handler = rig.handler.clone();
18597 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18598 std::future::poll_fn(|cx| {
18599 assert!(open.as_mut().poll(cx).is_pending());
18600 std::task::Poll::Ready(())
18601 })
18602 .await;
18603 rig.sync(vec![
18604 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
18605 ])
18606 .await;
18607 rx.try_recv().unwrap();
18608 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18609 let bind = std::future::poll_fn(|cx| {
18610 assert!(open.as_mut().poll(cx).is_pending());
18611 module_rx.poll_recv(cx)
18612 })
18613 .await
18614 .unwrap()
18615 .frame;
18616 handler
18617 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
18618 .await
18619 .unwrap();
18620 let replies = tokio::time::timeout(Duration::from_secs(2), open)
18621 .await
18622 .unwrap()
18623 .unwrap();
18624 assert_eq!(replies.len(), 1);
18625 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
18628 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18629 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
18630 let route = rig
18631 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
18632 .await;
18633 assert!(rig.live(&route));
18634 }
18635
18636 #[tokio::test]
18637 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
18638 {
18639 let mut rig = rig_with_flow_support(false).await;
18640 rig.sync(vec![session(1)]).await;
18641 let route = rig
18642 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18643 .await;
18644 assert!(rig.live(&route));
18645 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
18646 let mut record = session(1);
18647 record.attributes.flow_id = Some("flow:7".to_string());
18648 rig.sync(vec![record]).await;
18649 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
18650 assert!(rig.live(&unscoped));
18651 assert_eq!(unscoped.stamp(), None);
18652 }
18653
18654 #[tokio::test]
18655 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
18656 let mut rig = rig().await;
18657 let mut record = session(1);
18658 record.attributes.flow_id = Some("flow:7".to_string());
18659 rig.sync(vec![record.clone()]).await;
18660 let before = rig.forwarding.published_scope_tag(OWNER, "s");
18661 let mut owner_route = rig
18662 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
18663 .await;
18664 let mut carrier_route = rig
18665 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
18666 .await;
18667 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18668 rig.sync(vec![record.clone()]).await;
18669 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
18670 assert!(rig.live(&owner_route) && owner_route.untouched());
18671 assert!(rig.live(&carrier_route) && carrier_route.untouched());
18672
18673 record.attributes.flow_id = Some("flow:8".to_string());
18674 rig.sync(vec![record]).await;
18675 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
18676 let before = before.unwrap();
18677 assert_eq!(after.scope_epoch, before.scope_epoch);
18678 assert!(after.version > before.version);
18679 for route in [&mut owner_route, &mut carrier_route] {
18680 assert!(!rig.live(route));
18681 assert_eq!(
18682 route.closed_reason(),
18683 RouteCloseReason::ScopeDelegationChanged
18684 );
18685 }
18686 assert!(rig.live(&unscoped) && unscoped.untouched());
18687 for target in [PLEXUS, OTHER] {
18690 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
18691 let goodbye = module_rx
18692 .try_recv()
18693 .expect("the provider sees the drain")
18694 .frame;
18695 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
18696 assert!(module_rx.try_recv().is_err());
18697 }
18698 let rebound = rig
18699 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18700 .await;
18701 assert_eq!(
18702 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
18703 Some("flow:8")
18704 );
18705 }
18706
18707 #[tokio::test]
18712 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
18713 let mut rig = rig().await;
18714 rig.sync(vec![session(1)]).await;
18715 let mut cotenant = rig
18716 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
18717 .await;
18718
18719 let mut changed = session(1);
18720 changed.child_owners.push(Principal::Reserved {
18721 module_id: MAGIC.to_string(),
18722 });
18723 let mut ended = None;
18724 for (code, next) in [
18725 (error_codes::SCOPE_CHANGED, vec![changed]),
18726 (error_codes::SCOPE_ENDED, Vec::new()),
18727 ] {
18728 let relayed = rig
18729 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18730 .await;
18731 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
18732 ended = Some(next.is_empty());
18733 rig.sync(next).await;
18734 rig.ack(&relayed).await;
18735 let replies = relayed.task.await.unwrap();
18736 assert_eq!(replies.len(), 1, "{replies:?}");
18737 assert_eq!(parse_error(&replies[0])["code"], code);
18738 assert_eq!(
18739 subc_protocol::error_codes::is_retryable_route_open(code),
18740 code == error_codes::SCOPE_CHANGED
18741 );
18742 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18743 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18745 let mut goodbyes = Vec::new();
18748 while let Ok(outbound) = plexus_rx.try_recv() {
18749 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
18750 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
18751 }
18752 assert!(
18753 goodbyes.contains(&(bind_channel, bind_epoch)),
18754 "{goodbyes:?}"
18755 );
18756 assert!(rig
18757 .handler
18758 .registry
18759 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
18760 .unwrap()
18761 .is_some());
18762 }
18763 assert_eq!(ended, Some(true));
18764 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
18767 }
18768
18769 #[tokio::test]
18772 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
18773 struct Case {
18774 name: &'static str,
18775 change: fn(&mut ScopeRecord),
18776 closed: [Option<RouteCloseReason>; 4],
18779 }
18780 use RouteCloseReason::*;
18781 let cases = [
18782 Case {
18783 name: "a carrier entry removed",
18784 change: |r| {
18785 r.carriers.retain(|c| {
18786 c.principal
18787 != Principal::Reserved {
18788 module_id: AFT.to_string(),
18789 }
18790 })
18791 },
18792 closed: [None, Some(ScopeCarrierRemoved), None, None],
18793 },
18794 Case {
18795 name: "a target removed from a carrier",
18796 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
18797 closed: [None, None, None, Some(ScopeCarrierRemoved)],
18798 },
18799 Case {
18800 name: "a bare carrier narrowed to targets",
18801 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
18802 closed: [None, Some(ScopeCarrierRemoved), None, None],
18803 },
18804 Case {
18805 name: "delegates turned off",
18806 change: |r| r.attributes.delegates = false,
18807 closed: [Some(ScopeDelegationChanged); 4],
18808 },
18809 Case {
18810 name: "agent_id changed",
18811 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
18812 closed: [Some(ScopeDelegationChanged); 4],
18813 },
18814 Case {
18815 name: "a carrier added, child owners changed, the record re-sent",
18816 change: |r| {
18817 r.carriers.push(carrier(MAGIC, None));
18818 r.child_owners.push(Principal::Reserved {
18819 module_id: MAGIC.to_string(),
18820 });
18821 },
18822 closed: [None; 4],
18823 },
18824 Case {
18825 name: "a target added",
18826 change: |r| {
18827 r.carriers[1]
18828 .targets
18829 .as_mut()
18830 .unwrap()
18831 .push("third".to_string())
18832 },
18833 closed: [None; 4],
18834 },
18835 Case {
18836 name: "delegates turned on",
18837 change: |r| r.attributes.delegates = true,
18838 closed: [None; 4],
18839 },
18840 ];
18841 for case in cases {
18842 let mut rig = rig().await;
18843 rig.sync(vec![session(1)]).await;
18844 let scope = || Some(rig_selector("s", Some(1)));
18845 let mut routes = [
18846 rig.bound(Some(OWNER), PLEXUS, scope()).await,
18847 rig.bound(Some(AFT), PLEXUS, scope()).await,
18848 rig.bound(Some(BROCA), PLEXUS, scope()).await,
18849 rig.bound(Some(BROCA), OTHER, scope()).await,
18850 ];
18851 let mut record = session(1);
18852 (case.change)(&mut record);
18853 rig.sync(vec![record]).await;
18854 for (index, expected) in case.closed.iter().enumerate() {
18855 let route = &mut routes[index];
18856 match expected {
18857 Some(reason) => {
18858 assert!(!rig.live(route), "{}: route {index} still live", case.name);
18859 assert_eq!(
18860 route.closed_reason(),
18861 *reason,
18862 "{}: route {index}",
18863 case.name
18864 );
18865 }
18866 None => {
18867 assert!(rig.live(route), "{}: route {index} closed", case.name);
18868 assert!(
18869 route.untouched(),
18870 "{}: route {index} was told something",
18871 case.name
18872 );
18873 }
18874 }
18875 }
18876 }
18877 }
18878
18879 #[tokio::test]
18880 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
18881 for next in [Vec::new(), vec![session(2)]] {
18883 let mut rig = rig().await;
18884 rig.sync(vec![session(1)]).await;
18885 let mut route = rig
18886 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18887 .await;
18888 rig.sync(next).await;
18889 assert!(!rig.live(&route));
18890 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
18891 }
18892
18893 let mut rig = rig().await;
18896 let mut child = session(1);
18897 child.scope_ref = "child".to_string();
18898 child.kind = ScopeKind::Worker;
18899 child.parent = Some(ScopeParent::new(
18900 Principal::Reserved {
18901 module_id: OWNER.to_string(),
18902 },
18903 "s".to_string(),
18904 1,
18905 ));
18906 rig.sync(vec![session(1), child.clone()]).await;
18907 let mut child_route = rig
18908 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
18909 .await;
18910 assert_eq!(
18911 child_route.stamp().unwrap().parent_state,
18912 Some(ParentState::Linked)
18913 );
18914 rig.sync(vec![child]).await;
18915 assert!(!rig.live(&child_route));
18916 assert_eq!(
18917 child_route.closed_reason(),
18918 RouteCloseReason::ScopeParentEnded
18919 );
18920 }
18921
18922 #[tokio::test]
18923 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
18924 ) {
18925 let mut rig = rig().await;
18926 rig.sync(vec![session(1)]).await;
18927 let mut route = rig
18928 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18929 .await;
18930 let before = rig.forwarding.published_scope_tag(OWNER, "s");
18931 rig.sync(vec![session(1)]).await;
18932 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
18933 assert!(rig.live(&route) && route.untouched());
18934
18935 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
18939 .forwarding
18940 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
18941 .unwrap()
18942 else {
18943 panic!("the route is bound");
18944 };
18945 binding.flow.acquire_tagged(9, false).await.unwrap();
18946 let mut widened = session(1);
18947 widened.carriers.push(carrier(MAGIC, None));
18948 rig.sync(vec![widened]).await;
18949 assert!(rig.live(&route) && route.untouched());
18950 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18951 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
18952 assert_eq!(binding.flow.in_flight(), 1);
18955 binding
18956 .flow
18957 .acquire_tagged(10, false)
18958 .await
18959 .expect("the flow is still open");
18960 }
18961
18962 #[tokio::test]
18965 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
18966 let mut rig = rig().await;
18967 rig.sync(vec![session(1)]).await;
18968 let mut on_incumbent = rig
18969 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18970 .await;
18971
18972 let (candidate, _candidate_rx) = wide_ctx(9);
18975 let registration = rig
18976 .handler
18977 .registry
18978 .register_candidate_with_control_ops(
18979 manifest(PLEXUS, PROTOCOL_VERSION),
18980 PROTOCOL_VERSION,
18981 candidate.connection_id,
18982 module_baseline_control_ops(),
18983 )
18984 .unwrap();
18985 rig.forwarding
18986 .register_candidate_module_connection(
18987 candidate.connection_id,
18988 PLEXUS.to_string(),
18989 PROTOCOL_VERSION,
18990 manifest_concurrency(®istration.manifest),
18991 candidate.egress.clone(),
18992 )
18993 .unwrap();
18994 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
18995 rig.handler
18996 .registry
18997 .promote_candidate(PLEXUS)
18998 .unwrap()
18999 .unwrap();
19000 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
19001
19002 rig.sync(Vec::new()).await;
19003 assert!(!rig.live(&on_incumbent));
19004 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
19005 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19006 let goodbye = incumbent_rx
19007 .try_recv()
19008 .expect("the superseded endpoint is told")
19009 .frame;
19010 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19011 }
19012 }
19013}
19014
19015#[cfg(test)]
19016mod concurrency_default_exposure_tests {
19017 use super::*;
19018
19019 fn hello_body(role_json: &str) -> Vec<u8> {
19020 format!(
19021 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
19022 )
19023 .into_bytes()
19024 }
19025
19026 fn manifest_from(body: &[u8]) -> ModuleManifest {
19027 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
19028 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
19029 .expect("manifest parses")
19030 }
19031
19032 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
19033
19034 #[test]
19035 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
19036 let body = hello_body(&format!(
19037 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
19038 ));
19039 let manifest = manifest_from(&body);
19040 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
19043 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
19044 }
19045
19046 #[test]
19047 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
19048 let body = hello_body(&format!(
19049 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
19050 ));
19051 let manifest = manifest_from(&body);
19052 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
19053 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
19054 }
19055
19056 #[test]
19057 fn non_management_roles_are_never_reported() {
19058 let body = hello_body(
19059 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
19060 );
19061 let manifest = manifest_from(&body);
19062 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
19063 }
19064}